Merge pull request #4425 from iptv-org/delete-index-m3u

Delete index.m3u
This commit is contained in:
Dum4G 2021-09-09 02:34:37 +03:00 committed by GitHub
commit deca148fe9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 96 additions and 553 deletions

View File

@ -17,176 +17,28 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
branch: 'bot/auto-update'
format:
create-matrix:
runs-on: ubuntu-latest
needs: create-branch
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/auto-update
- name: Install Dependencies
run: npm install
- name: Create Matrix
id: set-matrix
run: node scripts/create-matrix.js
format:
runs-on: ubuntu-latest
needs: create-matrix
continue-on-error: true
strategy:
fail-fast: false
matrix:
country:
[
ad,
ae,
af,
ag,
al,
am,
ao,
ar,
at,
au,
aw,
az,
ba,
bb,
bd,
be,
bf,
bg,
bh,
bn,
bo,
br,
bs,
by,
ca,
cd,
cg,
ch,
ci,
cl,
cm,
cn,
co,
cr,
cu,
cw,
cy,
cz,
de,
dk,
do,
dz,
ec,
ee,
eg,
es,
et,
fi,
fj,
fo,
fr,
pf,
ge,
gh,
gm,
gn,
gp,
gq,
gr,
gt,
hk,
hn,
hr,
ht,
hu,
id,
ie,
il,
in,
iq,
ir,
is,
it,
jm,
jo,
jp,
ke,
kg,
kh,
kp,
kr,
kw,
kz,
la,
lb,
li,
lk,
lt,
lu,
lv,
ly,
ma,
mc,
md,
me,
mk,
ml,
mm,
mn,
mo,
mt,
mv,
mx,
my,
mz,
ne,
ng,
ni,
nl,
no,
np,
nz,
om,
pa,
pe,
ph,
pk,
pl,
pr,
ps,
pt,
py,
qa,
ro,
rs,
ru,
rw,
sa,
sd,
se,
sg,
si,
sk,
sl,
sm,
sn,
so,
sv,
sy,
th,
tj,
tm,
tn,
tr,
tt,
tw,
tz,
ua,
ug,
uk,
us,
uy,
uz,
va,
ve,
vi,
vn,
xk,
ye,
zm
]
matrix: ${{fromJSON(needs.create-matrix.outputs.matrix)}}
steps:
- name: Checkout
uses: actions/checkout@v2

View File

@ -152,15 +152,14 @@ http://example.com/stream.m3u8
- `.github/`
- `ISSUE_TEMPLATE/`: issue templates for this repository.
- `workflows/`
- `auto-update.yml`: GitHub Action that automatically updates all playlists every day.
- `check.yml`: GitHub Action that automatically checks every pull request for syntax errors.
- `auto-update.yml`: GitHub workflow that automatically updates all playlists every day.
- `check.yml`: GitHub workflow that automatically checks every pull request for syntax errors.
- `cleanup.yml`: GitHub workflow that automatically removes broken links every week.
- `CODE_OF_CONDUCT.md`: rules you shouldn't break if you don't want to get banned.
- `.readme/`
- `_categories.md`: automatically generated list of all categories and their corresponding playlists.
- `_countries.md`: automatically generated list of all countries and their corresponding playlists.
- `_languages.md`: automatically generated list of all languages and their corresponding playlists.
- `config.json`: config for the `markdown-include` package, which is used to compile everything into one `README.md` file.
- `preview.png`: image displayed in the `README.md`.
- `supported-categories.md`: list of supported categories.
- `supported-region-codes.md`: list of supported region codes.
- `template.md`: template for `README.md`.
- `channels/`: contains all channels broken down by the country from which they are broadcast.
@ -168,13 +167,14 @@ http://example.com/stream.m3u8
- `unsorted.m3u`: playlist with channels not yet sorted.
- `scripts/`
- `data/`: data used in scripts.
- `helpers/`: helper scripts used in GitHub Actions.
- `filter.js`: used within GitHub Action to remove blacklisted channels from playlists.
- `format.js`: used within GitHub Action to format channel descriptions.
- `generate.js`: used within GitHub Action to generate all additional playlists.
- `remove-duplicates.js`: used in GitHub Action to remove duplicates from the playlist.
- `sort.js`: used within GitHub Action to sort channels by name.
- `update-readme.js`: used within GitHub Action to update the `README.md` file.
- `helpers/`: helper scripts.
- `create-matrix.js`: used within GitHub workflow to create matrix of files to process.
- `filter.js`: used within GitHub workflow to remove blacklisted channels from playlists.
- `format.js`: used within GitHub workflow to format channel descriptions.
- `generate.js`: used within GitHub workflow to generate all additional playlists.
- `remove-broken-links.js`: used in GitHub workflow to remove broken links from the playlist.
- `remove-duplicates.js`: used in GitHub workflow to remove duplicates from the playlist.
- `sort.js`: used within GitHub workflow to sort channels by name.
- `update-readme.js`: used within GitHub workflow to update the `README.md` file.
- `CONTRIBUTING.md`: file you are currently reading.
- `index.m3u`: main playlist that contains links to all playlists in the `channels/` folder.
- `README.md`: project description generated from the contents of the `.readme/` folder.

323
index.m3u
View File

@ -1,323 +0,0 @@
#EXTM3U
#EXTINF:-1,Afghanistan
channels/af.m3u
#EXTINF:-1,Albania
channels/al.m3u
#EXTINF:-1,Algeria
channels/dz.m3u
#EXTINF:-1,Andorra
channels/ad.m3u
#EXTINF:-1,Angola
channels/ao.m3u
#EXTINF:-1,Antigua and Barbuda
channels/ag.m3u
#EXTINF:-1,Argentina
channels/ar.m3u
#EXTINF:-1,Armenia
channels/am.m3u
#EXTINF:-1,Aruba
channels/aw.m3u
#EXTINF:-1,Australia
channels/au.m3u
#EXTINF:-1,Austria
channels/at.m3u
#EXTINF:-1,Azerbaijan
channels/az.m3u
#EXTINF:-1,Bahamas
channels/bs.m3u
#EXTINF:-1,Bahrain
channels/bh.m3u
#EXTINF:-1,Bangladesh
channels/bd.m3u
#EXTINF:-1,Barbados
channels/bb.m3u
#EXTINF:-1,Belarus
channels/by.m3u
#EXTINF:-1,Belgium
channels/be.m3u
#EXTINF:-1,Bolivia
channels/bo.m3u
#EXTINF:-1,Bosnia and Herzegovina
channels/ba.m3u
#EXTINF:-1,Brazil
channels/br.m3u
#EXTINF:-1,Brunei
channels/bn.m3u
#EXTINF:-1,Bulgaria
channels/bg.m3u
#EXTINF:-1,Burkina Faso
channels/bf.m3u
#EXTINF:-1,Cambodia
channels/kh.m3u
#EXTINF:-1,Cameroon
channels/cm.m3u
#EXTINF:-1,Canada
channels/ca.m3u
#EXTINF:-1,Chile
channels/cl.m3u
#EXTINF:-1,China
channels/cn.m3u
#EXTINF:-1,Colombia
channels/co.m3u
#EXTINF:-1,Costa Rica
channels/cr.m3u
#EXTINF:-1,Croatia
channels/hr.m3u
#EXTINF:-1,Cuba
channels/cu.m3u
#EXTINF:-1,Curaçao
channels/cw.m3u
#EXTINF:-1,Cyprus
channels/cy.m3u
#EXTINF:-1,Czech Republic
channels/cz.m3u
#EXTINF:-1,Democratic Republic of the Congo
channels/cd.m3u
#EXTINF:-1,Republic of the Congo
channels/cg.m3u
#EXTINF:-1,Denmark
channels/dk.m3u
#EXTINF:-1,Dominican Republic
channels/do.m3u
#EXTINF:-1,Ecuador
channels/ec.m3u
#EXTINF:-1,Egypt
channels/eg.m3u
#EXTINF:-1,El Salvador
channels/sv.m3u
#EXTINF:-1,Equatorial Guinea
channels/gq.m3u
#EXTINF:-1,Estonia
channels/ee.m3u
#EXTINF:-1,Ethiopia
channels/et.m3u
#EXTINF:-1,Faroe Islands
channels/fo.m3u
#EXTINF:-1,Finland
channels/fi.m3u
#EXTINF:-1,Fiji
channels/fj.m3u
#EXTINF:-1,France
channels/fr.m3u
#EXTINF:-1,French Polynesia
channels/pf.m3u
#EXTINF:-1,Gambia
channels/gm.m3u
#EXTINF:-1,Georgia
channels/ge.m3u
#EXTINF:-1,Germany
channels/de.m3u
#EXTINF:-1,Ghana
channels/gh.m3u
#EXTINF:-1,Greece
channels/gr.m3u
#EXTINF:-1,Guadeloupe
channels/gp.m3u
#EXTINF:-1,Guatemala
channels/gt.m3u
#EXTINF:-1,Guinea
channels/gn.m3u
#EXTINF:-1,Haiti
channels/ht.m3u
#EXTINF:-1,Honduras
channels/hn.m3u
#EXTINF:-1,Hong Kong
channels/hk.m3u
#EXTINF:-1,Hungary
channels/hu.m3u
#EXTINF:-1,Iceland
channels/is.m3u
#EXTINF:-1,India
channels/in.m3u
#EXTINF:-1,Indonesia
channels/id.m3u
#EXTINF:-1,Iran
channels/ir.m3u
#EXTINF:-1,Iraq
channels/iq.m3u
#EXTINF:-1,Ireland
channels/ie.m3u
#EXTINF:-1,Israel
channels/il.m3u
#EXTINF:-1,Italy
channels/it.m3u
#EXTINF:-1,Ivory Coast
channels/ci.m3u
#EXTINF:-1,Jamaica
channels/jm.m3u
#EXTINF:-1,Japan
channels/jp.m3u
#EXTINF:-1,Jordan
channels/jo.m3u
#EXTINF:-1,Kazakhstan
channels/kz.m3u
#EXTINF:-1,Kenya
channels/ke.m3u
#EXTINF:-1,Kosovo
channels/xk.m3u
#EXTINF:-1,Kuwait
channels/kw.m3u
#EXTINF:-1,Kyrgyzstan
channels/kg.m3u
#EXTINF:-1,Laos
channels/la.m3u
#EXTINF:-1,Latvia
channels/lv.m3u
#EXTINF:-1,Lebanon
channels/lb.m3u
#EXTINF:-1,Libya
channels/ly.m3u
#EXTINF:-1,Liechtenstein
channels/li.m3u
#EXTINF:-1,Lithuania
channels/lt.m3u
#EXTINF:-1,Luxembourg
channels/lu.m3u
#EXTINF:-1,Macau
channels/mo.m3u
#EXTINF:-1,Malaysia
channels/my.m3u
#EXTINF:-1,Maldives
channels/mv.m3u
#EXTINF:-1,Mali
channels/ml.m3u
#EXTINF:-1,Malta
channels/mt.m3u
#EXTINF:-1,Mexico
channels/mx.m3u
#EXTINF:-1,Moldova
channels/md.m3u
#EXTINF:-1,Monaco
channels/mc.m3u
#EXTINF:-1,Mongolia
channels/mn.m3u
#EXTINF:-1,Montenegro
channels/me.m3u
#EXTINF:-1,Morocco
channels/ma.m3u
#EXTINF:-1,Mozambique
channels/mz.m3u
#EXTINF:-1,Myanmar
channels/mm.m3u
#EXTINF:-1,Nepal
channels/np.m3u
#EXTINF:-1,Netherlands
channels/nl.m3u
#EXTINF:-1,New Zealand
channels/nz.m3u
#EXTINF:-1,Nicaragua
channels/ni.m3u
#EXTINF:-1,Niger
channels/ne.m3u
#EXTINF:-1,Nigeria
channels/ng.m3u
#EXTINF:-1,North Korea
channels/kp.m3u
#EXTINF:-1,North Macedonia
channels/mk.m3u
#EXTINF:-1,Norway
channels/no.m3u
#EXTINF:-1,Oman
channels/om.m3u
#EXTINF:-1,Pakistan
channels/pk.m3u
#EXTINF:-1,Palestine
channels/ps.m3u
#EXTINF:-1,Panama
channels/pa.m3u
#EXTINF:-1,Paraguay
channels/py.m3u
#EXTINF:-1,Peru
channels/pe.m3u
#EXTINF:-1,Philippines
channels/ph.m3u
#EXTINF:-1,Poland
channels/pl.m3u
#EXTINF:-1,Portugal
channels/pt.m3u
#EXTINF:-1,Puerto Rico
channels/pr.m3u
#EXTINF:-1,Qatar
channels/qa.m3u
#EXTINF:-1,Romania
channels/ro.m3u
#EXTINF:-1,Russia
channels/ru.m3u
#EXTINF:-1,Rwanda
channels/rw.m3u
#EXTINF:-1,San Marino
channels/sm.m3u
#EXTINF:-1,Saudi Arabia
channels/sa.m3u
#EXTINF:-1,Senegal
channels/sn.m3u
#EXTINF:-1,Serbia
channels/rs.m3u
#EXTINF:-1,Sierra Leone
channels/sl.m3u
#EXTINF:-1,Singapore
channels/sg.m3u
#EXTINF:-1,Slovakia
channels/sk.m3u
#EXTINF:-1,Slovenia
channels/si.m3u
#EXTINF:-1,Somalia
channels/so.m3u
#EXTINF:-1,South Korea
channels/kr.m3u
#EXTINF:-1,Spain
channels/es.m3u
#EXTINF:-1,Sri Lanka
channels/lk.m3u
#EXTINF:-1,Sudan
channels/sd.m3u
#EXTINF:-1,Sweden
channels/se.m3u
#EXTINF:-1,Switzerland
channels/ch.m3u
#EXTINF:-1,Syria
channels/sy.m3u
#EXTINF:-1,Taiwan
channels/tw.m3u
#EXTINF:-1,Tajikistan
channels/tj.m3u
#EXTINF:-1,Tanzania
channels/tz.m3u
#EXTINF:-1,Thailand
channels/th.m3u
#EXTINF:-1,Trinidad and Tobago
channels/tt.m3u
#EXTINF:-1,Tunisia
channels/tn.m3u
#EXTINF:-1,Turkey
channels/tr.m3u
#EXTINF:-1,Turkmenistan
channels/tm.m3u
#EXTINF:-1,Uganda
channels/ug.m3u
#EXTINF:-1,Ukraine
channels/ua.m3u
#EXTINF:-1,United Arab Emirates
channels/ae.m3u
#EXTINF:-1,United Kingdom
channels/uk.m3u
#EXTINF:-1,United States
channels/us.m3u
#EXTINF:-1,Uruguay
channels/uy.m3u
#EXTINF:-1,Uzbekistan
channels/uz.m3u
#EXTINF:-1,Vatican City
channels/va.m3u
#EXTINF:-1,Venezuela
channels/ve.m3u
#EXTINF:-1,Vietnam
channels/vn.m3u
#EXTINF:-1,Virgin Islands of the United States
channels/vi.m3u
#EXTINF:-1,Yemen
channels/ye.m3u
#EXTINF:-1,Zambia
channels/zm.m3u
#EXTINF:-1,Unsorted
channels/unsorted.m3u

9
scripts/create-matrix.js Normal file
View File

@ -0,0 +1,9 @@
const file = require('./helpers/file')
file.list().then(files => {
files = files.filter(file => file !== 'channels/unsorted.m3u')
const country = files.map(file => file.replace(/channels\/|\.m3u/gi, ''))
const matrix = { country }
const output = `::set-output name=matrix::${JSON.stringify(matrix)}`
console.log(output)
})

View File

@ -1,16 +1,17 @@
const blacklist = require('./data/blacklist.json')
const parser = require('./helpers/parser')
const file = require('./helpers/file')
const log = require('./helpers/log')
async function main() {
log.start()
log.print(`Parsing 'index.m3u'...`)
const playlists = parser.parseIndex()
for (const playlist of playlists) {
log.print(`\nProcessing '${playlist.url}'...`)
const files = await file.list()
if (!files.length) log.print(`No files is selected\n`)
for (const file of files) {
log.print(`\nProcessing '${file}'...`)
await parser
.parsePlaylist(playlist.url)
.parsePlaylist(file)
.then(removeBlacklisted)
.then(p => p.save())
}

View File

@ -27,11 +27,12 @@ let buffer, origins
async function main() {
log.start()
let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
if (!playlists.length) log.print(`No playlist is selected\n`)
for (const playlist of playlists) {
await parser.parsePlaylist(playlist.url).then(updatePlaylist).then(savePlaylist)
const include = config.country.split(',').filter(i => i)
const exclude = config.exclude.split(',').filter(i => i)
let files = await file.list(include, exclude)
if (!files.length) log.print(`No files is selected\n`)
for (const file of files) {
await parser.parsePlaylist(file).then(updatePlaylist).then(savePlaylist)
}
log.finish()

View File

@ -1,13 +1,14 @@
const categories = require('../data/categories')
const parser = require('./parser')
const utils = require('./utils')
const file = require('./file')
const db = {}
db.load = async function () {
const items = parser.parseIndex()
for (const item of items) {
const playlist = await parser.parsePlaylist(item.url)
let files = await file.list()
for (const file of files) {
const playlist = await parser.parsePlaylist(file)
db.playlists.add(playlist)
for (const channel of playlist.channels) {
db.channels.add(channel)

View File

@ -1,10 +1,29 @@
const markdownInclude = require('markdown-include')
const path = require('path')
const glob = require('glob')
const fs = require('fs')
const rootPath = path.resolve(__dirname) + '/../../'
const file = {}
file.list = function (include = [], exclude = []) {
return new Promise(resolve => {
glob('channels/**/*.m3u', function (err, files) {
if (include.length) {
include = include.map(filename => `channels/${filename}.m3u`)
files = files.filter(filename => include.includes(filename))
}
if (exclude.length) {
exclude = exclude.map(filename => `channels/${filename}.m3u`)
files = files.filter(filename => !exclude.includes(filename))
}
resolve(files)
})
})
}
file.getFilename = function (filename) {
return path.parse(filename).name
}

View File

@ -5,13 +5,6 @@ const file = require('./file')
const parser = {}
parser.parseIndex = function () {
const content = file.read('index.m3u')
const result = playlistParser.parse(content)
return result.items
}
parser.parsePlaylist = async function (url) {
const content = file.read(url)
const result = playlistParser.parse(content)

View File

@ -74,22 +74,6 @@ utils.removeProtocol = function (string) {
return string.replace(/(^\w+:|^)\/\//, '')
}
utils.filterPlaylists = function (arr, include = '', exclude = '') {
if (include) {
const included = include.split(',').map(filename => `channels/${filename}.m3u`)
return arr.filter(i => included.indexOf(i.url) > -1)
}
if (exclude) {
const excluded = exclude.split(',').map(filename => `channels/${filename}.m3u`)
return arr.filter(i => excluded.indexOf(i.url) === -1)
}
return arr
}
utils.sleep = function (ms) {
return function (x) {
return new Promise(resolve => setTimeout(() => resolve(x), ms))

View File

@ -1,15 +1,17 @@
const parser = require('./helpers/parser')
const file = require('./helpers/file')
const log = require('./helpers/log')
async function main() {
log.start()
log.print(`Parsing 'index.m3u'...`)
const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
for (const playlist of playlists) {
log.print(`\nProcessing '${playlist.url}'...`)
let files = await file.list()
if (!files.length) log.print(`No files is selected\n`)
files = files.filter(file => file !== 'channels/unsorted.m3u')
for (const file of files) {
log.print(`\nProcessing '${file}'...`)
await parser
.parsePlaylist(playlist.url)
.parsePlaylist(file)
.then(removeBrokenLinks)
.then(p => p.save())
}

View File

@ -1,5 +1,6 @@
const parser = require('./helpers/parser')
const utils = require('./helpers/utils')
const file = require('./helpers/file')
const log = require('./helpers/log')
let globalBuffer = []
@ -7,18 +8,19 @@ let globalBuffer = []
async function main() {
log.start()
log.print(`Parsing 'index.m3u'...`)
const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
for (const playlist of playlists) {
log.print(`\nProcessing '${playlist.url}'...`)
let files = await file.list()
if (!files.length) log.print(`No files is selected\n`)
files = files.filter(file => file !== 'channels/unsorted.m3u')
for (const file of files) {
log.print(`\nProcessing '${file}'...`)
await parser
.parsePlaylist(playlist.url)
.parsePlaylist(file)
.then(addToGlobalBuffer)
.then(removeDuplicates)
.then(p => p.save())
}
if (playlists.length) {
if (files.length) {
log.print(`\nProcessing 'channels/unsorted.m3u'...`)
await parser
.parsePlaylist('channels/unsorted.m3u')

View File

@ -1,16 +1,18 @@
const parser = require('./helpers/parser')
const utils = require('./helpers/utils')
const file = require('./helpers/file')
const log = require('./helpers/log')
async function main() {
log.start()
log.print(`Parsing 'index.m3u'...`)
let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
for (const playlist of playlists) {
log.print(`\nProcessing '${playlist.url}'...`)
let files = await file.list()
if (!files.length) log.print(`No files is selected\n`)
files = files.filter(file => file !== 'channels/unsorted.m3u')
for (const file of files) {
log.print(`\nProcessing '${file}'...`)
await parser
.parsePlaylist(playlist.url)
.parsePlaylist(file)
.then(sortChannels)
.then(p => p.save())
}