diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 4ed5160ca8..e568487093 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -4,103 +4,382 @@ on: schedule: - cron: '0 0 * * *' jobs: - remove-duplicates: + create-branch: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 - - name: Install Dependencies - run: npm install - - name: Remove Duplicates - run: node scripts/remove-duplicates.js - - name: Upload Artifact - uses: actions/upload-artifact@v2 with: - name: channels - path: channels/ - filter: - runs-on: ubuntu-latest - needs: remove-duplicates - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Download Artifacts - uses: actions/download-artifact@v2 - - name: Install Dependencies - run: npm install - - name: Filter Playlists - run: node scripts/filter.js - - name: Upload Artifact - uses: actions/upload-artifact@v2 + ref: ${{ github.ref }} + - name: Create Branch + uses: peterjgrainger/action-create-branch@v2.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - name: channels - path: channels/ + branch: 'bot/auto-update' format: runs-on: ubuntu-latest - needs: filter + needs: create-branch steps: - name: Checkout uses: actions/checkout@v2 - - name: Download Artifacts - uses: actions/download-artifact@v2 + with: + ref: bot/auto-update - name: Install Dependencies run: npm install - name: Format Playlists run: node scripts/format.js - - name: Upload Artifact - uses: actions/upload-artifact@v2 + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 with: - name: channels - path: channels/ - generate: + commit_message: '[Bot] Formate playlists' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + remove-duplicates: runs-on: ubuntu-latest needs: format steps: - name: Checkout uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Remove Duplicates + run: node scripts/remove-duplicates.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Remove duplicates' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + sort: + runs-on: ubuntu-latest + needs: remove-duplicates + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Sort Channels + run: node scripts/sort.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Sort channels' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + filter: + runs-on: ubuntu-latest + needs: sort + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Filter Playlists + run: node scripts/filter.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Filter channels' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + detect-resolution: + runs-on: ubuntu-latest + needs: filter + continue-on-error: true + strategy: + fail-fast: false + matrix: + country: + [ + ad, + ae, + af, + ag, + al, + am, + an, + ao, + ar, + at, + au, + aw, + az, + ba, + bb, + bd, + be, + bf, + bg, + bh, + bn, + bo, + br, + bs, + by, + ca, + cd, + cg, + ch, + ci, + cl, + cm, + cn, + co, + cr, + cu, + cw, + cy, + cz, + de, + dk, + do, + dz, + ec, + ee, + eg, + es, + et, + fi, + fj, + fo, + fr, + ge, + gh, + gm, + gn, + gp, + gq, + gr, + gt, + hk, + hn, + hr, + ht, + hu, + id, + ie, + il, + in, + iq, + ir, + is, + it, + jm, + jo, + jp, + ke, + kg, + kh, + kp, + kr, + kw, + kz, + la, + lb, + li, + lk, + lt, + lu, + lv, + ly, + ma, + mc, + md, + me, + mk, + ml, + mm, + mn, + mo, + mt, + mv, + mx, + my, + mz, + ne, + ng, + ni, + nl, + no, + np, + nz, + om, + pa, + pe, + ph, + pk, + pl, + pr, + ps, + pt, + py, + qa, + ro, + rs, + ru, + rw, + sa, + sd, + se, + sg, + si, + sk, + sl, + sm, + sn, + so, + sv, + sy, + th, + tj, + tm, + tn, + tr, + tt, + tw, + tz, + ua, + ug, + uk, + us, + uy, + uz, + va, + ve, + vi, + vn, + xk, + ye, + zm + ] + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Detect Resolution + run: node scripts/detect-resolution.js --country=${{ matrix.country }} + - name: Upload Artifact + uses: actions/upload-artifact@v2 + with: + name: channels + path: channels/${{ matrix.country }}.m3u + commit-changes: + runs-on: ubuntu-latest + needs: detect-resolution + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update - name: Download Artifacts uses: actions/download-artifact@v2 + with: + name: channels + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Detect resolution' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + generate: + runs-on: ubuntu-latest + needs: commit-changes + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update - name: Install Dependencies run: npm install - name: Generate Playlists run: node scripts/generate.js + - name: Upload Artifact + uses: actions/upload-artifact@v2 + with: + name: gh-pages + path: .gh-pages/ + deploy: + if: ${{ github.ref == 'refs/heads/master' }} + runs-on: ubuntu-latest + needs: generate + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Download Artifacts + uses: actions/download-artifact@v2 + with: + name: gh-pages + - name: Generate Token + uses: tibdex/github-app-token@v1 + id: generate-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} - name: Deploy to GitHub Pages uses: JamesIves/github-pages-deploy-action@4.1.1 with: branch: gh-pages - folder: .gh-pages + folder: gh-pages + token: ${{ steps.generate-token.outputs.token }} + git-config-name: iptv-bot + git-config-email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit-message: '[Bot] Deploy to GitHub Pages' update-readme: runs-on: ubuntu-latest needs: generate steps: - name: Checkout uses: actions/checkout@v2 - - name: Download Artifacts - uses: actions/download-artifact@v2 + with: + ref: bot/auto-update - name: Install Dependencies run: npm install - name: Update README.md run: node scripts/update-readme.js - - name: Upload Artifact - uses: actions/upload-artifact@v2 + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 with: - name: README.md - path: README.md + commit_message: '[Bot] Update README.md' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: README.md pull-request: needs: update-readme runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 - - name: Download /channels - uses: actions/download-artifact@v2 with: - name: channels - path: channels/ - - name: Download README.md - uses: actions/download-artifact@v2 - with: - name: README.md + ref: bot/auto-update - name: Generate Token uses: tibdex/github-app-token@v1 id: generate-token @@ -109,26 +388,25 @@ jobs: private_key: ${{ secrets.APP_PRIVATE_KEY }} - name: Create Pull Request id: pr - uses: peter-evans/create-pull-request@v3 + uses: repo-sync/pull-request@v2 with: - title: '[Bot] Update playlists' - body: | - This pull request is created automatically by `auto-update` action. - commit-message: '[Bot] Update playlists' - committer: GitHub - branch: bot/auto-update - delete-branch: true - token: ${{ steps.generate-token.outputs.token }} + source_branch: 'bot/auto-update' + destination_branch: 'master' + pr_title: '[Bot] Update playlists' + pr_body: | + This pull request is created by [auto-update][1] workflow. + + [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} + github_token: ${{ steps.generate-token.outputs.token }} - name: Enable Pull Request Automerge - if: steps.pr.outputs.pull-request-operation == 'created' uses: peter-evans/enable-pull-request-automerge@v1 with: token: ${{ secrets.PAT }} - pull-request-number: ${{ steps.pr.outputs.pull-request-number }} + pull-request-number: ${{ steps.pr.outputs.pr_number }} merge-method: squash - name: Approve Pull Request - if: steps.pr.outputs.pull-request-operation == 'created' + if: github.ref == 'refs/heads/master' uses: juliangruber/approve-pull-request-action@v1 with: github-token: ${{ secrets.PAT }} - number: ${{ steps.pr.outputs.pull-request-number }} + number: ${{ steps.pr.outputs.pr_number }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3933d33a43..a111b67135 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -167,18 +167,15 @@ http://example.com/stream.m3u8 - ... - `unsorted.m3u`: playlist with channels not yet sorted. - `scripts/` - - `blacklist.json`: list of channels banned for addition to the repository. - - `categories.json`: list of supported categories. + - `helpers/`: helper scripts used in GitHub Actions. - `clean.js`: used in GitHub Action to check all links and remove broken ones. - - `db.js`: contains functions for retrieving and managing the channel list. + - `detect-resolution.js`: used in GitHub Action to detect resolution of the streams. - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists. - - `format.js`: used within GitHub Action to format channel descriptions and sort playlists. + - `format.js`: used within GitHub Action to format channel descriptions. - `generate.js`: used within GitHub Action to generate all additional playlists. - - `parser.js`: contains functions for parsing playlists. - - `regions.json`: list of supported region codes. - `remove-duplicates.js`: used in GitHub Action to remove duplicates from the playlist. + - `sort.js`: used within GitHub Action to sort channels by name. - `update-readme.js`: used within GitHub Action to update the `README.md` file. - - `utils.js`: contains functions that are used in other scripts. - `CONTRIBUTING.md`: file you are currently reading. - `index.m3u`: main playlist that contains links to all playlists in the `channels/` folder. - `README.md`: project description generated from the contents of the `.readme/` folder. diff --git a/channels/ir.m3u b/channels/ir.m3u index ba5e77d21b..0d46e0ddea 100644 --- a/channels/ir.m3u +++ b/channels/ir.m3u @@ -39,16 +39,16 @@ http://159.69.58.154/ekran/ekrantv.m3u8 https://faraztv.net/hls/stream.m3u8 #EXTINF:-1 tvg-id="Film1.ir" tvg-name="Film 1" tvg-country="IR" tvg-language="Persian" tvg-logo="" group-title="Movies",Film 1 http://159.69.58.154/film1/film1tv.m3u8 -#EXTINF:-1 tvg-id="GEM24b.ir" tvg-name="GEM 24b" tvg-country="IR" tvg-language="Persian" tvg-logo="https://www.gemonline.tv/Assets/channels-box/24b.png" group-title="Entertainment",GEM 24b [Geo-blocked] +#EXTINF:-1 tvg-id="GEM24b.ir" tvg-name="GEM 24b" tvg-country="IR" tvg-language="Persian" tvg-logo="https://www.gemonline.tv/Assets/channels-box/24b.png" group-title="Music",GEM 24b [Geo-blocked] #EXTVLCOPT:http-referrer=https://www.gemonline.tv/en-US/Live/Index?channelname=24b https://d2e40kvaojifd6.cloudfront.net/stream/24b/playlist.m3u8 #EXTINF:-1 tvg-id="GEMAcademy.ir" tvg-name="GEM Academy" tvg-country="IR" tvg-language="Persian" tvg-logo="https://www.gemonline.tv/Assets/channels-box/gemusa.png" group-title="Movies",GEM Academy [Geo-blocked] #EXTVLCOPT:http-referrer=https://www.gemonline.tv/en-US/Live/Index?channelname=gemusa https://d2e40kvaojifd6.cloudfront.net/stream/gem_usa/playlist.m3u8 -#EXTINF:-1 tvg-id="GEMArabia.ir" tvg-name="GEM Arabia" tvg-country="IR" tvg-language="Arabic" tvg-logo="https://www.gemonline.tv/Assets/channels-box/gem_arabia.png" group-title="Entertainment",GEM Arabia [Geo-blocked] +#EXTINF:-1 tvg-id="GEMArabia.ir" tvg-name="GEM Arabia" tvg-country="IR" tvg-language="Arabic" tvg-logo="https://www.gemonline.tv/Assets/channels-box/gem_arabia.png" group-title="Music",GEM Arabia [Geo-blocked] #EXTVLCOPT:http-referrer=https://www.gemonline.tv/en-US/Live/Index?channelname=gem_arabia https://d2e40kvaojifd6.cloudfront.net/stream/gem_arabia/playlist.m3u8 -#EXTINF:-1 tvg-id="GEMAZ.ir" tvg-name="GEM AZ" tvg-country="IR" tvg-language="Persian" tvg-logo="https://www.gemonline.tv/Assets/channels-box/gem_az.png" group-title="Entertainment",GEM AZ [Geo-blocked] +#EXTINF:-1 tvg-id="GEMAZ.ir" tvg-name="GEM AZ" tvg-country="IR" tvg-language="Persian" tvg-logo="https://www.gemonline.tv/Assets/channels-box/gem_az.png" group-title="Music",GEM AZ [Geo-blocked] #EXTVLCOPT:http-referrer=https://www.gemonline.tv/en-US/Live/Index?channelname=gem_az https://d2e40kvaojifd6.cloudfront.net/stream/gem_az/playlist.m3u8 #EXTINF:-1 tvg-id="GEMBollywood.ir" tvg-name="GEM Bollywood" tvg-country="IR" tvg-language="Persian" tvg-logo="https://www.gemonline.tv/Assets/channels-box/gembollywood.png" group-title="Movies",GEM Bollywood [Geo-blocked] @@ -87,7 +87,7 @@ https://d2e40kvaojifd6.cloudfront.net/stream/gem_life/playlist.m3u8 #EXTINF:-1 tvg-id="GEMMaxx.ir" tvg-name="GEM Maxx" tvg-country="IR" tvg-language="Persian" tvg-logo="https://www.gemonline.tv/Assets/channels-box/maxx.png?14" group-title="Entertainment",GEM Maxx [Geo-blocked] #EXTVLCOPT:http-referrer=https://www.gemonline.tv/en-US/Live/Index?channelname=maxx https://d2e40kvaojifd6.cloudfront.net/stream/maxx/playlist.m3u8 -#EXTINF:-1 tvg-id="GEMMifa.ir" tvg-name="GEM Mifa" tvg-country="IR" tvg-language="Persian" tvg-logo="https://www.gemonline.tv/Assets/channels-box/mifa.png" group-title="Entertainment",GEM Mifa [Geo-blocked] +#EXTINF:-1 tvg-id="GEMMifa.ir" tvg-name="GEM Mifa" tvg-country="IR" tvg-language="Persian" tvg-logo="https://www.gemonline.tv/Assets/channels-box/mifa.png" group-title="Music",GEM Mifa [Geo-blocked] #EXTVLCOPT:http-referrer=https://www.gemonline.tv/en-US/Live/Index?channelname=mifa https://d2e40kvaojifd6.cloudfront.net/stream/mifa/playlist.m3u8 #EXTINF:-1 tvg-id="GEMModernEconomy.ir" tvg-name="GEM ModernEconomy" tvg-country="IR" tvg-language="Persian" tvg-logo="https://www.gemonline.tv/Assets/channels-box/modern_economy.png" group-title="Documentary",GEM Modern Economy [Geo-blocked] diff --git a/scripts/clean.js b/scripts/clean.js index 9b4bd2095c..6b372cf2ab 100644 --- a/scripts/clean.js +++ b/scripts/clean.js @@ -1,14 +1,14 @@ const { program } = require('commander') -const parser = require('./parser') -const utils = require('./utils') -const axios = require('axios') const ProgressBar = require('progress') +const axios = require('axios') const https = require('https') const chalk = require('chalk') +const parser = require('./helpers/parser') +const utils = require('./helpers/utils') +const log = require('./helpers/log') program .usage('[OPTIONS]...') - .option('-d, --debug', 'Debug mode') .option('-c, --country ', 'Comma-separated list of country codes', '') .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') .option('--delay ', 'Delay between parser requests', 1000) @@ -16,8 +16,8 @@ program .parse(process.argv) const config = program.opts() - const offlineStatusCodes = [404, 410, 500, 501] +const ignore = ['Geo-blocked', 'Not 24/7'] const instance = axios.create({ timeout: config.timeout, maxContentLength: 200000, @@ -29,60 +29,39 @@ const instance = axios.create({ } }) -const ignore = ['Geo-blocked', 'Not 24/7'] - -const stats = { broken: 0 } +let broken = 0 async function main() { - console.info(`\nStarting...`) - console.time('Done in') - if (config.debug) { - console.info(chalk.yellow(`INFO: Debug mode enabled\n`)) - } - const playlists = parseIndex() + log.start() - for (const playlist of playlists) { - await loadPlaylist(playlist.url).then(checkStatus).then(savePlaylist).then(done) - } - - finish() -} - -function parseIndex() { - console.info(`Parsing 'index.m3u'...`) + log.print(`Parsing 'index.m3u'...`) let playlists = parser.parseIndex() playlists = utils.filterPlaylists(playlists, config.country, config.exclude) - console.info(`Found ${playlists.length} playlist(s)\n`) + for (const playlist of playlists) { + await parser + .parsePlaylist(playlist.url) + .then(checkStatus) + .then(p => p.save()) + } - return playlists -} - -async function loadPlaylist(url) { - console.info(`Processing '${url}'...`) - return parser.parsePlaylist(url) + log.finish() } async function checkStatus(playlist) { - let bar - if (!config.debug) { - bar = new ProgressBar(' Testing: [:bar] :current/:total (:percent) ', { - total: playlist.channels.length - }) - } - const results = [] + let bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, { + total: playlist.channels.length + }) + const channels = [] const total = playlist.channels.length for (const [index, channel] of playlist.channels.entries()) { const current = index + 1 const counter = chalk.gray(`[${current}/${total}]`) - if (bar) bar.tick() + bar.tick() if ( (channel.status && ignore.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())) || (!channel.url.startsWith('http://') && !channel.url.startsWith('https://')) ) { - results.push(channel) - if (config.debug) { - console.info(` ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`) - } + channels.push(channel) } else { const CancelToken = axios.CancelToken const source = CancelToken.source() @@ -94,55 +73,27 @@ async function checkStatus(playlist) { .get(channel.url, { cancelToken: source.token }) .then(() => { clearTimeout(timeout) - results.push(channel) - if (config.debug) { - console.info(` ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`) - } + channels.push(channel) }) .then(utils.sleep(config.delay)) .catch(err => { clearTimeout(timeout) if (err.response && offlineStatusCodes.includes(err.response.status)) { - if (config.debug) { - console.info(` ${counter} ${chalk.red('offline')} ${chalk.white(channel.url)}`) - } - stats.broken++ + broken++ } else { - results.push(channel) - if (config.debug) { - console.info(` ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`) - } + channels.push(channel) } }) } } - playlist.channels = results + if (playlist.channels.length !== channels.length) { + log.print(`File '${playlist.url}' has been updated\n`) + playlist.channels = channels + playlist.updated = true + } return playlist } -async function savePlaylist(playlist) { - const original = utils.readFile(playlist.url) - const output = playlist.toString({ raw: true }) - - if (original === output) { - console.info(`No changes have been made.`) - return false - } else { - utils.createFile(playlist.url, output) - console.info(`Playlist has been updated. Removed ${stats.broken} links.`) - } - - return true -} - -async function done() { - console.info(` `) -} - -function finish() { - console.timeEnd('Done in') -} - main() diff --git a/scripts/detect-resolution.js b/scripts/detect-resolution.js new file mode 100644 index 0000000000..5c01841f5d --- /dev/null +++ b/scripts/detect-resolution.js @@ -0,0 +1,110 @@ +const { program } = require('commander') +const ProgressBar = require('progress') +const axios = require('axios') +const https = require('https') +const parser = require('./helpers/parser') +const utils = require('./helpers/utils') +const log = require('./helpers/log') + +program + .usage('[OPTIONS]...') + .option('-c, --country ', 'Comma-separated list of country codes', '') + .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') + .option('--delay ', 'Delay between parser requests', 1000) + .option('--timeout ', 'Set timeout for each request', 5000) + .parse(process.argv) + +const config = program.opts() +const instance = axios.create({ + timeout: config.timeout, + maxContentLength: 200000, + httpsAgent: new https.Agent({ + rejectUnauthorized: false + }) +}) + +async function main() { + log.start() + + log.print(`Parsing 'index.m3u'...\n`) + let playlists = parser.parseIndex() + playlists = utils + .filterPlaylists(playlists, config.country, config.exclude) + .filter(i => i.url !== 'channels/unsorted.m3u') + + for (const playlist of playlists) { + await parser + .parsePlaylist(playlist.url) + .then(detectResolution) + .then(p => p.save()) + } + + log.finish() +} + +async function detectResolution(playlist) { + const channels = [] + const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { + total: playlist.channels.length + }) + let updated = false + for (const channel of playlist.channels) { + bar.tick() + if (!channel.resolution.height) { + const CancelToken = axios.CancelToken + const source = CancelToken.source() + const timeout = setTimeout(() => { + source.cancel() + }, config.timeout) + + const response = await instance + .get(channel.url, { cancelToken: source.token }) + .then(res => { + clearTimeout(timeout) + + return res + }) + .then(utils.sleep(config.delay)) + .catch(err => { + clearTimeout(timeout) + }) + + if (response && response.status === 200) { + if (/^#EXTM3U/.test(response.data)) { + const resolution = parseResolution(response.data) + if (resolution) { + channel.resolution = resolution + updated = true + } + } + } + } + + channels.push(channel) + } + + if (updated) { + log.print(`File '${playlist.url}' has been updated\n`) + playlist.channels = channels + playlist.updated = true + } + + return playlist +} + +function parseResolution(string) { + const regex = /RESOLUTION=(\d+)x(\d+)/gm + const match = string.matchAll(regex) + const arr = Array.from(match).map(m => ({ + width: parseInt(m[1]), + height: parseInt(m[2]) + })) + + return arr.length + ? arr.reduce(function (prev, current) { + return prev.height > current.height ? prev : current + }) + : undefined +} + +main() diff --git a/scripts/filter.js b/scripts/filter.js index ec90cab56b..da0d30b7f8 100644 --- a/scripts/filter.js +++ b/scripts/filter.js @@ -1,67 +1,43 @@ -const parser = require('./parser') -const utils = require('./utils') -const blacklist = require('./blacklist.json') +const blacklist = require('./helpers/blacklist.json') +const parser = require('./helpers/parser') +const log = require('./helpers/log') async function main() { - const playlists = parseIndex() + log.start() + + log.print(`Parsing 'index.m3u'...`) + const playlists = parser.parseIndex() for (const playlist of playlists) { - await loadPlaylist(playlist.url).then(removeBlacklisted).then(savePlaylist).then(done) + log.print(`\nProcessing '${playlist.url}'...`) + await parser + .parsePlaylist(playlist.url) + .then(removeBlacklisted) + .then(p => p.save()) } - finish() -} - -function parseIndex() { - console.info(`Parsing 'index.m3u'...`) - let playlists = parser.parseIndex() - console.info(`Found ${playlists.length} playlist(s)\n`) - - return playlists -} - -async function loadPlaylist(url) { - console.info(`Processing '${url}'...`) - return parser.parsePlaylist(url) + log.print('\n') + log.finish() } async function removeBlacklisted(playlist) { - console.info(` Looking for blacklisted channels...`) - playlist.channels = playlist.channels.filter(channel => { - return !blacklist.find(i => { - const channelName = channel.name.toLowerCase() - return ( - (i.name.toLowerCase() === channelName || - i.aliases.map(i => i.toLowerCase()).includes(channelName)) && - i.country === channel.filename - ) + const channels = playlist.channels.filter(channel => { + return !blacklist.find(item => { + const hasSameName = + item.name.toLowerCase() === channel.name.toLowerCase() || + item.aliases.map(alias => alias.toLowerCase()).includes(channel.name.toLowerCase()) + const fromSameCountry = channel.countries.find(c => c.code === item.country) + + return hasSameName && fromSameCountry }) }) + if (playlist.channels.length !== channels.length) { + log.print(`updated`) + playlist.channels = channels + playlist.updated = true + } + return playlist } -async function savePlaylist(playlist) { - console.info(` Saving playlist...`) - const original = utils.readFile(playlist.url) - const output = playlist.toString({ raw: true }) - - if (original === output) { - console.info(`No changes have been made.`) - return false - } else { - utils.createFile(playlist.url, output) - console.info(`Playlist has been updated.`) - } - - return true -} - -async function done() { - console.info(` `) -} - -function finish() { - console.info('Done.') -} - main() diff --git a/scripts/format.js b/scripts/format.js index 38673dd6bd..fb77cab20f 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -1,152 +1,55 @@ -const { program } = require('commander') -const parser = require('./parser') -const utils = require('./utils') -const axios = require('axios') -const ProgressBar = require('progress') -const https = require('https') - -program - .usage('[OPTIONS]...') - .option('-d, --debug', 'Debug mode') - .option('-r --resolution', 'Parse stream resolution') - .option('-c, --country ', 'Comma-separated list of country codes', '') - .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') - .option('--delay ', 'Delay between parser requests', 1000) - .option('--timeout ', 'Set timeout for each request', 5000) - .parse(process.argv) - -const config = program.opts() - -const instance = axios.create({ - timeout: config.timeout, - maxContentLength: 200000, - httpsAgent: new https.Agent({ - rejectUnauthorized: false - }) -}) +const parser = require('./helpers/parser') +const utils = require('./helpers/utils') +const file = require('./helpers/file') +const log = require('./helpers/log') async function main() { - console.info('Starting...') - console.time('Done in') - - const playlists = parseIndex() + log.start() + log.print(`Parsing 'index.m3u'...`) + let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') for (const playlist of playlists) { - await loadPlaylist(playlist.url) - .then(sortChannels) - .then(detectResolution) - .then(savePlaylist) - .then(done) - } - - finish() -} - -function parseIndex() { - console.info(`\nParsing 'index.m3u'...`) - let playlists = parser.parseIndex() - playlists = utils - .filterPlaylists(playlists, config.country, config.exclude) - .filter(i => i.url !== 'channels/unsorted.m3u') - console.info(`Found ${playlists.length} playlist(s)\n`) - - return playlists -} - -async function loadPlaylist(url) { - console.info(`Processing '${url}'...`) - return parser.parsePlaylist(url) -} - -async function sortChannels(playlist) { - console.info(` Sorting channels...`) - playlist.channels = utils.sortBy(playlist.channels, ['name', 'url']) - - return playlist -} - -async function detectResolution(playlist) { - if (!config.resolution) return playlist - console.log(' Detecting resolution...') - const bar = new ProgressBar(' Progress: [:bar] :current/:total (:percent) ', { - total: playlist.channels.length - }) - const results = [] - for (const channel of playlist.channels) { - bar.tick() - if (!channel.resolution.height) { - const CancelToken = axios.CancelToken - const source = CancelToken.source() - const timeout = setTimeout(() => { - source.cancel() - }, config.timeout) - - const response = await instance - .get(channel.url, { cancelToken: source.token }) - .then(res => { - clearTimeout(timeout) - - return res - }) - .then(utils.sleep(config.delay)) - .catch(err => { - clearTimeout(timeout) - }) - - if (response && response.status === 200) { - if (/^#EXTM3U/.test(response.data)) { - const resolution = parseResolution(response.data) - if (resolution) { - channel.resolution = resolution - } + log.print(`\nProcessing '${playlist.url}'...`) + await parser + .parsePlaylist(playlist.url) + .then(formatPlaylist) + .then(playlist => { + if (file.read(playlist.url) !== playlist.toString()) { + log.print('updated') + playlist.updated = true } - } - } - results.push(channel) + playlist.save() + }) } - playlist.channels = results + log.print('\n') + log.finish() +} + +async function formatPlaylist(playlist) { + for (const channel of playlist.channels) { + const code = file.getBasename(playlist.url) + // add missing tvg-name + if (!channel.tvg.name && code !== 'unsorted' && channel.name) { + channel.tvg.name = channel.name.replace(/\"/gi, '') + } + // add missing tvg-id + if (!channel.tvg.id && code !== 'unsorted' && channel.tvg.name) { + const id = utils.name2id(channel.tvg.name) + channel.tvg.id = id ? `${id}.${code}` : '' + } + // add missing country + if (!channel.countries.length) { + const name = utils.code2name(code) + channel.countries = name ? [{ code, name }] : [] + channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') + } + // update group-title + channel.group.title = channel.category + } return playlist } -function parseResolution(string) { - const regex = /RESOLUTION=(\d+)x(\d+)/gm - const match = string.matchAll(regex) - const arr = Array.from(match).map(m => ({ - width: parseInt(m[1]), - height: parseInt(m[2]) - })) - - return arr.length - ? arr.reduce(function (prev, current) { - return prev.height > current.height ? prev : current - }) - : undefined -} - -async function savePlaylist(playlist) { - const original = utils.readFile(playlist.url) - const output = playlist.toString() - - if (original === output) { - console.info(`No changes have been made.`) - return false - } else { - utils.createFile(playlist.url, output) - console.info(`Playlist has been updated.`) - } - - return true -} - -async function done() { - console.info(` `) -} - -function finish() { - console.timeEnd('Done in') -} - main() diff --git a/scripts/generate.js b/scripts/generate.js index f561c9e237..6aa31c2245 100644 --- a/scripts/generate.js +++ b/scripts/generate.js @@ -1,11 +1,11 @@ -const db = require('./db') -const utils = require('./utils') +const file = require('./helpers/file') +const log = require('./helpers/log') +const db = require('./helpers/db') const ROOT_DIR = './.gh-pages' -db.load() - -function main() { +async function main() { + await loadDatabase() createRootDirectory() createNoJekyllFile() generateIndex() @@ -16,51 +16,56 @@ function main() { generateCountries() generateLanguages() generateChannelsJson() - finish() + showResults() +} + +async function loadDatabase() { + log.print('Loading database...\n') + await db.load() } function createRootDirectory() { - console.log('Creating .gh-pages folder...') - utils.createDir(ROOT_DIR) + log.print('Creating .gh-pages folder...\n') + file.createDir(ROOT_DIR) } function createNoJekyllFile() { - console.log('Creating .nojekyll...') - utils.createFile(`${ROOT_DIR}/.nojekyll`) + log.print('Creating .nojekyll...\n') + file.create(`${ROOT_DIR}/.nojekyll`) } function generateIndex() { - console.log('Generating index.m3u...') + log.print('Generating index.m3u...\n') const filename = `${ROOT_DIR}/index.m3u` - utils.createFile(filename, '#EXTM3U\n') + file.create(filename, '#EXTM3U\n') const nsfwFilename = `${ROOT_DIR}/index.nsfw.m3u` - utils.createFile(nsfwFilename, '#EXTM3U\n') + file.create(nsfwFilename, '#EXTM3U\n') const channels = db.channels.sortBy(['name', 'url']).removeDuplicates().get() for (const channel of channels) { if (!channel.isNSFW()) { - utils.appendToFile(filename, channel.toString()) + file.append(filename, channel.toString()) } - utils.appendToFile(nsfwFilename, channel.toString()) + file.append(nsfwFilename, channel.toString()) } } function generateCategoryIndex() { - console.log('Generating index.category.m3u...') + log.print('Generating index.category.m3u...\n') const filename = `${ROOT_DIR}/index.category.m3u` - utils.createFile(filename, '#EXTM3U\n') + file.create(filename, '#EXTM3U\n') const channels = db.channels.sortBy(['category', 'name', 'url']).removeDuplicates().get() for (const channel of channels) { - utils.appendToFile(filename, channel.toString()) + file.append(filename, channel.toString()) } } function generateCountryIndex() { - console.log('Generating index.country.m3u...') + log.print('Generating index.country.m3u...\n') const filename = `${ROOT_DIR}/index.country.m3u` - utils.createFile(filename, '#EXTM3U\n') + file.create(filename, '#EXTM3U\n') for (const country of [{ code: 'undefined' }, ...db.countries.sortBy(['name']).all()]) { const channels = db.channels @@ -69,21 +74,21 @@ function generateCountryIndex() { .removeDuplicates() .get() for (const channel of channels) { - const category = channel.category + const groupTitle = channel.group.title const nsfw = channel.isNSFW() - channel.category = country.name || '' + channel.group.title = country.name || '' if (!nsfw) { - utils.appendToFile(filename, channel.toString()) + file.append(filename, channel.toString()) } - channel.category = category + channel.group.title = groupTitle } } } function generateLanguageIndex() { - console.log('Generating index.language.m3u...') + log.print('Generating index.language.m3u...\n') const filename = `${ROOT_DIR}/index.language.m3u` - utils.createFile(filename, '#EXTM3U\n') + file.create(filename, '#EXTM3U\n') for (const language of [{ code: 'undefined' }, ...db.languages.sortBy(['name']).all()]) { const channels = db.channels @@ -92,25 +97,25 @@ function generateLanguageIndex() { .removeDuplicates() .get() for (const channel of channels) { - const category = channel.category + const groupTitle = channel.group.title const nsfw = channel.isNSFW() - channel.category = language.name || '' + channel.group.title = language.name || '' if (!nsfw) { - utils.appendToFile(filename, channel.toString()) + file.append(filename, channel.toString()) } - channel.category = category + channel.group.title = groupTitle } } } function generateCategories() { - console.log(`Generating /categories...`) + log.print(`Generating /categories...\n`) const outputDir = `${ROOT_DIR}/categories` - utils.createDir(outputDir) + file.createDir(outputDir) for (const category of [...db.categories.all(), { id: 'other' }]) { const filename = `${outputDir}/${category.id}.m3u` - utils.createFile(filename, '#EXTM3U\n') + file.create(filename, '#EXTM3U\n') const channels = db.channels .sortBy(['name', 'url']) @@ -118,19 +123,19 @@ function generateCategories() { .removeDuplicates() .get() for (const channel of channels) { - utils.appendToFile(filename, channel.toString()) + file.append(filename, channel.toString()) } } } function generateCountries() { - console.log(`Generating /countries...`) + log.print(`Generating /countries...\n`) const outputDir = `${ROOT_DIR}/countries` - utils.createDir(outputDir) + file.createDir(outputDir) for (const country of [...db.countries.all(), { code: 'undefined' }]) { const filename = `${outputDir}/${country.code}.m3u` - utils.createFile(filename, '#EXTM3U\n') + file.create(filename, '#EXTM3U\n') const channels = db.channels .sortBy(['name', 'url']) @@ -139,20 +144,20 @@ function generateCountries() { .get() for (const channel of channels) { if (!channel.isNSFW()) { - utils.appendToFile(filename, channel.toString()) + file.append(filename, channel.toString()) } } } } function generateLanguages() { - console.log(`Generating /languages...`) + log.print(`Generating /languages...\n`) const outputDir = `${ROOT_DIR}/languages` - utils.createDir(outputDir) + file.createDir(outputDir) for (const language of [...db.languages.all(), { code: 'undefined' }]) { const filename = `${outputDir}/${language.code}.m3u` - utils.createFile(filename, '#EXTM3U\n') + file.create(filename, '#EXTM3U\n') const channels = db.channels .sortBy(['name', 'url']) @@ -161,25 +166,25 @@ function generateLanguages() { .get() for (const channel of channels) { if (!channel.isNSFW()) { - utils.appendToFile(filename, channel.toString()) + file.append(filename, channel.toString()) } } } } function generateChannelsJson() { - console.log('Generating channels.json...') + log.print('Generating channels.json...\n') const filename = `${ROOT_DIR}/channels.json` const channels = db.channels .sortBy(['name', 'url']) .get() .map(c => c.toObject()) - utils.createFile(filename, JSON.stringify(channels)) + file.create(filename, JSON.stringify(channels)) } -function finish() { - console.log( - `\nTotal: ${db.channels.count()} channels, ${db.countries.count()} countries, ${db.languages.count()} languages, ${db.categories.count()} categories.` +function showResults() { + log.print( + `Total: ${db.channels.count()} channels, ${db.countries.count()} countries, ${db.languages.count()} languages, ${db.categories.count()} categories.\n` ) } diff --git a/scripts/helpers/Channel.js b/scripts/helpers/Channel.js new file mode 100644 index 0000000000..71f3d4d1a1 --- /dev/null +++ b/scripts/helpers/Channel.js @@ -0,0 +1,145 @@ +const categories = require('./categories') +const utils = require('./utils') +const file = require('./file') + +const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name) +const nsfwCategories = categories.filter(c => c.nsfw).map(c => c.name) + +module.exports = class Channel { + constructor(data) { + this.raw = data.raw + this.tvg = data.tvg + this.http = data.http + this.url = data.url + this.logo = data.tvg.logo + this.group = data.group + this.name = this.parseName(data.name) + this.status = this.parseStatus(data.name) + this.resolution = this.parseResolution(data.name) + this.category = this.parseCategory(data.group.title) + this.countries = this.parseCountries(data.tvg.country) + this.languages = this.parseLanguages(data.tvg.language) + } + + parseName(title) { + return title + .trim() + .split(' ') + .map(s => s.trim()) + .filter(s => { + return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s) + }) + .join(' ') + } + + parseStatus(title) { + const match = title.match(/\[(.*)\]/i) + return match ? match[1] : null + } + + parseResolution(title) { + const match = title.match(/\((\d+)P\)/i) + const height = match ? parseInt(match[1]) : null + + return { width: null, height } + } + + parseCategory(string) { + const category = categories.find(c => c.id === string.toLowerCase()) + if (!category) return '' + + return category.name + } + + parseCountries(string) { + const list = string.split(';') + return list + .reduce((acc, curr) => { + const codes = utils.region2codes(curr) + if (codes.length) { + for (let code of codes) { + if (!acc.includes(code)) { + acc.push(code) + } + } + } else { + acc.push(curr) + } + + return acc + }, []) + .map(code => { + const name = code ? utils.code2name(code) : null + if (!name) return null + + return { code: code.toLowerCase(), name } + }) + .filter(c => c) + } + + parseLanguages(string) { + const list = string.split(';') + return list + .map(name => { + const code = name ? utils.language2code(name) : null + if (!code) return null + + return { code, name } + }) + .filter(l => l) + } + + isSFW() { + return sfwCategories.includes(this.category) + } + + isNSFW() { + return nsfwCategories.includes(this.category) + } + + getInfo() { + let info = `-1 tvg-id="${this.tvg.id}" tvg-name="${this.tvg.name}" tvg-country="${this.tvg.country}" tvg-language="${this.tvg.language}" tvg-logo="${this.logo}"` + + info += ` group-title="${this.group.title}",${this.name}` + + if (this.resolution.height) { + info += ` (${this.resolution.height}p)` + } + + if (this.status) { + info += ` [${this.status}]` + } + + if (this.http['referrer']) { + info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}` + } + + if (this.http['user-agent']) { + info += `\n#EXTVLCOPT:http-user-agent=${this.http['user-agent']}` + } + + return info + } + + toString(raw = false) { + if (raw) return this.raw + '\n' + + return '#EXTINF:' + this.getInfo() + '\n' + this.url + '\n' + } + + toObject() { + return { + name: this.name, + logo: this.logo || null, + url: this.url, + category: this.category || null, + languages: this.languages, + countries: this.countries, + tvg: { + id: this.tvg.id || null, + name: this.tvg.name || null, + url: this.tvg.url || null + } + } + } +} diff --git a/scripts/helpers/Playlist.js b/scripts/helpers/Playlist.js new file mode 100644 index 0000000000..6b797c96c6 --- /dev/null +++ b/scripts/helpers/Playlist.js @@ -0,0 +1,37 @@ +const Channel = require('./Channel') +const file = require('./file') + +module.exports = class Playlist { + constructor({ header, items, url, name, country }) { + this.url = url + this.name = name + this.country = country + this.header = header + this.channels = items.map(item => new Channel(item)).filter(channel => channel.url) + this.updated = false + } + + toString(options = {}) { + const config = { raw: false, ...options } + let parts = ['#EXTM3U'] + for (let key in this.header.attrs) { + let value = this.header.attrs[key] + if (value) { + parts.push(`${key}="${value}"`) + } + } + + let output = `${parts.join(' ')}\n` + for (let channel of this.channels) { + output += channel.toString(config.raw) + } + + return output + } + + save() { + if (this.updated) { + file.create(this.url, this.toString()) + } + } +} diff --git a/scripts/blacklist.json b/scripts/helpers/blacklist.json similarity index 100% rename from scripts/blacklist.json rename to scripts/helpers/blacklist.json diff --git a/scripts/categories.json b/scripts/helpers/categories.json similarity index 100% rename from scripts/categories.json rename to scripts/helpers/categories.json diff --git a/scripts/db.js b/scripts/helpers/db.js similarity index 94% rename from scripts/db.js rename to scripts/helpers/db.js index a0d67cb4dd..417742f847 100644 --- a/scripts/db.js +++ b/scripts/helpers/db.js @@ -2,14 +2,12 @@ const categories = require('./categories') const parser = require('./parser') const utils = require('./utils') -const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name) - const db = {} -db.load = function () { +db.load = async function () { const items = parser.parseIndex() for (const item of items) { - const playlist = parser.parsePlaylist(item.url) + const playlist = await parser.parsePlaylist(item.url) db.playlists.add(playlist) for (const channel of playlist.channels) { db.channels.add(channel) @@ -107,9 +105,6 @@ db.channels = { all() { return this.list }, - sfw() { - return this.list.filter(i => sfwCategories.includes(i.category)) - }, forCountry(country) { this.filter = { field: 'countries', diff --git a/scripts/helpers/file.js b/scripts/helpers/file.js new file mode 100644 index 0000000000..cb946c3378 --- /dev/null +++ b/scripts/helpers/file.js @@ -0,0 +1,38 @@ +const markdownInclude = require('markdown-include') +const path = require('path') +const fs = require('fs') + +const rootPath = path.resolve(__dirname) + '/../../' +const file = {} + +file.getBasename = function (filename) { + return path.basename(filename, path.extname(filename)) +} + +file.getFilename = function (filename) { + return path.parse(filename).name +} + +file.createDir = function (dir) { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir) + } +} + +file.read = function (filename) { + return fs.readFileSync(rootPath + filename, { encoding: 'utf8' }) +} + +file.append = function (filename, data) { + fs.appendFileSync(rootPath + filename, data) +} + +file.create = function (filename, data = '') { + fs.writeFileSync(rootPath + filename, data) +} + +file.compileMarkdown = function (filename) { + markdownInclude.compileFiles(rootPath + filename) +} + +module.exports = file diff --git a/scripts/helpers/log.js b/scripts/helpers/log.js new file mode 100644 index 0000000000..a2e8fe0a2e --- /dev/null +++ b/scripts/helpers/log.js @@ -0,0 +1,16 @@ +const log = {} + +log.print = function (string) { + process.stdout.write(string) +} + +log.start = function () { + this.print('Starting...\n') + console.time('Done in') +} + +log.finish = function () { + console.timeEnd('Done in') +} + +module.exports = log diff --git a/scripts/helpers/parser.js b/scripts/helpers/parser.js new file mode 100644 index 0000000000..815c819ec0 --- /dev/null +++ b/scripts/helpers/parser.js @@ -0,0 +1,24 @@ +const playlistParser = require('iptv-playlist-parser') +const Playlist = require('./Playlist') +const utils = require('./utils') +const file = require('./file') + +const parser = {} + +parser.parseIndex = function () { + const content = file.read('index.m3u') + const result = playlistParser.parse(content) + + return result.items +} + +parser.parsePlaylist = async function (url) { + const content = file.read(url) + const result = playlistParser.parse(content) + const name = file.getFilename(url) + const country = utils.code2name(name) + + return new Playlist({ header: result.header, items: result.items, url, country, name }) +} + +module.exports = parser diff --git a/scripts/regions.json b/scripts/helpers/regions.json similarity index 100% rename from scripts/regions.json rename to scripts/helpers/regions.json diff --git a/scripts/utils.js b/scripts/helpers/utils.js similarity index 52% rename from scripts/utils.js rename to scripts/helpers/utils.js index 0c290ba5f4..7a2bae0633 100644 --- a/scripts/utils.js +++ b/scripts/helpers/utils.js @@ -1,21 +1,15 @@ -const fs = require('fs') -const path = require('path') -const axios = require('axios') -const zlib = require('zlib') -const urlParser = require('url') const escapeStringRegexp = require('escape-string-regexp') -const markdownInclude = require('markdown-include') -const iso6393 = require('@freearhey/iso-639-3') const transliteration = require('transliteration') -const regions = require('./regions') +const iso6393 = require('@freearhey/iso-639-3') const categories = require('./categories') +const regions = require('./regions') + +const utils = {} const intlDisplayNames = new Intl.DisplayNames(['en'], { style: 'narrow', type: 'region' }) -const utils = {} - utils.name2id = function (name) { return transliteration .transliterate(name) @@ -66,36 +60,29 @@ utils.sortBy = function (arr, fields) { for (let field of fields) { let propA = a[field] ? a[field].toLowerCase() : '' let propB = b[field] ? b[field].toLowerCase() : '' - - if (propA === 'undefined') { - return 1 - } - - if (propB === 'undefined') { - return -1 - } - - if (propA === 'other') { - return 1 - } - - if (propB === 'other') { - return -1 - } - - if (propA < propB) { - return -1 - } - if (propA > propB) { - return 1 - } + if (propA === 'undefined') return 1 + if (propB === 'undefined') return -1 + if (propA === 'other') return 1 + if (propB === 'other') return -1 + if (propA < propB) return -1 + if (propA > propB) return 1 } return 0 }) } -utils.getBasename = function (filename) { - return path.basename(filename, path.extname(filename)) +utils.escapeStringRegexp = function (scring) { + return escapeStringRegexp(string) +} + +utils.sleep = function (ms) { + return function (x) { + return new Promise(resolve => setTimeout(() => resolve(x), ms)) + } +} + +utils.removeProtocol = function (string) { + return string.replace(/(^\w+:|^)\/\//, '') } utils.filterPlaylists = function (arr, include = '', exclude = '') { @@ -114,75 +101,4 @@ utils.filterPlaylists = function (arr, include = '', exclude = '') { return arr } -utils.generateTable = function (data, options) { - let output = '\n' - - output += '\t\n\t\t' - for (let column of options.columns) { - output += `` - } - output += '\n\t\n' - - output += '\t\n' - for (let item of data) { - output += '\t\t' - let i = 0 - for (let prop in item) { - const column = options.columns[i] - let nowrap = column.nowrap - let align = column.align - output += `` - i++ - } - output += '\n' - } - output += '\t\n' - - output += '
${column.name}
${item[prop]}
' - - return output -} - -utils.createDir = function (dir) { - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir) - } -} - -utils.readFile = function (filename) { - return fs.readFileSync(path.resolve(__dirname) + `/../${filename}`, { encoding: 'utf8' }) -} - -utils.appendToFile = function (filename, data) { - fs.appendFileSync(path.resolve(__dirname) + '/../' + filename, data) -} - -utils.compileMarkdown = function (filepath) { - return markdownInclude.compileFiles(path.resolve(__dirname, filepath)) -} - -utils.escapeStringRegexp = function (scring) { - return escapeStringRegexp(string) -} - -utils.createFile = function (filename, data = '') { - fs.writeFileSync(path.resolve(__dirname) + '/../' + filename, data) -} - -utils.writeToLog = function (country, msg, url) { - var now = new Date() - var line = `${country}: ${msg} '${url}'` - this.appendToFile('error.log', now.toISOString() + ' ' + line + '\n') -} - -utils.sleep = function (ms) { - return function (x) { - return new Promise(resolve => setTimeout(() => resolve(x), ms)) - } -} - -utils.removeProtocol = function (string) { - return string.replace(/(^\w+:|^)\/\//, '') -} - module.exports = utils diff --git a/scripts/parser.js b/scripts/parser.js deleted file mode 100644 index 2348c91455..0000000000 --- a/scripts/parser.js +++ /dev/null @@ -1,244 +0,0 @@ -const playlistParser = require('iptv-playlist-parser') -const utils = require('./utils') -const categories = require('./categories') -const path = require('path') - -const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name) -const nsfwCategories = categories.filter(c => c.nsfw).map(c => c.name) - -const parser = {} - -parser.parseIndex = function () { - const content = utils.readFile('index.m3u') - const result = playlistParser.parse(content) - - return result.items -} - -parser.parsePlaylist = function (filename) { - const content = utils.readFile(filename) - const result = playlistParser.parse(content) - const name = path.parse(filename).name - const country = utils.code2name(name) - - return new Playlist({ header: result.header, items: result.items, url: filename, country, name }) -} - -class Playlist { - constructor({ header, items, url, name, country }) { - this.url = url - this.name = name - this.country = country - this.header = header - this.channels = items - .map(item => new Channel({ data: item, header, sourceUrl: url })) - .filter(channel => channel.url) - } - - toString(options = {}) { - const config = { raw: false, ...options } - let parts = ['#EXTM3U'] - for (let key in this.header.attrs) { - let value = this.header.attrs[key] - if (value) { - parts.push(`${key}="${value}"`) - } - } - - let output = `${parts.join(' ')}\n` - for (let channel of this.channels) { - output += channel.toString(config.raw) - } - - return output - } -} - -class Channel { - constructor({ data, header, sourceUrl }) { - this.parseData(data) - - this.filename = utils.getBasename(sourceUrl) - if (!this.countries.length) { - const countryName = utils.code2name(this.filename) - this.countries = countryName ? [{ code: this.filename, name: countryName }] : [] - this.tvg.country = this.countries.map(c => c.code.toUpperCase()).join(';') - } - } - - parseData(data) { - const title = this.parseTitle(data.name) - - this.tvg = data.tvg - this.http = data.http - this.url = data.url - this.logo = data.tvg.logo - this.name = title.channelName - this.status = title.streamStatus - this.resolution = title.streamResolution - this.countries = this.parseCountries(data.tvg.country) - this.languages = this.parseLanguages(data.tvg.language) - this.category = this.parseCategory(data.group.title) - this.raw = data.raw - } - - parseCountries(string) { - let arr = string - .split(';') - .reduce((acc, curr) => { - const codes = utils.region2codes(curr) - if (codes.length) { - for (let code of codes) { - if (!acc.includes(code)) { - acc.push(code) - } - } - } else { - acc.push(curr) - } - - return acc - }, []) - .filter(code => code && utils.code2name(code)) - - return arr.map(code => { - return { code: code.toLowerCase(), name: utils.code2name(code) } - }) - } - - parseLanguages(string) { - return string - .split(';') - .map(name => { - const code = name ? utils.language2code(name) : null - if (!code) return null - - return { - code, - name - } - }) - .filter(l => l) - } - - parseCategory(string) { - const category = categories.find(c => c.id === string.toLowerCase()) - - return category ? category.name : '' - } - - parseTitle(title) { - const channelName = title - .trim() - .split(' ') - .map(s => s.trim()) - .filter(s => { - return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s) - }) - .join(' ') - - const streamStatusMatch = title.match(/\[(.*)\]/i) - const streamStatus = streamStatusMatch ? streamStatusMatch[1] : null - - const streamResolutionMatch = title.match(/\((\d+)P\)/i) - const streamResolutionHeight = streamResolutionMatch ? parseInt(streamResolutionMatch[1]) : null - const streamResolution = { width: null, height: streamResolutionHeight } - - return { channelName, streamStatus, streamResolution } - } - - get tvgCountry() { - return this.tvg.country - .split(';') - .map(code => utils.code2name(code)) - .join(';') - } - - get tvgLanguage() { - return this.tvg.language - } - - get tvgUrl() { - return this.tvg.id && this.tvg.url ? this.tvg.url : '' - } - - get tvgId() { - if (this.tvg.id) { - return this.tvg.id - } else if (this.filename !== 'unsorted') { - const id = utils.name2id(this.tvgName) - - return id ? `${id}.${this.filename}` : '' - } - - return '' - } - - get tvgName() { - if (this.tvg.name) { - return this.tvg.name - } else if (this.filename !== 'unsorted') { - return this.name.replace(/\"/gi, '') - } - - return '' - } - - getInfo() { - this.tvg.country = this.tvg.country.toUpperCase() - - let info = `-1 tvg-id="${this.tvgId}" tvg-name="${this.tvgName}" tvg-country="${this.tvg.country}" tvg-language="${this.tvg.language}" tvg-logo="${this.logo}"` - - info += ` group-title="${this.category}",${this.name}` - - if (this.resolution.height) { - info += ` (${this.resolution.height}p)` - } - - if (this.status) { - info += ` [${this.status}]` - } - - if (this.http['referrer']) { - info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}` - } - - if (this.http['user-agent']) { - info += `\n#EXTVLCOPT:http-user-agent=${this.http['user-agent']}` - } - - return info - } - - toString(raw = false) { - if (raw) return this.raw + '\n' - - return '#EXTINF:' + this.getInfo() + '\n' + this.url + '\n' - } - - toObject() { - return { - name: this.name, - logo: this.logo || null, - url: this.url, - category: this.category || null, - languages: this.languages, - countries: this.countries, - tvg: { - id: this.tvgId || null, - name: this.tvgName || null, - url: this.tvgUrl || null - } - } - } - - isSFW() { - return sfwCategories.includes(this.category) - } - - isNSFW() { - return nsfwCategories.includes(this.category) - } -} - -module.exports = parser diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js index f7fbd94327..650fbab472 100644 --- a/scripts/remove-duplicates.js +++ b/scripts/remove-duplicates.js @@ -1,52 +1,42 @@ -const parser = require('./parser') -const utils = require('./utils') +const parser = require('./helpers/parser') +const utils = require('./helpers/utils') +const log = require('./helpers/log') let globalBuffer = [] async function main() { - const playlists = parseIndex() + log.start() + log.print(`Parsing 'index.m3u'...`) + const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') for (const playlist of playlists) { - await loadPlaylist(playlist.url) + log.print(`\nProcessing '${playlist.url}'...`) + await parser + .parsePlaylist(playlist.url) .then(addToBuffer) .then(removeDuplicates) - .then(savePlaylist) - .then(done) + .then(p => p.save()) } if (playlists.length) { - await loadPlaylist('channels/unsorted.m3u') + log.print(`\nProcessing 'channels/unsorted.m3u'...`) + await parser + .parsePlaylist('channels/unsorted.m3u') .then(removeUnsortedDuplicates) - .then(savePlaylist) - .then(done) + .then(p => p.save()) } - finish() -} - -function parseIndex() { - console.info(`Parsing 'index.m3u'...`) - let playlists = parser.parseIndex() - playlists = playlists.filter(i => i.url !== 'channels/unsorted.m3u') - console.info(`Found ${playlists.length} playlist(s)\n`) - - return playlists -} - -async function loadPlaylist(url) { - console.info(`Processing '${url}'...`) - return parser.parsePlaylist(url) + log.print('\n') + log.finish() } async function addToBuffer(playlist) { - if (playlist.url === 'channels/unsorted.m3u') return playlist globalBuffer = globalBuffer.concat(playlist.channels) return playlist } async function removeDuplicates(playlist) { - console.info(` Looking for duplicates...`) let buffer = {} const channels = playlist.channels.filter(i => { const url = utils.removeProtocol(i.url) @@ -58,13 +48,16 @@ async function removeDuplicates(playlist) { return result }) - playlist.channels = channels + if (playlist.channels.length !== channels.length) { + log.print('updated') + playlist.channels = channels + playlist.updated = true + } return playlist } async function removeUnsortedDuplicates(playlist) { - console.info(` Looking for duplicates...`) // locally let buffer = {} let channels = playlist.channels.filter(i => { @@ -74,37 +67,18 @@ async function removeUnsortedDuplicates(playlist) { return result }) + // globally const urls = globalBuffer.map(i => utils.removeProtocol(i.url)) channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url))) - if (channels.length === playlist.channels.length) return playlist - playlist.channels = channels + if (channels.length !== playlist.channels.length) { + log.print('updated') + playlist.channels = channels + playlist.updated = true + } return playlist } -async function savePlaylist(playlist) { - const original = utils.readFile(playlist.url) - const output = playlist.toString({ raw: true }) - - if (original === output) { - console.info(`No changes have been made.`) - return false - } else { - utils.createFile(playlist.url, output) - console.info(`Playlist has been updated.`) - } - - return true -} - -async function done() { - console.info(` `) -} - -function finish() { - console.info('Done.') -} - main() diff --git a/scripts/sort.js b/scripts/sort.js new file mode 100644 index 0000000000..974f2592ce --- /dev/null +++ b/scripts/sort.js @@ -0,0 +1,35 @@ +const parser = require('./helpers/parser') +const utils = require('./helpers/utils') +const log = require('./helpers/log') + +async function main() { + log.start() + + log.print(`Parsing 'index.m3u'...`) + let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') + for (const playlist of playlists) { + log.print(`\nProcessing '${playlist.url}'...`) + await parser + .parsePlaylist(playlist.url) + .then(sortChannels) + .then(p => p.save()) + } + + log.print('\n') + log.finish() +} + +async function sortChannels(playlist) { + const channels = [...playlist.channels] + utils.sortBy(channels, ['name', 'url']) + + if (JSON.stringify(channels) !== JSON.stringify(playlist.channels)) { + log.print('updated') + playlist.channels = channels + playlist.updated = true + } + + return playlist +} + +main() diff --git a/scripts/update-readme.js b/scripts/update-readme.js index 3f20e0a73f..e4a96489c9 100644 --- a/scripts/update-readme.js +++ b/scripts/update-readme.js @@ -1,20 +1,25 @@ -const utils = require('./utils') -const db = require('./db') -const parser = require('./parser') +const utils = require('./helpers/utils') +const file = require('./helpers/file') +const log = require('./helpers/log') +const db = require('./helpers/db') -db.load() - -function main() { - start() +async function main() { + log.start() + await loadDatabase() generateCategoriesTable() generateCountriesTable() generateLanguagesTable() generateReadme() - finish() + log.finish() +} + +async function loadDatabase() { + log.print('Loading database...\n') + await db.load() } function generateCategoriesTable() { - console.log(`Generating categories table...`) + log.print('Generating categories table...\n') const categories = [] for (const category of [...db.categories.all(), { name: 'Other', id: 'other' }]) { @@ -25,7 +30,7 @@ function generateCategoriesTable() { }) } - const table = utils.generateTable(categories, { + const table = generateTable(categories, { columns: [ { name: 'Category', align: 'left' }, { name: 'Channels', align: 'right' }, @@ -33,11 +38,11 @@ function generateCategoriesTable() { ] }) - utils.createFile('./.readme/_categories.md', table) + file.create('./.readme/_categories.md', table) } function generateCountriesTable() { - console.log(`Generating countries table...`) + log.print('Generating countries table...\n') const countries = [] for (const country of [ @@ -53,7 +58,7 @@ function generateCountriesTable() { }) } - const table = utils.generateTable(countries, { + const table = generateTable(countries, { columns: [ { name: 'Country', align: 'left' }, { name: 'Channels', align: 'right' }, @@ -61,11 +66,11 @@ function generateCountriesTable() { ] }) - utils.createFile('./.readme/_countries.md', table) + file.create('./.readme/_countries.md', table) } function generateLanguagesTable() { - console.log(`Generating languages table...`) + log.print('Generating languages table...\n') const languages = [] for (const language of [ @@ -79,7 +84,7 @@ function generateLanguagesTable() { }) } - const table = utils.generateTable(languages, { + const table = generateTable(languages, { columns: [ { name: 'Language', align: 'left' }, { name: 'Channels', align: 'right' }, @@ -87,20 +92,41 @@ function generateLanguagesTable() { ] }) - utils.createFile('./.readme/_languages.md', table) + file.create('./.readme/_languages.md', table) +} + +function generateTable(data, options) { + let output = '\n' + + output += '\t\n\t\t' + for (let column of options.columns) { + output += `` + } + output += '\n\t\n' + + output += '\t\n' + for (let item of data) { + output += '\t\t' + let i = 0 + for (let prop in item) { + const column = options.columns[i] + let nowrap = column.nowrap + let align = column.align + output += `` + i++ + } + output += '\n' + } + output += '\t\n' + + output += '
${column.name}
${item[prop]}
' + + return output } function generateReadme() { - console.log(`Generating README.md...`) - utils.compileMarkdown('../.readme/config.json') -} - -function start() { - console.log(`Starting...`) -} - -function finish() { - console.log(`Done.`) + log.print('Generating README.md...\n') + file.compileMarkdown('.readme/config.json') } main()