From 589d29287e24b45ba81e309d2b0fde80b9896eba Mon Sep 17 00:00:00 2001 From: MapGuy11 <43685307+MapGuy11@users.noreply.github.com> Date: Mon, 9 Aug 2021 22:01:44 -0400 Subject: [PATCH 01/10] Add Channels 1. WSNN-LD1 2. WISC-DT2 3. WFMZ-DT2 --- channels/us.m3u | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/channels/us.m3u b/channels/us.m3u index d09a98ee10..74d97ca009 100644 --- a/channels/us.m3u +++ b/channels/us.m3u @@ -6615,3 +6615,9 @@ https://ln-zen.localnowlive.com/v1/master/385c85a93929f94966d0fb186fc33b431e6f1e https://rockentertainment-zoomoo-1.samsung.wurl.com/manifest/playlist.m3u8 #EXTINF:-1 tvg-id="KanalDisney.us" tvg-name="Канал Disney" tvg-country="RU" tvg-language="Russian" tvg-logo="https://i.imgur.com/Q9KoVy9.png" group-title="Kids",Канал Disney (576p) http://188.40.68.167/russia/disney/playlist.m3u8 +#EXTINF:-1 tvg-id="WSNN-LD1.us" tvg-name="WSNN-LD1" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/75548/s75548_h3_aa.png" group-title="Locals",WSNN-LD1 (Suncoast News Network) +https://cdn88.theus6tv.tk/united-states/locals/tampa/wsnn-ld.m3u8 +#EXTINF:-1 tvg-id="WISC-DT2.us" tvg-name="WISC-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/25125/s51307_h3_aa.png" group-title="Locals",WISC-DT2 (My Madison TV) +https://cdn88.theus6tv.tk/united-states/locals/madison/wisc-dt2.m3u8 +#EXTINF:-1 tvg-id="WFMZ-DT2.us" tvg-name="WFMZ-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/35859/s35859_h3_aa.png" group-title="Locals",WFMZ-DT2 (69News Weather Channel) +https://cdn88.theus6tv.tk/united-states/locals/madison/wisc-dt2.m3u8 \ No newline at end of file From 149c59d706f3715ae30c68a7fffdac24f3c46eeb Mon Sep 17 00:00:00 2001 From: MapGuy11 <43685307+MapGuy11@users.noreply.github.com> Date: Mon, 9 Aug 2021 22:11:20 -0400 Subject: [PATCH 02/10] Update us.m3u --- channels/us.m3u | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/us.m3u b/channels/us.m3u index 74d97ca009..dc47782432 100644 --- a/channels/us.m3u +++ b/channels/us.m3u @@ -6620,4 +6620,4 @@ https://cdn88.theus6tv.tk/united-states/locals/tampa/wsnn-ld.m3u8 #EXTINF:-1 tvg-id="WISC-DT2.us" tvg-name="WISC-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/25125/s51307_h3_aa.png" group-title="Locals",WISC-DT2 (My Madison TV) https://cdn88.theus6tv.tk/united-states/locals/madison/wisc-dt2.m3u8 #EXTINF:-1 tvg-id="WFMZ-DT2.us" tvg-name="WFMZ-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/35859/s35859_h3_aa.png" group-title="Locals",WFMZ-DT2 (69News Weather Channel) -https://cdn88.theus6tv.tk/united-states/locals/madison/wisc-dt2.m3u8 \ No newline at end of file +https://cdn88.theus6tv.tk/united-states/locals/philadelphia/wfmz-dt2.m3u8 From c3696ad47969103b8526c493de62dee525882ac6 Mon Sep 17 00:00:00 2001 From: MapGuy11 <43685307+MapGuy11@users.noreply.github.com> Date: Tue, 10 Aug 2021 12:53:16 -0400 Subject: [PATCH 03/10] Update us.m3u --- channels/us.m3u | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/channels/us.m3u b/channels/us.m3u index dc47782432..cb0ebf5423 100644 --- a/channels/us.m3u +++ b/channels/us.m3u @@ -6615,9 +6615,9 @@ https://ln-zen.localnowlive.com/v1/master/385c85a93929f94966d0fb186fc33b431e6f1e https://rockentertainment-zoomoo-1.samsung.wurl.com/manifest/playlist.m3u8 #EXTINF:-1 tvg-id="KanalDisney.us" tvg-name="Канал Disney" tvg-country="RU" tvg-language="Russian" tvg-logo="https://i.imgur.com/Q9KoVy9.png" group-title="Kids",Канал Disney (576p) http://188.40.68.167/russia/disney/playlist.m3u8 -#EXTINF:-1 tvg-id="WSNN-LD1.us" tvg-name="WSNN-LD1" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/75548/s75548_h3_aa.png" group-title="Locals",WSNN-LD1 (Suncoast News Network) +#EXTINF:-1 tvg-id="WSNNLD1.us" tvg-name="WSNN-LD1" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/75548/s75548_h3_aa.png" group-title="Locals",WSNN-LD1 (Suncoast News Network) https://cdn88.theus6tv.tk/united-states/locals/tampa/wsnn-ld.m3u8 -#EXTINF:-1 tvg-id="WISC-DT2.us" tvg-name="WISC-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/25125/s51307_h3_aa.png" group-title="Locals",WISC-DT2 (My Madison TV) +#EXTINF:-1 tvg-id="WISCDT2.us" tvg-name="WISC-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/25125/s51307_h3_aa.png" group-title="Locals",WISC-DT2 (My Madison TV) https://cdn88.theus6tv.tk/united-states/locals/madison/wisc-dt2.m3u8 -#EXTINF:-1 tvg-id="WFMZ-DT2.us" tvg-name="WFMZ-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/35859/s35859_h3_aa.png" group-title="Locals",WFMZ-DT2 (69News Weather Channel) +#EXTINF:-1 tvg-id="WFMZDT2.us" tvg-name="WFMZ-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/35859/s35859_h3_aa.png" group-title="Locals",WFMZ-DT2 (69News Weather Channel) [Offline] https://cdn88.theus6tv.tk/united-states/locals/philadelphia/wfmz-dt2.m3u8 From 82a3f824f22416b7cbbb2704572a4ce3b9aeda82 Mon Sep 17 00:00:00 2001 From: Dum4G Date: Tue, 10 Aug 2021 23:45:32 +0300 Subject: [PATCH 04/10] Revert "Update "format" script" --- .github/workflows/auto-update.yml | 161 ++++++++++--------- .github/workflows/clean.yml | 257 ++++++++++++++++++++++++++++++ CONTRIBUTING.md | 7 +- package-lock.json | 20 +-- package.json | 4 +- scripts/clean.js | 74 +++++++++ scripts/detect-resolution.js | 114 +++++++++++++ scripts/format.js | 153 +++--------------- scripts/helpers/Channel.js | 5 - scripts/helpers/utils.js | 11 ++ 10 files changed, 579 insertions(+), 227 deletions(-) create mode 100644 .github/workflows/clean.yml create mode 100644 scripts/clean.js create mode 100644 scripts/detect-resolution.js diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 51cf6df50f..213deb39b0 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -20,6 +20,90 @@ jobs: format: runs-on: ubuntu-latest needs: create-branch + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Format Playlists + run: node scripts/format.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Formate playlists' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + sort: + runs-on: ubuntu-latest + needs: format + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Sort Channels + run: node scripts/sort.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Sort channels' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + remove-duplicates: + runs-on: ubuntu-latest + needs: sort + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Remove Duplicates + run: node scripts/remove-duplicates.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Remove duplicates' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + filter: + runs-on: ubuntu-latest + needs: remove-duplicates + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Filter Playlists + run: node scripts/filter.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Filter channels' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + detect-resolution: + runs-on: ubuntu-latest + needs: filter continue-on-error: true strategy: fail-fast: false @@ -193,12 +277,10 @@ jobs: uses: actions/checkout@v2 with: ref: bot/auto-update - - name: Setup FFmpeg - uses: FedericoCarboni/setup-ffmpeg@v1 - name: Install Dependencies run: npm install - - name: Format Playlists - run: node scripts/format.js --country=${{ matrix.country }} --status --resolution --debug + - name: Detect Resolution + run: node scripts/detect-resolution.js --country=${{ matrix.country }} - name: Upload Artifact uses: actions/upload-artifact@v2 with: @@ -206,7 +288,7 @@ jobs: path: channels/${{ matrix.country }}.m3u commit-changes: runs-on: ubuntu-latest - needs: format + needs: detect-resolution steps: - name: Checkout uses: actions/checkout@v2 @@ -220,70 +302,7 @@ jobs: - name: Commit Changes uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: '[Bot] Format playlists' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - remove-duplicates: - runs-on: ubuntu-latest - needs: commit-changes - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Remove Duplicates - run: node scripts/remove-duplicates.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Remove duplicates' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - sort: - runs-on: ubuntu-latest - needs: remove-duplicates - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Sort Channels - run: node scripts/sort.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Sort channels' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - filter: - runs-on: ubuntu-latest - needs: sort - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Filter Playlists - run: node scripts/filter.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Filter channels' + commit_message: '[Bot] Detect resolution' commit_user_name: iptv-bot commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' @@ -291,7 +310,7 @@ jobs: file_pattern: channels/* generate: runs-on: ubuntu-latest - needs: filter + needs: commit-changes steps: - name: Checkout uses: actions/checkout@v2 @@ -357,7 +376,6 @@ jobs: branch: bot/auto-update file_pattern: README.md pull-request: - if: ${{ github.ref == 'refs/heads/master' }} needs: update-readme runs-on: ubuntu-latest steps: @@ -390,6 +408,7 @@ jobs: pull-request-number: ${{ steps.pr.outputs.pr_number }} merge-method: squash - name: Approve Pull Request + if: github.ref == 'refs/heads/master' uses: juliangruber/approve-pull-request-action@v1 with: github-token: ${{ secrets.PAT }} diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml new file mode 100644 index 0000000000..3c1f9a4520 --- /dev/null +++ b/.github/workflows/clean.yml @@ -0,0 +1,257 @@ +name: clean +on: + workflow_dispatch: + schedule: + - cron: '0 6 * * 0' +jobs: + create-branch: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Create Branch + uses: peterjgrainger/action-create-branch@v2.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + branch: 'bot/remove-broken-links' + check: + runs-on: ubuntu-latest + needs: create-branch + continue-on-error: true + strategy: + fail-fast: false + matrix: + country: + [ + ad, + ae, + af, + ag, + al, + am, + an, + ao, + ar, + at, + au, + aw, + az, + ba, + bb, + bd, + be, + bf, + bg, + bh, + bn, + bo, + br, + bs, + by, + ca, + cd, + cg, + ch, + ci, + cl, + cm, + cn, + co, + cr, + cu, + cw, + cy, + cz, + de, + dk, + do, + dz, + ec, + ee, + eg, + es, + et, + fi, + fj, + fo, + fr, + pf, + ge, + gh, + gm, + gn, + gp, + gq, + gr, + gt, + hk, + hn, + hr, + ht, + hu, + id, + ie, + il, + in, + iq, + ir, + is, + it, + jm, + jo, + jp, + ke, + kg, + kh, + kp, + kr, + kw, + kz, + la, + lb, + li, + lk, + lt, + lu, + lv, + ly, + ma, + mc, + md, + me, + mk, + ml, + mm, + mn, + mo, + mt, + mv, + mx, + my, + mz, + ne, + ng, + ni, + nl, + no, + np, + nz, + om, + pa, + pe, + ph, + pk, + pl, + pr, + ps, + pt, + py, + qa, + ro, + rs, + ru, + rw, + sa, + sd, + se, + sg, + si, + sk, + sl, + sm, + sn, + so, + sv, + sy, + th, + tj, + tm, + tn, + tr, + tt, + tw, + tz, + ua, + ug, + uk, + us, + uy, + uz, + va, + ve, + vi, + vn, + xk, + ye, + zm, + unsorted + ] + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/remove-broken-links + - name: Setup FFmpeg + uses: FedericoCarboni/setup-ffmpeg@v1 + - name: Install Dependencies + run: npm install + - name: Remove Broken Links + run: node scripts/clean.js --country=${{ matrix.country }} --debug + - name: Upload Artifact + uses: actions/upload-artifact@v2 + with: + name: channels + path: channels/${{ matrix.country }}.m3u + commit-changes: + runs-on: ubuntu-latest + needs: check + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/remove-broken-links + - name: Download Artifacts + uses: actions/download-artifact@v2 + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Remove broken links' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/remove-broken-links + file_pattern: channels/* + pull-request: + if: ${{ github.ref == 'refs/heads/master' }} + runs-on: ubuntu-latest + needs: commit-changes + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/remove-broken-links + - name: Generate Token + uses: tibdex/github-app-token@v1 + id: generate-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Create Pull Request + uses: repo-sync/pull-request@v2 + with: + source_branch: 'bot/remove-broken-links' + destination_branch: 'master' + pr_title: '[Bot] Remove broken links' + pr_body: | + This pull request is created by [clean][1] workflow. + + The script checks all links except those with labels `[Geo-blocked]`, `[Offline]` or `[Not 24/7]` in the title. + + **IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. If the link works for you but occasionally return an HTTP code 403 (Forbidden) then it should be marked as `[Geo-blocked]`. If the link does not work but has no alternative, you can mark it as `[Offline]` to save it in the playlist along with a description. Working links should be marked as `[Not 24/7]` so that the script will skip them next time. + + [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} + pr_draft: true + github_token: ${{ steps.generate-token.outputs.token }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef08f6c077..a111b67135 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -125,7 +125,7 @@ STREAM_URL | `LANGUAGE` | Channel language. The name of the language must conform to the standard [ISO 639-3](https://iso639-3.sil.org/code_tables/639/data?title=&field_iso639_cd_st_mmbrshp_639_1_tid=94671&name_3=&field_iso639_element_scope_tid=All&field_iso639_language_type_tid=51&items_per_page=500). If the channel is broadcast in several languages you can list them separated by a semicolon. (optional) | | `LOGO_URL` | The logo of the channel that will be displayed if the player supports it. Supports files in png, jpeg and gif format. (optional) | | `CATEGORY` | The category to which the channel belongs. The list of currently supported categories can be found [here](https://github.com/iptv-org/iptv#playlists-by-category). (optional) | -| `FULL_NAME` | Full name of the channel. It is recommended to use the name listed on [lyngsat](https://www.lyngsat.com/search.html) or [wikipedia](https://www.wikipedia.org/) if possible. May contain any characters except round and square brackets. | +| `FULL_NAME` | Full name of the channel. It is recommended to use the name listed on [lyngsat](https://www.lyngsat.com/search.html) or [wikipedia](https://www.wikipedia.org/) if possible. May contain any characters except plus sign, minus sign, round and square brackets. | | `STREAM_TIME_SHIFT` | Must be specified if the channel is broadcast with a shift in time relative to the main stream. Should only contain a number and a sign. (optional) | | `ALTERNATIVE_NAME` | Can be used to specify a short name or name in another language. May contain any characters except round and square brackets. (optional) | | `STREAM_RESOLUTION` | The maximum height of the frame with a "p" at the end. In case of VLC Player this information can be found in `Window > Media Information... > Codec Details`. (optional) | @@ -153,8 +153,7 @@ http://example.com/stream.m3u8 - `.github/` - `ISSUE_TEMPLATE/`: issue templates for this repository. - `workflows/` - - `auto-update.yml`: GitHub Action that automatically updates all playlists every day. - - `check.yml`: GitHub Action that automatically checks every pull request for syntax errors. + - `auto-update.yml`: contain actions that automatically updates all playlists every day. - `CODE_OF_CONDUCT.md`: rules you shouldn't break if you don't want to get banned. - `.readme/` - `_categories.md`: automatically generated list of all categories and their corresponding playlists. @@ -169,6 +168,8 @@ http://example.com/stream.m3u8 - `unsorted.m3u`: playlist with channels not yet sorted. - `scripts/` - `helpers/`: helper scripts used in GitHub Actions. + - `clean.js`: used in GitHub Action to check all links and remove broken ones. + - `detect-resolution.js`: used in GitHub Action to detect resolution of the streams. - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists. - `format.js`: used within GitHub Action to format channel descriptions. - `generate.js`: used within GitHub Action to generate all additional playlists. diff --git a/package-lock.json b/package-lock.json index ffc2acf870..06cebc1d6c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,12 +7,14 @@ "license": "MIT", "dependencies": { "@freearhey/iso-639-3": "^1.0.0", + "axios": "^0.21.1", + "chalk": "^4.1.1", "commander": "^7.0.0", + "escape-string-regexp": "^2.0.0", "iptv-checker": "^0.20.2", "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.1.3", "markdown-include": "^0.4.3", - "normalize-url": "^6.1.0", "pre-push": "^0.1.1", "progress": "^2.0.3", "transliteration": "^2.2.0" @@ -2961,17 +2963,6 @@ "node": ">=0.10.0" } }, - "node_modules/normalize-url": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", - "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -6054,11 +6045,6 @@ "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" }, - "normalize-url": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", - "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==" - }, "npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", diff --git a/package.json b/package.json index 64d55e8f7b..ea37000635 100644 --- a/package.json +++ b/package.json @@ -11,12 +11,14 @@ "license": "MIT", "dependencies": { "@freearhey/iso-639-3": "^1.0.0", + "axios": "^0.21.1", + "chalk": "^4.1.1", "commander": "^7.0.0", + "escape-string-regexp": "^2.0.0", "iptv-checker": "^0.20.2", "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.1.3", "markdown-include": "^0.4.3", - "normalize-url": "^6.1.0", "pre-push": "^0.1.1", "progress": "^2.0.3", "transliteration": "^2.2.0" diff --git a/scripts/clean.js b/scripts/clean.js new file mode 100644 index 0000000000..d1ab7c2e7e --- /dev/null +++ b/scripts/clean.js @@ -0,0 +1,74 @@ +const IPTVChecker = require('iptv-checker') +const { program } = require('commander') +const ProgressBar = require('progress') +const parser = require('./helpers/parser') +const utils = require('./helpers/utils') +const log = require('./helpers/log') + +program + .usage('[OPTIONS]...') + .option('-d, --debug', 'Enable debug mode') + .option('-c, --country ', 'Comma-separated list of country codes', '') + .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') + .option('--timeout ', 'Set timeout for each request', 5000) + .parse(process.argv) + +let bar +const config = program.opts() +const ignoreStatus = ['Geo-blocked', 'Not 24/7', 'Offline'] +const checker = new IPTVChecker({ + timeout: config.timeout +}) + +async function main() { + log.start() + + if (config.debug) log.print(`Debug mode enabled\n`) + + let playlists = parser.parseIndex() + playlists = utils.filterPlaylists(playlists, config.country, config.exclude) + for (const playlist of playlists) { + await parser + .parsePlaylist(playlist.url) + .then(checkPlaylist) + .then(p => p.save()) + } + + log.finish() +} + +async function checkPlaylist(playlist) { + if (!config.debug) { + bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, { + total: playlist.channels.length + }) + } + const channels = [] + const total = playlist.channels.length + for (const [index, channel] of playlist.channels.entries()) { + const skipChannel = + channel.status && + ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) + if (skipChannel) { + channels.push(channel) + } else { + const result = await checker.checkStream(channel.data) + if (result.status.ok || result.status.reason.includes('timed out')) { + channels.push(channel) + } else { + if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`) + } + } + if (!config.debug) bar.tick() + } + + if (playlist.channels.length !== channels.length) { + log.print(`File '${playlist.url}' has been updated\n`) + playlist.channels = channels + playlist.updated = true + } + + return playlist +} + +main() diff --git a/scripts/detect-resolution.js b/scripts/detect-resolution.js new file mode 100644 index 0000000000..5da6a282f5 --- /dev/null +++ b/scripts/detect-resolution.js @@ -0,0 +1,114 @@ +const { program } = require('commander') +const ProgressBar = require('progress') +const axios = require('axios') +const https = require('https') +const parser = require('./helpers/parser') +const utils = require('./helpers/utils') +const log = require('./helpers/log') + +program + .usage('[OPTIONS]...') + .option('-c, --country ', 'Comma-separated list of country codes', '') + .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') + .option('--delay ', 'Delay between parser requests', 1000) + .option('--timeout ', 'Set timeout for each request', 5000) + .parse(process.argv) + +const config = program.opts() +const ignoreStatus = ['Offline'] +const instance = axios.create({ + timeout: config.timeout, + maxContentLength: 200000, + httpsAgent: new https.Agent({ + rejectUnauthorized: false + }) +}) + +async function main() { + log.start() + + log.print(`Parsing 'index.m3u'...\n`) + let playlists = parser.parseIndex() + playlists = utils + .filterPlaylists(playlists, config.country, config.exclude) + .filter(i => i.url !== 'channels/unsorted.m3u') + + for (const playlist of playlists) { + await parser + .parsePlaylist(playlist.url) + .then(detectResolution) + .then(p => p.save()) + } + + log.finish() +} + +async function detectResolution(playlist) { + const channels = [] + const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { + total: playlist.channels.length + }) + let updated = false + for (const channel of playlist.channels) { + bar.tick() + const skipChannel = + channel.status && + ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) + if (!channel.resolution.height && !skipChannel) { + const CancelToken = axios.CancelToken + const source = CancelToken.source() + const timeout = setTimeout(() => { + source.cancel() + }, config.timeout) + + const response = await instance + .get(channel.url, { cancelToken: source.token }) + .then(res => { + clearTimeout(timeout) + + return res + }) + .then(utils.sleep(config.delay)) + .catch(err => { + clearTimeout(timeout) + }) + + if (response && response.status === 200) { + if (/^#EXTM3U/.test(response.data)) { + const resolution = parseResolution(response.data) + if (resolution) { + channel.resolution = resolution + updated = true + } + } + } + } + + channels.push(channel) + } + + if (updated) { + log.print(`File '${playlist.url}' has been updated\n`) + playlist.channels = channels + playlist.updated = true + } + + return playlist +} + +function parseResolution(string) { + const regex = /RESOLUTION=(\d+)x(\d+)/gm + const match = string.matchAll(regex) + const arr = Array.from(match).map(m => ({ + width: parseInt(m[1]), + height: parseInt(m[2]) + })) + + return arr.length + ? arr.reduce(function (prev, current) { + return prev.height > current.height ? prev : current + }) + : undefined +} + +main() diff --git a/scripts/format.js b/scripts/format.js index acc2fca3c8..fb77cab20f 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -1,46 +1,21 @@ -const IPTVChecker = require('iptv-checker') -const normalize = require('normalize-url') -const { program } = require('commander') -const ProgressBar = require('progress') const parser = require('./helpers/parser') const utils = require('./helpers/utils') const file = require('./helpers/file') const log = require('./helpers/log') -program - .usage('[OPTIONS]...') - .option('-d, --debug', 'Enable debug mode') - .option('-s, --status', 'Update stream status') - .option('-r, --resolution', 'Detect stream resolution') - .option('-c, --country ', 'Comma-separated list of country codes', '') - .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') - .option('--timeout ', 'Set timeout for each request', 5000) - .parse(process.argv) - -let bar -const ignoreStatus = ['Geo-blocked', 'Not 24/7'] -const config = program.opts() -const checker = new IPTVChecker({ - timeout: config.timeout -}) - async function main() { log.start() - if (config.debug) log.print(`Debug mode enabled\n`) - if (config.status) log.print(`Status check enabled\n`) - if (config.resolution) log.print(`Resolution detection enabled\n`) - + log.print(`Parsing 'index.m3u'...`) let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') - playlists = utils.filterPlaylists(playlists, config.country, config.exclude) - if (!playlists.length) log.print(`No playlist is selected\n`) for (const playlist of playlists) { + log.print(`\nProcessing '${playlist.url}'...`) await parser .parsePlaylist(playlist.url) - .then(updatePlaylist) + .then(formatPlaylist) .then(playlist => { if (file.read(playlist.url) !== playlist.toString()) { - log.print(`File '${playlist.url}' has been updated\n`) + log.print('updated') playlist.updated = true } @@ -48,115 +23,33 @@ async function main() { }) } + log.print('\n') log.finish() } -async function updatePlaylist(playlist) { - if (!config.debug) { - bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { - total: playlist.channels.length - }) - } else { - log.print(`Processing '${playlist.url}'...\n`) - } - +async function formatPlaylist(playlist) { for (const channel of playlist.channels) { - addMissingData(channel) - updateGroupTitle(channel) - normalizeUrl(channel) - - const checkOnline = config.status || config.resolution - const skipChannel = - channel.status && - ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) - if (checkOnline && !skipChannel) { - await checker - .checkStream(channel.data) - .then(result => { - const status = parseStatus(result.status) - - if (config.status) { - updateStatus(channel, status) - } - - if (config.resolution && status === 'online') { - updateResolution(channel, result.status.metadata) - } - - if (config.debug && status === 'offline') { - log.print(` ERR: ${channel.url} (${result.status.reason})\n`) - } - }) - .catch(err => { - if (config.debug) log.print(` ERR: ${channel.url} (${err.message})\n`) - }) + const code = file.getBasename(playlist.url) + // add missing tvg-name + if (!channel.tvg.name && code !== 'unsorted' && channel.name) { + channel.tvg.name = channel.name.replace(/\"/gi, '') } - if (!config.debug) bar.tick() + // add missing tvg-id + if (!channel.tvg.id && code !== 'unsorted' && channel.tvg.name) { + const id = utils.name2id(channel.tvg.name) + channel.tvg.id = id ? `${id}.${code}` : '' + } + // add missing country + if (!channel.countries.length) { + const name = utils.code2name(code) + channel.countries = name ? [{ code, name }] : [] + channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') + } + // update group-title + channel.group.title = channel.category } return playlist } -function parseStatus(status) { - if (status.ok) { - return 'online' - } else if (status.reason.includes('timed out')) { - return 'timeout' - } else if (status.reason.includes('403')) { - return 'error_403' - } else if (status.reason.includes('not one of 40{0,1,3,4}')) { - return 'error_40x' // 402, 451 - } else { - return 'offline' - } -} - -function updateStatus(channel, status) { - switch (status) { - case 'online': - channel.status = null - break - case 'offline': - channel.status = 'Offline' - break - } -} - -function addMissingData(channel) { - // tvg-name - if (!channel.tvg.name && channel.name) { - channel.tvg.name = channel.name.replace(/\"/gi, '') - } - // tvg-id - if (!channel.tvg.id && channel.tvg.name) { - const id = utils.name2id(channel.tvg.name) - channel.tvg.id = id ? `${id}.${code}` : '' - } - // country - if (!channel.countries.length) { - const name = utils.code2name(code) - channel.countries = name ? [{ code, name }] : [] - channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') - } -} - -function updateGroupTitle(channel) { - channel.group.title = channel.category -} - -function normalizeUrl(channel) { - const normalized = normalize(channel.url, { stripWWW: false }) - channel.updateUrl(normalized) -} - -function updateResolution(channel, metadata) { - const streams = metadata ? metadata.streams.filter(stream => stream.codec_type === 'video') : [] - if (!channel.resolution.height && streams.length) { - channel.resolution = streams.reduce((acc, curr) => { - if (curr.height > acc.height) return { width: curr.width, height: curr.height } - return acc - }) - } -} - main() diff --git a/scripts/helpers/Channel.js b/scripts/helpers/Channel.js index 8c4b0c397a..043f7886f3 100644 --- a/scripts/helpers/Channel.js +++ b/scripts/helpers/Channel.js @@ -22,11 +22,6 @@ module.exports = class Channel { this.languages = this.parseLanguages(data.tvg.language) } - updateUrl(url) { - this.url = url - this.data.url = url - } - parseName(title) { return title .trim() diff --git a/scripts/helpers/utils.js b/scripts/helpers/utils.js index f0d9b3e50d..7a2bae0633 100644 --- a/scripts/helpers/utils.js +++ b/scripts/helpers/utils.js @@ -1,3 +1,4 @@ +const escapeStringRegexp = require('escape-string-regexp') const transliteration = require('transliteration') const iso6393 = require('@freearhey/iso-639-3') const categories = require('./categories') @@ -70,6 +71,16 @@ utils.sortBy = function (arr, fields) { }) } +utils.escapeStringRegexp = function (scring) { + return escapeStringRegexp(string) +} + +utils.sleep = function (ms) { + return function (x) { + return new Promise(resolve => setTimeout(() => resolve(x), ms)) + } +} + utils.removeProtocol = function (string) { return string.replace(/(^\w+:|^)\/\//, '') } From 9da553e51ecbd243849cb50c8a8a5f1867c1f8f9 Mon Sep 17 00:00:00 2001 From: "iptv-bot[bot]" <84861620+iptv-bot[bot]@users.noreply.github.com> Date: Wed, 11 Aug 2021 00:56:38 +0000 Subject: [PATCH 05/10] [Bot] Update playlists (#4039) * [Bot] Formate playlists * [Bot] Sort channels * [Bot] Detect resolution * [Bot] Update README.md Co-authored-by: iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com> --- README.md | 16 ++++++++-------- channels/kw.m3u | 2 +- channels/lb.m3u | 2 +- channels/ru.m3u | 2 +- channels/us.m3u | 12 ++++++------ 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 19431a44fe..9848dbd624 100644 --- a/README.md +++ b/README.md @@ -45,13 +45,13 @@ Or select one of the playlists from the list below. Education22https://iptv-org.github.io/iptv/categories/education.m3u Entertainment212https://iptv-org.github.io/iptv/categories/entertainment.m3u Family54https://iptv-org.github.io/iptv/categories/family.m3u - General338https://iptv-org.github.io/iptv/categories/general.m3u + General339https://iptv-org.github.io/iptv/categories/general.m3u Kids213https://iptv-org.github.io/iptv/categories/kids.m3u Legislative61https://iptv-org.github.io/iptv/categories/legislative.m3u Lifestyle94https://iptv-org.github.io/iptv/categories/lifestyle.m3u Local549https://iptv-org.github.io/iptv/categories/local.m3u Movies278https://iptv-org.github.io/iptv/categories/movies.m3u - Music410https://iptv-org.github.io/iptv/categories/music.m3u + Music411https://iptv-org.github.io/iptv/categories/music.m3u News492https://iptv-org.github.io/iptv/categories/news.m3u Outdoor52https://iptv-org.github.io/iptv/categories/outdoor.m3u Relax19https://iptv-org.github.io/iptv/categories/relax.m3u @@ -63,7 +63,7 @@ Or select one of the playlists from the list below. Travel28https://iptv-org.github.io/iptv/categories/travel.m3u Weather19https://iptv-org.github.io/iptv/categories/weather.m3u XXX59https://iptv-org.github.io/iptv/categories/xxx.m3u - Other4877https://iptv-org.github.io/iptv/categories/other.m3u + Other4880https://iptv-org.github.io/iptv/categories/other.m3u @@ -84,7 +84,7 @@ Or select one of the playlists from the list below. Akan2https://iptv-org.github.io/iptv/languages/aka.m3u Albanian17https://iptv-org.github.io/iptv/languages/sqi.m3u Amharic1https://iptv-org.github.io/iptv/languages/amh.m3u - Arabic413https://iptv-org.github.io/iptv/languages/ara.m3u + Arabic415https://iptv-org.github.io/iptv/languages/ara.m3u Armenian28https://iptv-org.github.io/iptv/languages/hye.m3u Assyrian Neo-Aramaic1https://iptv-org.github.io/iptv/languages/aii.m3u Azerbaijani10https://iptv-org.github.io/iptv/languages/aze.m3u @@ -99,7 +99,7 @@ Or select one of the playlists from the list below. Czech21https://iptv-org.github.io/iptv/languages/ces.m3u Danish4https://iptv-org.github.io/iptv/languages/dan.m3u Dutch60https://iptv-org.github.io/iptv/languages/nld.m3u - English2438https://iptv-org.github.io/iptv/languages/eng.m3u + English2441https://iptv-org.github.io/iptv/languages/eng.m3u Estonian3https://iptv-org.github.io/iptv/languages/est.m3u Faroese1https://iptv-org.github.io/iptv/languages/fao.m3u Finnish1https://iptv-org.github.io/iptv/languages/fin.m3u @@ -286,7 +286,7 @@ Or select one of the playlists from the list below. 🇰🇬 Kyrgyzstan3https://iptv-org.github.io/iptv/countries/kg.m3u 🇱🇦 Laos6https://iptv-org.github.io/iptv/countries/la.m3u 🇱🇻 Latvia9https://iptv-org.github.io/iptv/countries/lv.m3u - 🇱🇧 Lebanon69https://iptv-org.github.io/iptv/countries/lb.m3u + 🇱🇧 Lebanon71https://iptv-org.github.io/iptv/countries/lb.m3u 🇱🇸 Lesotho1https://iptv-org.github.io/iptv/countries/ls.m3u 🇱🇷 Liberia1https://iptv-org.github.io/iptv/countries/lr.m3u 🇱🇾 Libya53https://iptv-org.github.io/iptv/countries/ly.m3u @@ -336,7 +336,7 @@ Or select one of the playlists from the list below. 🇵🇦 Panama30https://iptv-org.github.io/iptv/countries/pa.m3u 🇵🇬 Papua New Guinea3https://iptv-org.github.io/iptv/countries/pg.m3u 🇵🇾 Paraguay28https://iptv-org.github.io/iptv/countries/py.m3u - 🇵🇪 Peru106https://iptv-org.github.io/iptv/countries/pe.m3u + 🇵🇪 Peru108https://iptv-org.github.io/iptv/countries/pe.m3u 🇵🇭 Philippines20https://iptv-org.github.io/iptv/countries/ph.m3u 🇵🇳 Pitcairn Islands3https://iptv-org.github.io/iptv/countries/pn.m3u 🇵🇱 Poland45https://iptv-org.github.io/iptv/countries/pl.m3u @@ -390,7 +390,7 @@ Or select one of the playlists from the list below. 🇺🇦 Ukraine116https://iptv-org.github.io/iptv/countries/ua.m3u 🇦🇪 United Arab Emirates88https://iptv-org.github.io/iptv/countries/ae.m3u 🇬🇧 United Kingdom229https://iptv-org.github.io/iptv/countries/uk.m3u - 🇺🇸 United States2266https://iptv-org.github.io/iptv/countries/us.m3u + 🇺🇸 United States2269https://iptv-org.github.io/iptv/countries/us.m3u 🇺🇾 Uruguay22https://iptv-org.github.io/iptv/countries/uy.m3u 🇺🇿 Uzbekistan3https://iptv-org.github.io/iptv/countries/uz.m3u 🇻🇺 Vanuatu3https://iptv-org.github.io/iptv/countries/vu.m3u diff --git a/channels/kw.m3u b/channels/kw.m3u index 5cb39f24a4..7ee08a4d8e 100644 --- a/channels/kw.m3u +++ b/channels/kw.m3u @@ -17,7 +17,7 @@ http://www.elahmad.com/tv/m3u8/alkass.m3u8?id=alkass1 http://www.elahmad.com/tv/m3u8/alkass.m3u8?id=alkass2 #EXTINF:-1 tvg-id="Alkasssport3.kw" tvg-name="Alkass sport 3" tvg-country="KW" tvg-language="" tvg-logo="" group-title="Sports",Alkass sport 3 http://www.elahmad.com/tv/m3u8/alkass.m3u8?id=alkass3 -#EXTINF:-1 tvg-id="Alkasssport4.kw" tvg-name="Alkass sport 4" tvg-country="KW" tvg-language="" tvg-logo="" group-title="Sports",Alkass sport 4 +#EXTINF:-1 tvg-id="Alkasssport4.kw" tvg-name="Alkass sport 4" tvg-country="KW" tvg-language="" tvg-logo="" group-title="Sports",Alkass sport 4 (576p) http://www.elahmad.com/tv/m3u8/alkass.m3u8?id=alkass4 #EXTINF:-1 tvg-id="Alkasssport5.kw" tvg-name="Alkass sport 5" tvg-country="KW" tvg-language="" tvg-logo="" group-title="Sports",Alkass sport 5 http://www.elahmad.com/tv/m3u8/alkass.m3u8?id=alkass5 diff --git a/channels/lb.m3u b/channels/lb.m3u index 2ed9d974b3..2a1aea6a09 100644 --- a/channels/lb.m3u +++ b/channels/lb.m3u @@ -1,5 +1,5 @@ #EXTM3U -#EXTINF:-1 tvg-id="" tvg-name="" tvg-language="Arabic" tvg-logo="https://aghaniaghani.com/images/logo.png" tvg-country="LB" group-title="Music",Aghani Aghani (1080p) +#EXTINF:-1 tvg-id="AghaniAghani.lb" tvg-name="Aghani Aghani" tvg-country="LB" tvg-language="Arabic" tvg-logo="https://aghaniaghani.com/images/logo.png" group-title="Music",Aghani Aghani (1080p) https://svs.itworkscdn.net/aghanilive/aghanilive/playlist.m3u8 #EXTINF:-1 tvg-id="AlIttihad.lb" tvg-name="Al Ittihad" tvg-country="LB" tvg-language="Arabic" tvg-logo="http://alittihad.tv/assets/images/logo.png" group-title="General",Al Ittihad (552p) https://live.alittihad.tv/ittihad/index.m3u8 diff --git a/channels/ru.m3u b/channels/ru.m3u index 7a93158f4c..5a83cdf8a1 100644 --- a/channels/ru.m3u +++ b/channels/ru.m3u @@ -705,7 +705,7 @@ https://live2.mediacdn.ru/sr1/sever-mobile/playlist.m3u8 https://live2.mediacdn.ru/sr1/sever/playlist.m3u8 #EXTINF:-1 tvg-id="SelengaTV.ru" tvg-name="Селенга ТВ" tvg-country="RU" tvg-language="" tvg-logo="" group-title="",Селенга ТВ http://90.188.37.86/live/playlist.m3u8 -#EXTINF:-1 tvg-id="Siesta.ru" tvg-name="Сиеста" tvg-country="RU" tvg-language="Russian" tvg-logo="" group-title="",Сиеста +#EXTINF:-1 tvg-id="Siesta.ru" tvg-name="Сиеста" tvg-country="RU" tvg-language="Russian" tvg-logo="" group-title="",Сиеста (720p) https://1hdru-hls-otcnet.cdnvideo.ru/siesta/index.m3u8 #EXTINF:-1 tvg-id="SmaylikTV.ru" tvg-name="Смайлик ТВ" tvg-country="RU" tvg-language="Russian" tvg-logo="http://smilik.tv/wp-content/uploads/2016/02/Desktop_-1.png" group-title="Kids",Смайлик ТВ (720p) http://62.32.67.187:1935/WEB_Smilik/ngrp:Smilik.stream-adaptive/playlist.m3u8 diff --git a/channels/us.m3u b/channels/us.m3u index cb0ebf5423..76a7f5260d 100644 --- a/channels/us.m3u +++ b/channels/us.m3u @@ -6455,6 +6455,8 @@ https://jukin-weatherspy-2-eu.rakuten.wurl.com/manifest/playlist.m3u8 https://dai.google.com/linear/hls/event/fLqJePs_QR-FRTttC8fMIA/master.m3u8 #EXTINF:-1 tvg-id="WETA.us" tvg-name="WETA Metro (WETA) (Do Not Broadcast .1)" tvg-country="US" tvg-language="English" tvg-logo="https://image.pbs.org/stations/weta-color-single-brand-logo-ZeUHAPm.png" group-title="General",WETA Metro (WETA) (Do Not Broadcast .1) (1080p) [Geo-blocked] https://wetadt5.lls.cdn.pbs.org/wetadt5/92d89794-5ff0-4fe6-a443-cc888104e021/primary.m3u8 +#EXTINF:-1 tvg-id="WFMZDT2.us" tvg-name="WFMZ-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/35859/s35859_h3_aa.png" group-title="",WFMZ-DT2 (69News Weather Channel) [Offline] +https://cdn88.theus6tv.tk/united-states/locals/philadelphia/wfmz-dt2.m3u8 #EXTINF:-1 tvg-id="WGN.us" tvg-name="WGN 9 Chicago IL" tvg-country="US" tvg-language="English" tvg-logo="https://i.imgur.com/dpI43Wg.png" group-title="Local",WGN 9 Chicago IL (720p) https://bozztv.com/teleyupp1/teleup-mBm5MQ50rA/playlist.m3u8 #EXTINF:-1 tvg-id="WhistleSports.us" tvg-name="Whistle Sports" tvg-country="US" tvg-language="English" tvg-logo="https://i.imgur.com/CNO8eQ7.png" group-title="Sports",Whistle Sports (720p) @@ -6501,6 +6503,8 @@ https://dai.google.com/linear/hls/event/0DG8p66IRES7ZzEe1WJS-w/master.m3u8 https://dai2.xumo.com/amagi_hls_data_xumo1212A-redboxwired/CDN/playlist.m3u8 #EXTINF:-1 tvg-id="Wired2Fish.us" tvg-name="Wired2Fish" tvg-country="US" tvg-language="English" tvg-logo="https://f9q4g5j6.ssl.hwcdn.net/605d1769345cf61eb60b5893" group-title="Sports",Wired2Fish (720p) https://linear-80.frequency.stream/dist/localnow/80/hls/master/playlist.m3u8 +#EXTINF:-1 tvg-id="WISCDT2.us" tvg-name="WISC-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/25125/s51307_h3_aa.png" group-title="",WISC-DT2 (My Madison TV) (720p) +https://cdn88.theus6tv.tk/united-states/locals/madison/wisc-dt2.m3u8 #EXTINF:-1 tvg-id="WisconsinGreenBaySTIRR.us" tvg-name="Wisconsin: Green Bay (STIRR)" tvg-country="US" tvg-language="English" tvg-logo="https://i.imgur.com/7JgPJxx.png" group-title="Local",Wisconsin: Green Bay (STIRR) (720p) https://dai.google.com/linear/hls/event/a6lsWNYDQwyM9fjytUCrcw/master.m3u8 #EXTINF:-1 tvg-id="WisconsinMilwaukeeSTIRR.us" tvg-name="Wisconsin: Milwaukee (STIRR)" tvg-country="US" tvg-language="English" tvg-logo="https://i.imgur.com/7JgPJxx.png" group-title="Local",Wisconsin: Milwaukee (STIRR) (720p) @@ -6541,6 +6545,8 @@ https://world-poker-tour.samsung.wurl.com/manifest/playlist.m3u8 http://cdn.elsalvadordigital.com:1935/wowtv/wowtv/playlist.m3u8 #EXTINF:-1 tvg-id="wowkids.us" tvg-name="wow!! kids" tvg-country="US" tvg-language="English" tvg-logo="https://i.imgur.com/AWXw9LW.jpg" group-title="Kids",wow!! kids (1080p) https://wownow-wownowkids-1.vizio.wurl.com/manifest/playlist.m3u8 +#EXTINF:-1 tvg-id="WSNNLD1.us" tvg-name="WSNN-LD1" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/75548/s75548_h3_aa.png" group-title="",WSNN-LD1 (Suncoast News Network) (720p) +https://cdn88.theus6tv.tk/united-states/locals/tampa/wsnn-ld.m3u8 #EXTINF:-1 tvg-id="WuTangCollection.us" tvg-name="Wu Tang Collection" tvg-country="US" tvg-language="English" tvg-logo="https://i.imgur.com/jbMmsjI.png" group-title="Music",Wu Tang Collection (720p) https://stream-us-east-1.getpublica.com/playlist.m3u8?network_id=73 #EXTINF:-1 tvg-id="Xcorps.us" tvg-name="Xcorps" tvg-country="US" tvg-language="English" tvg-logo="https://i.imgur.com/SDKWWQu.png" group-title="Series",Xcorps (720p) @@ -6615,9 +6621,3 @@ https://ln-zen.localnowlive.com/v1/master/385c85a93929f94966d0fb186fc33b431e6f1e https://rockentertainment-zoomoo-1.samsung.wurl.com/manifest/playlist.m3u8 #EXTINF:-1 tvg-id="KanalDisney.us" tvg-name="Канал Disney" tvg-country="RU" tvg-language="Russian" tvg-logo="https://i.imgur.com/Q9KoVy9.png" group-title="Kids",Канал Disney (576p) http://188.40.68.167/russia/disney/playlist.m3u8 -#EXTINF:-1 tvg-id="WSNNLD1.us" tvg-name="WSNN-LD1" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/75548/s75548_h3_aa.png" group-title="Locals",WSNN-LD1 (Suncoast News Network) -https://cdn88.theus6tv.tk/united-states/locals/tampa/wsnn-ld.m3u8 -#EXTINF:-1 tvg-id="WISCDT2.us" tvg-name="WISC-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/25125/s51307_h3_aa.png" group-title="Locals",WISC-DT2 (My Madison TV) -https://cdn88.theus6tv.tk/united-states/locals/madison/wisc-dt2.m3u8 -#EXTINF:-1 tvg-id="WFMZDT2.us" tvg-name="WFMZ-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/35859/s35859_h3_aa.png" group-title="Locals",WFMZ-DT2 (69News Weather Channel) [Offline] -https://cdn88.theus6tv.tk/united-states/locals/philadelphia/wfmz-dt2.m3u8 From 8b2e4c73257490d5140640996d9c8f43636a4caa Mon Sep 17 00:00:00 2001 From: MapGuy11 <43685307+MapGuy11@users.noreply.github.com> Date: Tue, 10 Aug 2021 21:05:35 -0400 Subject: [PATCH 06/10] Remove Offline Bracket --- channels/us.m3u | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/us.m3u b/channels/us.m3u index 76a7f5260d..b5dbc725ad 100644 --- a/channels/us.m3u +++ b/channels/us.m3u @@ -6455,7 +6455,7 @@ https://jukin-weatherspy-2-eu.rakuten.wurl.com/manifest/playlist.m3u8 https://dai.google.com/linear/hls/event/fLqJePs_QR-FRTttC8fMIA/master.m3u8 #EXTINF:-1 tvg-id="WETA.us" tvg-name="WETA Metro (WETA) (Do Not Broadcast .1)" tvg-country="US" tvg-language="English" tvg-logo="https://image.pbs.org/stations/weta-color-single-brand-logo-ZeUHAPm.png" group-title="General",WETA Metro (WETA) (Do Not Broadcast .1) (1080p) [Geo-blocked] https://wetadt5.lls.cdn.pbs.org/wetadt5/92d89794-5ff0-4fe6-a443-cc888104e021/primary.m3u8 -#EXTINF:-1 tvg-id="WFMZDT2.us" tvg-name="WFMZ-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/35859/s35859_h3_aa.png" group-title="",WFMZ-DT2 (69News Weather Channel) [Offline] +#EXTINF:-1 tvg-id="WFMZDT2.us" tvg-name="WFMZ-DT2" tvg-country="US" tvg-language="English" tvg-logo="https://zap2it.tmsimg.com/h3/NowShowing/35859/s35859_h3_aa.png" group-title="",WFMZ-DT2 (69News Weather Channel) https://cdn88.theus6tv.tk/united-states/locals/philadelphia/wfmz-dt2.m3u8 #EXTINF:-1 tvg-id="WGN.us" tvg-name="WGN 9 Chicago IL" tvg-country="US" tvg-language="English" tvg-logo="https://i.imgur.com/dpI43Wg.png" group-title="Local",WGN 9 Chicago IL (720p) https://bozztv.com/teleyupp1/teleup-mBm5MQ50rA/playlist.m3u8 From d800aa0175e1d2a59081933baa4bba07cf644324 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 11 Aug 2021 18:52:51 +0300 Subject: [PATCH 07/10] Revert "Revert "Update "format" script"" This reverts commit 82a3f824f22416b7cbbb2704572a4ce3b9aeda82. --- .github/workflows/auto-update.yml | 161 +++++++++---------- .github/workflows/clean.yml | 257 ------------------------------ CONTRIBUTING.md | 7 +- package-lock.json | 20 ++- package.json | 4 +- scripts/clean.js | 74 --------- scripts/detect-resolution.js | 114 ------------- scripts/format.js | 153 +++++++++++++++--- scripts/helpers/Channel.js | 5 + scripts/helpers/utils.js | 11 -- 10 files changed, 227 insertions(+), 579 deletions(-) delete mode 100644 .github/workflows/clean.yml delete mode 100644 scripts/clean.js delete mode 100644 scripts/detect-resolution.js diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 213deb39b0..51cf6df50f 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -20,90 +20,6 @@ jobs: format: runs-on: ubuntu-latest needs: create-branch - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Format Playlists - run: node scripts/format.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Formate playlists' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - sort: - runs-on: ubuntu-latest - needs: format - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Sort Channels - run: node scripts/sort.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Sort channels' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - remove-duplicates: - runs-on: ubuntu-latest - needs: sort - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Remove Duplicates - run: node scripts/remove-duplicates.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Remove duplicates' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - filter: - runs-on: ubuntu-latest - needs: remove-duplicates - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Filter Playlists - run: node scripts/filter.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Filter channels' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - detect-resolution: - runs-on: ubuntu-latest - needs: filter continue-on-error: true strategy: fail-fast: false @@ -277,10 +193,12 @@ jobs: uses: actions/checkout@v2 with: ref: bot/auto-update + - name: Setup FFmpeg + uses: FedericoCarboni/setup-ffmpeg@v1 - name: Install Dependencies run: npm install - - name: Detect Resolution - run: node scripts/detect-resolution.js --country=${{ matrix.country }} + - name: Format Playlists + run: node scripts/format.js --country=${{ matrix.country }} --status --resolution --debug - name: Upload Artifact uses: actions/upload-artifact@v2 with: @@ -288,7 +206,7 @@ jobs: path: channels/${{ matrix.country }}.m3u commit-changes: runs-on: ubuntu-latest - needs: detect-resolution + needs: format steps: - name: Checkout uses: actions/checkout@v2 @@ -302,7 +220,70 @@ jobs: - name: Commit Changes uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: '[Bot] Detect resolution' + commit_message: '[Bot] Format playlists' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + remove-duplicates: + runs-on: ubuntu-latest + needs: commit-changes + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Remove Duplicates + run: node scripts/remove-duplicates.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Remove duplicates' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + sort: + runs-on: ubuntu-latest + needs: remove-duplicates + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Sort Channels + run: node scripts/sort.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Sort channels' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + filter: + runs-on: ubuntu-latest + needs: sort + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Filter Playlists + run: node scripts/filter.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Filter channels' commit_user_name: iptv-bot commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' @@ -310,7 +291,7 @@ jobs: file_pattern: channels/* generate: runs-on: ubuntu-latest - needs: commit-changes + needs: filter steps: - name: Checkout uses: actions/checkout@v2 @@ -376,6 +357,7 @@ jobs: branch: bot/auto-update file_pattern: README.md pull-request: + if: ${{ github.ref == 'refs/heads/master' }} needs: update-readme runs-on: ubuntu-latest steps: @@ -408,7 +390,6 @@ jobs: pull-request-number: ${{ steps.pr.outputs.pr_number }} merge-method: squash - name: Approve Pull Request - if: github.ref == 'refs/heads/master' uses: juliangruber/approve-pull-request-action@v1 with: github-token: ${{ secrets.PAT }} diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml deleted file mode 100644 index 3c1f9a4520..0000000000 --- a/.github/workflows/clean.yml +++ /dev/null @@ -1,257 +0,0 @@ -name: clean -on: - workflow_dispatch: - schedule: - - cron: '0 6 * * 0' -jobs: - create-branch: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: ${{ github.ref }} - - name: Create Branch - uses: peterjgrainger/action-create-branch@v2.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - branch: 'bot/remove-broken-links' - check: - runs-on: ubuntu-latest - needs: create-branch - continue-on-error: true - strategy: - fail-fast: false - matrix: - country: - [ - ad, - ae, - af, - ag, - al, - am, - an, - ao, - ar, - at, - au, - aw, - az, - ba, - bb, - bd, - be, - bf, - bg, - bh, - bn, - bo, - br, - bs, - by, - ca, - cd, - cg, - ch, - ci, - cl, - cm, - cn, - co, - cr, - cu, - cw, - cy, - cz, - de, - dk, - do, - dz, - ec, - ee, - eg, - es, - et, - fi, - fj, - fo, - fr, - pf, - ge, - gh, - gm, - gn, - gp, - gq, - gr, - gt, - hk, - hn, - hr, - ht, - hu, - id, - ie, - il, - in, - iq, - ir, - is, - it, - jm, - jo, - jp, - ke, - kg, - kh, - kp, - kr, - kw, - kz, - la, - lb, - li, - lk, - lt, - lu, - lv, - ly, - ma, - mc, - md, - me, - mk, - ml, - mm, - mn, - mo, - mt, - mv, - mx, - my, - mz, - ne, - ng, - ni, - nl, - no, - np, - nz, - om, - pa, - pe, - ph, - pk, - pl, - pr, - ps, - pt, - py, - qa, - ro, - rs, - ru, - rw, - sa, - sd, - se, - sg, - si, - sk, - sl, - sm, - sn, - so, - sv, - sy, - th, - tj, - tm, - tn, - tr, - tt, - tw, - tz, - ua, - ug, - uk, - us, - uy, - uz, - va, - ve, - vi, - vn, - xk, - ye, - zm, - unsorted - ] - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/remove-broken-links - - name: Setup FFmpeg - uses: FedericoCarboni/setup-ffmpeg@v1 - - name: Install Dependencies - run: npm install - - name: Remove Broken Links - run: node scripts/clean.js --country=${{ matrix.country }} --debug - - name: Upload Artifact - uses: actions/upload-artifact@v2 - with: - name: channels - path: channels/${{ matrix.country }}.m3u - commit-changes: - runs-on: ubuntu-latest - needs: check - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/remove-broken-links - - name: Download Artifacts - uses: actions/download-artifact@v2 - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Remove broken links' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/remove-broken-links - file_pattern: channels/* - pull-request: - if: ${{ github.ref == 'refs/heads/master' }} - runs-on: ubuntu-latest - needs: commit-changes - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/remove-broken-links - - name: Generate Token - uses: tibdex/github-app-token@v1 - id: generate-token - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - - name: Create Pull Request - uses: repo-sync/pull-request@v2 - with: - source_branch: 'bot/remove-broken-links' - destination_branch: 'master' - pr_title: '[Bot] Remove broken links' - pr_body: | - This pull request is created by [clean][1] workflow. - - The script checks all links except those with labels `[Geo-blocked]`, `[Offline]` or `[Not 24/7]` in the title. - - **IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. If the link works for you but occasionally return an HTTP code 403 (Forbidden) then it should be marked as `[Geo-blocked]`. If the link does not work but has no alternative, you can mark it as `[Offline]` to save it in the playlist along with a description. Working links should be marked as `[Not 24/7]` so that the script will skip them next time. - - [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} - pr_draft: true - github_token: ${{ steps.generate-token.outputs.token }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a111b67135..ef08f6c077 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -125,7 +125,7 @@ STREAM_URL | `LANGUAGE` | Channel language. The name of the language must conform to the standard [ISO 639-3](https://iso639-3.sil.org/code_tables/639/data?title=&field_iso639_cd_st_mmbrshp_639_1_tid=94671&name_3=&field_iso639_element_scope_tid=All&field_iso639_language_type_tid=51&items_per_page=500). If the channel is broadcast in several languages you can list them separated by a semicolon. (optional) | | `LOGO_URL` | The logo of the channel that will be displayed if the player supports it. Supports files in png, jpeg and gif format. (optional) | | `CATEGORY` | The category to which the channel belongs. The list of currently supported categories can be found [here](https://github.com/iptv-org/iptv#playlists-by-category). (optional) | -| `FULL_NAME` | Full name of the channel. It is recommended to use the name listed on [lyngsat](https://www.lyngsat.com/search.html) or [wikipedia](https://www.wikipedia.org/) if possible. May contain any characters except plus sign, minus sign, round and square brackets. | +| `FULL_NAME` | Full name of the channel. It is recommended to use the name listed on [lyngsat](https://www.lyngsat.com/search.html) or [wikipedia](https://www.wikipedia.org/) if possible. May contain any characters except round and square brackets. | | `STREAM_TIME_SHIFT` | Must be specified if the channel is broadcast with a shift in time relative to the main stream. Should only contain a number and a sign. (optional) | | `ALTERNATIVE_NAME` | Can be used to specify a short name or name in another language. May contain any characters except round and square brackets. (optional) | | `STREAM_RESOLUTION` | The maximum height of the frame with a "p" at the end. In case of VLC Player this information can be found in `Window > Media Information... > Codec Details`. (optional) | @@ -153,7 +153,8 @@ http://example.com/stream.m3u8 - `.github/` - `ISSUE_TEMPLATE/`: issue templates for this repository. - `workflows/` - - `auto-update.yml`: contain actions that automatically updates all playlists every day. + - `auto-update.yml`: GitHub Action that automatically updates all playlists every day. + - `check.yml`: GitHub Action that automatically checks every pull request for syntax errors. - `CODE_OF_CONDUCT.md`: rules you shouldn't break if you don't want to get banned. - `.readme/` - `_categories.md`: automatically generated list of all categories and their corresponding playlists. @@ -168,8 +169,6 @@ http://example.com/stream.m3u8 - `unsorted.m3u`: playlist with channels not yet sorted. - `scripts/` - `helpers/`: helper scripts used in GitHub Actions. - - `clean.js`: used in GitHub Action to check all links and remove broken ones. - - `detect-resolution.js`: used in GitHub Action to detect resolution of the streams. - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists. - `format.js`: used within GitHub Action to format channel descriptions. - `generate.js`: used within GitHub Action to generate all additional playlists. diff --git a/package-lock.json b/package-lock.json index 06cebc1d6c..ffc2acf870 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,14 +7,12 @@ "license": "MIT", "dependencies": { "@freearhey/iso-639-3": "^1.0.0", - "axios": "^0.21.1", - "chalk": "^4.1.1", "commander": "^7.0.0", - "escape-string-regexp": "^2.0.0", "iptv-checker": "^0.20.2", "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.1.3", "markdown-include": "^0.4.3", + "normalize-url": "^6.1.0", "pre-push": "^0.1.1", "progress": "^2.0.3", "transliteration": "^2.2.0" @@ -2963,6 +2961,17 @@ "node": ">=0.10.0" } }, + "node_modules/normalize-url": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", + "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -6045,6 +6054,11 @@ "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" }, + "normalize-url": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", + "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==" + }, "npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", diff --git a/package.json b/package.json index ea37000635..64d55e8f7b 100644 --- a/package.json +++ b/package.json @@ -11,14 +11,12 @@ "license": "MIT", "dependencies": { "@freearhey/iso-639-3": "^1.0.0", - "axios": "^0.21.1", - "chalk": "^4.1.1", "commander": "^7.0.0", - "escape-string-regexp": "^2.0.0", "iptv-checker": "^0.20.2", "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.1.3", "markdown-include": "^0.4.3", + "normalize-url": "^6.1.0", "pre-push": "^0.1.1", "progress": "^2.0.3", "transliteration": "^2.2.0" diff --git a/scripts/clean.js b/scripts/clean.js deleted file mode 100644 index d1ab7c2e7e..0000000000 --- a/scripts/clean.js +++ /dev/null @@ -1,74 +0,0 @@ -const IPTVChecker = require('iptv-checker') -const { program } = require('commander') -const ProgressBar = require('progress') -const parser = require('./helpers/parser') -const utils = require('./helpers/utils') -const log = require('./helpers/log') - -program - .usage('[OPTIONS]...') - .option('-d, --debug', 'Enable debug mode') - .option('-c, --country ', 'Comma-separated list of country codes', '') - .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') - .option('--timeout ', 'Set timeout for each request', 5000) - .parse(process.argv) - -let bar -const config = program.opts() -const ignoreStatus = ['Geo-blocked', 'Not 24/7', 'Offline'] -const checker = new IPTVChecker({ - timeout: config.timeout -}) - -async function main() { - log.start() - - if (config.debug) log.print(`Debug mode enabled\n`) - - let playlists = parser.parseIndex() - playlists = utils.filterPlaylists(playlists, config.country, config.exclude) - for (const playlist of playlists) { - await parser - .parsePlaylist(playlist.url) - .then(checkPlaylist) - .then(p => p.save()) - } - - log.finish() -} - -async function checkPlaylist(playlist) { - if (!config.debug) { - bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, { - total: playlist.channels.length - }) - } - const channels = [] - const total = playlist.channels.length - for (const [index, channel] of playlist.channels.entries()) { - const skipChannel = - channel.status && - ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) - if (skipChannel) { - channels.push(channel) - } else { - const result = await checker.checkStream(channel.data) - if (result.status.ok || result.status.reason.includes('timed out')) { - channels.push(channel) - } else { - if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`) - } - } - if (!config.debug) bar.tick() - } - - if (playlist.channels.length !== channels.length) { - log.print(`File '${playlist.url}' has been updated\n`) - playlist.channels = channels - playlist.updated = true - } - - return playlist -} - -main() diff --git a/scripts/detect-resolution.js b/scripts/detect-resolution.js deleted file mode 100644 index 5da6a282f5..0000000000 --- a/scripts/detect-resolution.js +++ /dev/null @@ -1,114 +0,0 @@ -const { program } = require('commander') -const ProgressBar = require('progress') -const axios = require('axios') -const https = require('https') -const parser = require('./helpers/parser') -const utils = require('./helpers/utils') -const log = require('./helpers/log') - -program - .usage('[OPTIONS]...') - .option('-c, --country ', 'Comma-separated list of country codes', '') - .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') - .option('--delay ', 'Delay between parser requests', 1000) - .option('--timeout ', 'Set timeout for each request', 5000) - .parse(process.argv) - -const config = program.opts() -const ignoreStatus = ['Offline'] -const instance = axios.create({ - timeout: config.timeout, - maxContentLength: 200000, - httpsAgent: new https.Agent({ - rejectUnauthorized: false - }) -}) - -async function main() { - log.start() - - log.print(`Parsing 'index.m3u'...\n`) - let playlists = parser.parseIndex() - playlists = utils - .filterPlaylists(playlists, config.country, config.exclude) - .filter(i => i.url !== 'channels/unsorted.m3u') - - for (const playlist of playlists) { - await parser - .parsePlaylist(playlist.url) - .then(detectResolution) - .then(p => p.save()) - } - - log.finish() -} - -async function detectResolution(playlist) { - const channels = [] - const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { - total: playlist.channels.length - }) - let updated = false - for (const channel of playlist.channels) { - bar.tick() - const skipChannel = - channel.status && - ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) - if (!channel.resolution.height && !skipChannel) { - const CancelToken = axios.CancelToken - const source = CancelToken.source() - const timeout = setTimeout(() => { - source.cancel() - }, config.timeout) - - const response = await instance - .get(channel.url, { cancelToken: source.token }) - .then(res => { - clearTimeout(timeout) - - return res - }) - .then(utils.sleep(config.delay)) - .catch(err => { - clearTimeout(timeout) - }) - - if (response && response.status === 200) { - if (/^#EXTM3U/.test(response.data)) { - const resolution = parseResolution(response.data) - if (resolution) { - channel.resolution = resolution - updated = true - } - } - } - } - - channels.push(channel) - } - - if (updated) { - log.print(`File '${playlist.url}' has been updated\n`) - playlist.channels = channels - playlist.updated = true - } - - return playlist -} - -function parseResolution(string) { - const regex = /RESOLUTION=(\d+)x(\d+)/gm - const match = string.matchAll(regex) - const arr = Array.from(match).map(m => ({ - width: parseInt(m[1]), - height: parseInt(m[2]) - })) - - return arr.length - ? arr.reduce(function (prev, current) { - return prev.height > current.height ? prev : current - }) - : undefined -} - -main() diff --git a/scripts/format.js b/scripts/format.js index fb77cab20f..acc2fca3c8 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -1,21 +1,46 @@ +const IPTVChecker = require('iptv-checker') +const normalize = require('normalize-url') +const { program } = require('commander') +const ProgressBar = require('progress') const parser = require('./helpers/parser') const utils = require('./helpers/utils') const file = require('./helpers/file') const log = require('./helpers/log') +program + .usage('[OPTIONS]...') + .option('-d, --debug', 'Enable debug mode') + .option('-s, --status', 'Update stream status') + .option('-r, --resolution', 'Detect stream resolution') + .option('-c, --country ', 'Comma-separated list of country codes', '') + .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') + .option('--timeout ', 'Set timeout for each request', 5000) + .parse(process.argv) + +let bar +const ignoreStatus = ['Geo-blocked', 'Not 24/7'] +const config = program.opts() +const checker = new IPTVChecker({ + timeout: config.timeout +}) + async function main() { log.start() - log.print(`Parsing 'index.m3u'...`) + if (config.debug) log.print(`Debug mode enabled\n`) + if (config.status) log.print(`Status check enabled\n`) + if (config.resolution) log.print(`Resolution detection enabled\n`) + let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') + playlists = utils.filterPlaylists(playlists, config.country, config.exclude) + if (!playlists.length) log.print(`No playlist is selected\n`) for (const playlist of playlists) { - log.print(`\nProcessing '${playlist.url}'...`) await parser .parsePlaylist(playlist.url) - .then(formatPlaylist) + .then(updatePlaylist) .then(playlist => { if (file.read(playlist.url) !== playlist.toString()) { - log.print('updated') + log.print(`File '${playlist.url}' has been updated\n`) playlist.updated = true } @@ -23,33 +48,115 @@ async function main() { }) } - log.print('\n') log.finish() } -async function formatPlaylist(playlist) { +async function updatePlaylist(playlist) { + if (!config.debug) { + bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { + total: playlist.channels.length + }) + } else { + log.print(`Processing '${playlist.url}'...\n`) + } + for (const channel of playlist.channels) { - const code = file.getBasename(playlist.url) - // add missing tvg-name - if (!channel.tvg.name && code !== 'unsorted' && channel.name) { - channel.tvg.name = channel.name.replace(/\"/gi, '') + addMissingData(channel) + updateGroupTitle(channel) + normalizeUrl(channel) + + const checkOnline = config.status || config.resolution + const skipChannel = + channel.status && + ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) + if (checkOnline && !skipChannel) { + await checker + .checkStream(channel.data) + .then(result => { + const status = parseStatus(result.status) + + if (config.status) { + updateStatus(channel, status) + } + + if (config.resolution && status === 'online') { + updateResolution(channel, result.status.metadata) + } + + if (config.debug && status === 'offline') { + log.print(` ERR: ${channel.url} (${result.status.reason})\n`) + } + }) + .catch(err => { + if (config.debug) log.print(` ERR: ${channel.url} (${err.message})\n`) + }) } - // add missing tvg-id - if (!channel.tvg.id && code !== 'unsorted' && channel.tvg.name) { - const id = utils.name2id(channel.tvg.name) - channel.tvg.id = id ? `${id}.${code}` : '' - } - // add missing country - if (!channel.countries.length) { - const name = utils.code2name(code) - channel.countries = name ? [{ code, name }] : [] - channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') - } - // update group-title - channel.group.title = channel.category + if (!config.debug) bar.tick() } return playlist } +function parseStatus(status) { + if (status.ok) { + return 'online' + } else if (status.reason.includes('timed out')) { + return 'timeout' + } else if (status.reason.includes('403')) { + return 'error_403' + } else if (status.reason.includes('not one of 40{0,1,3,4}')) { + return 'error_40x' // 402, 451 + } else { + return 'offline' + } +} + +function updateStatus(channel, status) { + switch (status) { + case 'online': + channel.status = null + break + case 'offline': + channel.status = 'Offline' + break + } +} + +function addMissingData(channel) { + // tvg-name + if (!channel.tvg.name && channel.name) { + channel.tvg.name = channel.name.replace(/\"/gi, '') + } + // tvg-id + if (!channel.tvg.id && channel.tvg.name) { + const id = utils.name2id(channel.tvg.name) + channel.tvg.id = id ? `${id}.${code}` : '' + } + // country + if (!channel.countries.length) { + const name = utils.code2name(code) + channel.countries = name ? [{ code, name }] : [] + channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') + } +} + +function updateGroupTitle(channel) { + channel.group.title = channel.category +} + +function normalizeUrl(channel) { + const normalized = normalize(channel.url, { stripWWW: false }) + channel.updateUrl(normalized) +} + +function updateResolution(channel, metadata) { + const streams = metadata ? metadata.streams.filter(stream => stream.codec_type === 'video') : [] + if (!channel.resolution.height && streams.length) { + channel.resolution = streams.reduce((acc, curr) => { + if (curr.height > acc.height) return { width: curr.width, height: curr.height } + return acc + }) + } +} + main() diff --git a/scripts/helpers/Channel.js b/scripts/helpers/Channel.js index 043f7886f3..8c4b0c397a 100644 --- a/scripts/helpers/Channel.js +++ b/scripts/helpers/Channel.js @@ -22,6 +22,11 @@ module.exports = class Channel { this.languages = this.parseLanguages(data.tvg.language) } + updateUrl(url) { + this.url = url + this.data.url = url + } + parseName(title) { return title .trim() diff --git a/scripts/helpers/utils.js b/scripts/helpers/utils.js index 7a2bae0633..f0d9b3e50d 100644 --- a/scripts/helpers/utils.js +++ b/scripts/helpers/utils.js @@ -1,4 +1,3 @@ -const escapeStringRegexp = require('escape-string-regexp') const transliteration = require('transliteration') const iso6393 = require('@freearhey/iso-639-3') const categories = require('./categories') @@ -71,16 +70,6 @@ utils.sortBy = function (arr, fields) { }) } -utils.escapeStringRegexp = function (scring) { - return escapeStringRegexp(string) -} - -utils.sleep = function (ms) { - return function (x) { - return new Promise(resolve => setTimeout(() => resolve(x), ms)) - } -} - utils.removeProtocol = function (string) { return string.replace(/(^\w+:|^)\/\//, '') } From 11207cbbfde400f62bd0e99cf337b0c0b6af217a Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 11 Aug 2021 19:35:28 +0300 Subject: [PATCH 08/10] Update format.js --- scripts/format.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/format.js b/scripts/format.js index acc2fca3c8..22c633d385 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -146,7 +146,8 @@ function updateGroupTitle(channel) { function normalizeUrl(channel) { const normalized = normalize(channel.url, { stripWWW: false }) - channel.updateUrl(normalized) + const decoded = decodeURIComponent(normalized).replace(/\s/g, '+') + channel.updateUrl(decoded) } function updateResolution(channel, metadata) { From 2bce27f6cc82f9335f5b32e44e9c47427a35df39 Mon Sep 17 00:00:00 2001 From: MapGuy11 <43685307+MapGuy11@users.noreply.github.com> Date: Wed, 11 Aug 2021 14:13:00 -0400 Subject: [PATCH 09/10] Add El Nueve I don't know how this list works for group-titles so I didn't use one. --- channels/ar.m3u | 2 ++ 1 file changed, 2 insertions(+) diff --git a/channels/ar.m3u b/channels/ar.m3u index 97494f417f..4a3551cf52 100644 --- a/channels/ar.m3u +++ b/channels/ar.m3u @@ -120,3 +120,5 @@ https://wowza.telpin.com.ar:1935/telpintv/ttv.stream/playlist.m3u8 http://live-edge01.telecentro.net.ar/live/smil:trm.smil/playlist.m3u8 #EXTINF:-1 tvg-id="TVManaArgentina.ar" tvg-name="TV Maná Argentina" tvg-country="AR" tvg-language="Spanish" tvg-logo="https://i.imgur.com/GVI8kkp.jpg" group-title="Local",TV Maná Argentina (576p) http://streamspub.manasat.com:1935/tvar/tvmanaar2/playlist.m3u8 +#EXTINF:-1 tvg-id="elnueve.ar" tvg-name="El Nueve" tvg-country="AR" tvg-language="Spanish" tvg-logo="https://www.elnueve.com.ar/wp-content/themes/elnueve-ned-theme/img/logo.png" group-title="",El Nueve +https://cdn88.theus6tv.tk/argentina/broadcast/el-nueve.m3u8 From 76b851edbc8b0f7e39dbf6b41d005643a5f1d351 Mon Sep 17 00:00:00 2001 From: MapGuy11 <43685307+MapGuy11@users.noreply.github.com> Date: Wed, 11 Aug 2021 14:40:36 -0400 Subject: [PATCH 10/10] Add Costa Rica Canal 2, 4, 11, and 6 --- channels/cr.m3u | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/channels/cr.m3u b/channels/cr.m3u index 1b08d8364c..e9cc2d6818 100644 --- a/channels/cr.m3u +++ b/channels/cr.m3u @@ -55,3 +55,11 @@ http://tv.ticosmedia.com:1935/TVSUR/TVSUR/playlist.m3u8 http://k4.usastreams.com/videotour/videotour/playlist.m3u8 #EXTINF:-1 tvg-id="ZonaFilmsTV.cr" tvg-name="Zona Films TV" tvg-country="CR" tvg-language="" tvg-logo="" group-title="Music",Zona Films TV http://cdn2.envivo.win/HLS/?canal=zonafilmstv&c=ZwNjYwRkBF4kBQLhZmt=&p=ZGH4AGD1ZGtjZN== +#EXTINF:-1 tvg-id="Canaltwo.cr" tvg-name="Canal 2" tvg-country="CR" tvg-language="" tvg-logo="https://yt3.ggpht.com/ytc/AKedOLSalPTzbA5vd1NjQo6O28oTYy2b7qSZS0R7-Q55" group-title="",Canal 2 +http://cdn88.theus6tv.tk/costa-rica/broadcast/canal-two.m3u8 +#EXTINF:-1 tvg-id="Canalfour.cr" tvg-name="Canal 4" tvg-country="CR" tvg-language="" tvg-logo="https://upload.wikimedia.org/wikipedia/commons/thumb/5/5e/Repretel_4_logo.png/1200px-Repretel_4_logo.png" group-title="",Canal 4 +http://cdn88.theus6tv.tk/costa-rica/broadcast/canal-four.m3u8 +#EXTINF:-1 tvg-id="Canaleleven.cr" tvg-name="Canal 11" tvg-country="CR" tvg-language="" tvg-logo="https://upload.wikimedia.org/wikipedia/commons/thumb/a/ac/Repretel_11_logo.png/600px-Repretel_11_logo.png" group-title="",Canal 11 +http://cdn88.theus6tv.tk/costa-rica/broadcast/canal-eleven.m3u8 +#EXTINF:-1 tvg-id="Canalsix.cr" tvg-name="Canal 11" tvg-country="CR" tvg-language="" tvg-logo="https://upload.wikimedia.org/wikipedia/commons/e/eb/Repretel_6_logo.png" group-title="",Canal 6 +http://cdn88.theus6tv.tk/costa-rica/broadcast/canal-six.m3u8