From d45b8884c286d4fb79ce85d4a4aff6c268b352c7 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 16 Aug 2021 20:04:13 +0300 Subject: [PATCH 1/6] Update remove-duplicates.js --- scripts/remove-duplicates.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js index 19be9df4ff..aa2c3d827b 100644 --- a/scripts/remove-duplicates.js +++ b/scripts/remove-duplicates.js @@ -47,8 +47,6 @@ async function removeDuplicates(playlist) { return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url) }) if (sameUrl) return false - const sameHash = buffer.find(item => item.hash === channel.hash) - if (sameHash && channel.status === 'Offline') return false buffer.push(channel) return true From 0155e15b3e13e42ed901c8b70716a7ef2423cba0 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 16 Aug 2021 20:16:47 +0300 Subject: [PATCH 2/6] Create clear.js --- scripts/clear.js | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 scripts/clear.js diff --git a/scripts/clear.js b/scripts/clear.js new file mode 100644 index 0000000000..b66b3afd17 --- /dev/null +++ b/scripts/clear.js @@ -0,0 +1,41 @@ +const parser = require('./helpers/parser') +const utils = require('./helpers/utils') +const log = require('./helpers/log') + +async function main() { + log.start() + + log.print(`Parsing 'index.m3u'...`) + const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') + for (const playlist of playlists) { + log.print(`\nProcessing '${playlist.url}'...`) + await parser + .parsePlaylist(playlist.url) + .then(removeBrokenLinks) + .then(p => p.save()) + } + + log.print('\n') + log.finish() +} + +async function removeBrokenLinks(playlist) { + const buffer = [] + const channels = playlist.channels.filter(channel => { + const sameHash = buffer.find(item => item.hash === channel.hash) + if (sameHash && channel.status === 'Offline') return false + + buffer.push(channel) + return true + }) + + if (playlist.channels.length !== channels.length) { + log.print('updated') + playlist.channels = channels + playlist.updated = true + } + + return playlist +} + +main() From 1011925cfe262a1c684ce206ce4f5d50148a89a2 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 16 Aug 2021 20:51:02 +0300 Subject: [PATCH 3/6] Delete clear.js --- scripts/clear.js | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 scripts/clear.js diff --git a/scripts/clear.js b/scripts/clear.js deleted file mode 100644 index b66b3afd17..0000000000 --- a/scripts/clear.js +++ /dev/null @@ -1,41 +0,0 @@ -const parser = require('./helpers/parser') -const utils = require('./helpers/utils') -const log = require('./helpers/log') - -async function main() { - log.start() - - log.print(`Parsing 'index.m3u'...`) - const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') - for (const playlist of playlists) { - log.print(`\nProcessing '${playlist.url}'...`) - await parser - .parsePlaylist(playlist.url) - .then(removeBrokenLinks) - .then(p => p.save()) - } - - log.print('\n') - log.finish() -} - -async function removeBrokenLinks(playlist) { - const buffer = [] - const channels = playlist.channels.filter(channel => { - const sameHash = buffer.find(item => item.hash === channel.hash) - if (sameHash && channel.status === 'Offline') return false - - buffer.push(channel) - return true - }) - - if (playlist.channels.length !== channels.length) { - log.print('updated') - playlist.channels = channels - playlist.updated = true - } - - return playlist -} - -main() From 69024ed1313efcfe0f7039f7acd0dd9e2671a6ee Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 16 Aug 2021 20:51:05 +0300 Subject: [PATCH 4/6] Create remove-broken-links.js --- scripts/remove-broken-links.js | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 scripts/remove-broken-links.js diff --git a/scripts/remove-broken-links.js b/scripts/remove-broken-links.js new file mode 100644 index 0000000000..3faf9968b3 --- /dev/null +++ b/scripts/remove-broken-links.js @@ -0,0 +1,40 @@ +const parser = require('./helpers/parser') +const log = require('./helpers/log') + +async function main() { + log.start() + + log.print(`Parsing 'index.m3u'...`) + const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') + for (const playlist of playlists) { + log.print(`\nProcessing '${playlist.url}'...`) + await parser + .parsePlaylist(playlist.url) + .then(removeBrokenLinks) + .then(p => p.save()) + } + + log.print('\n') + log.finish() +} + +async function removeBrokenLinks(playlist) { + const buffer = [] + const channels = playlist.channels.filter(channel => { + const sameHash = buffer.find(item => item.hash === channel.hash) + if (sameHash && channel.status === 'Offline') return false + + buffer.push(channel) + return true + }) + + if (playlist.channels.length !== channels.length) { + log.print('updated') + playlist.channels = channels + playlist.updated = true + } + + return playlist +} + +main() From 063727bb2c9b235f38d0cca130ee8c2b10ed94f1 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 16 Aug 2021 20:51:10 +0300 Subject: [PATCH 5/6] Create cleanup.yml --- .github/workflows/cleanup.yml | 69 +++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/cleanup.yml diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 0000000000..8c1c7e28b6 --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,69 @@ +name: cleanup +on: + workflow_dispatch: + schedule: + - cron: '0 6 * * 0' +jobs: + create-branch: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Create Branch + uses: peterjgrainger/action-create-branch@v2.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + branch: 'bot/cleanup' + remove-broken-links: + runs-on: ubuntu-latest + needs: create-branch + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/cleanup + - name: Install Dependencies + run: npm install + - name: Remove Broken Links + run: node scripts/remove-broken-links.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Remove broken links' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/cleanup + file_pattern: channels/* + pull-request: + if: ${{ github.ref == 'refs/heads/master' }} + runs-on: ubuntu-latest + needs: remove-broken-links + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/cleanup + - name: Generate Token + uses: tibdex/github-app-token@v1 + id: generate-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Create Pull Request + uses: repo-sync/pull-request@v2 + with: + source_branch: 'bot/cleanup' + destination_branch: 'master' + pr_title: '[Bot] Cleaning playlists' + pr_body: | + This pull request is created by [cleanup][1] workflow. + + **IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. Working links should be marked as `[Not 24/7]` so that the script will skip them next time. + + [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} + pr_draft: true + github_token: ${{ steps.generate-token.outputs.token }} From b6d39acb28a9909a18f0a2644519f1814b63551a Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 16 Aug 2021 21:01:01 +0300 Subject: [PATCH 6/6] Enables PR automerge --- .github/workflows/cleanup.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 8c1c7e28b6..b1efac83e9 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -54,6 +54,7 @@ jobs: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} - name: Create Pull Request + id: pr uses: repo-sync/pull-request@v2 with: source_branch: 'bot/cleanup' @@ -62,8 +63,16 @@ jobs: pr_body: | This pull request is created by [cleanup][1] workflow. - **IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. Working links should be marked as `[Not 24/7]` so that the script will skip them next time. - [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} - pr_draft: true github_token: ${{ steps.generate-token.outputs.token }} + - name: Enable Pull Request Automerge + uses: peter-evans/enable-pull-request-automerge@v1 + with: + token: ${{ secrets.PAT }} + pull-request-number: ${{ steps.pr.outputs.pr_number }} + merge-method: squash + - name: Approve Pull Request + uses: juliangruber/approve-pull-request-action@v1 + with: + github-token: ${{ secrets.PAT }} + number: ${{ steps.pr.outputs.pr_number }}