diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 0000000000..b1efac83e9 --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,78 @@ +name: cleanup +on: + workflow_dispatch: + schedule: + - cron: '0 6 * * 0' +jobs: + create-branch: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Create Branch + uses: peterjgrainger/action-create-branch@v2.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + branch: 'bot/cleanup' + remove-broken-links: + runs-on: ubuntu-latest + needs: create-branch + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/cleanup + - name: Install Dependencies + run: npm install + - name: Remove Broken Links + run: node scripts/remove-broken-links.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Remove broken links' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/cleanup + file_pattern: channels/* + pull-request: + if: ${{ github.ref == 'refs/heads/master' }} + runs-on: ubuntu-latest + needs: remove-broken-links + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/cleanup + - name: Generate Token + uses: tibdex/github-app-token@v1 + id: generate-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Create Pull Request + id: pr + uses: repo-sync/pull-request@v2 + with: + source_branch: 'bot/cleanup' + destination_branch: 'master' + pr_title: '[Bot] Cleaning playlists' + pr_body: | + This pull request is created by [cleanup][1] workflow. + + [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} + github_token: ${{ steps.generate-token.outputs.token }} + - name: Enable Pull Request Automerge + uses: peter-evans/enable-pull-request-automerge@v1 + with: + token: ${{ secrets.PAT }} + pull-request-number: ${{ steps.pr.outputs.pr_number }} + merge-method: squash + - name: Approve Pull Request + uses: juliangruber/approve-pull-request-action@v1 + with: + github-token: ${{ secrets.PAT }} + number: ${{ steps.pr.outputs.pr_number }} diff --git a/scripts/remove-broken-links.js b/scripts/remove-broken-links.js new file mode 100644 index 0000000000..3faf9968b3 --- /dev/null +++ b/scripts/remove-broken-links.js @@ -0,0 +1,40 @@ +const parser = require('./helpers/parser') +const log = require('./helpers/log') + +async function main() { + log.start() + + log.print(`Parsing 'index.m3u'...`) + const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') + for (const playlist of playlists) { + log.print(`\nProcessing '${playlist.url}'...`) + await parser + .parsePlaylist(playlist.url) + .then(removeBrokenLinks) + .then(p => p.save()) + } + + log.print('\n') + log.finish() +} + +async function removeBrokenLinks(playlist) { + const buffer = [] + const channels = playlist.channels.filter(channel => { + const sameHash = buffer.find(item => item.hash === channel.hash) + if (sameHash && channel.status === 'Offline') return false + + buffer.push(channel) + return true + }) + + if (playlist.channels.length !== channels.length) { + log.print('updated') + playlist.channels = channels + playlist.updated = true + } + + return playlist +} + +main() diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js index 19be9df4ff..aa2c3d827b 100644 --- a/scripts/remove-duplicates.js +++ b/scripts/remove-duplicates.js @@ -47,8 +47,6 @@ async function removeDuplicates(playlist) { return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url) }) if (sameUrl) return false - const sameHash = buffer.find(item => item.hash === channel.hash) - if (sameHash && channel.status === 'Offline') return false buffer.push(channel) return true