Delete database/cleanup.js
This commit is contained in:
Aleksandr Statciuk
parent
8ded3338e3
commit
37b77a7d62
|
|
@ -1,39 +0,0 @@
|
|||
name: cleanup
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
cleanup:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: echo "::set-output name=branch_name::$(date +'bot/cleanup-%s')"
|
||||
id: create-branch-name
|
||||
- run: |
|
||||
git config user.name 'iptv-bot[bot]'
|
||||
git config user.email '84861620+iptv-bot[bot]@users.noreply.github.com'
|
||||
- run: git checkout -b ${{ steps.create-branch-name.outputs.branch_name }}
|
||||
- run: npm install
|
||||
- run: node scripts/commands/create-database.js
|
||||
- run: node scripts/commands/cleanup-database.js
|
||||
- run: node scripts/commands/update-playlists.js
|
||||
- run: |
|
||||
git add channels/*
|
||||
git commit -m "[Bot] Update playlists"
|
||||
- uses: tibdex/github-app-token@v1
|
||||
if: ${{ !env.ACT }}
|
||||
id: create-app-token
|
||||
with:
|
||||
app_id: ${{ secrets.APP_ID }}
|
||||
private_key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
- uses: repo-sync/pull-request@v2
|
||||
if: ${{ github.ref == 'refs/heads/master' }}
|
||||
id: pull-request
|
||||
with:
|
||||
github_token: ${{ steps.create-app-token.outputs.token }}
|
||||
source_branch: ${{ steps.create-branch-name.outputs.branch_name }}
|
||||
destination_branch: 'master'
|
||||
pr_title: '[Bot] Remove broken links'
|
||||
pr_body: |
|
||||
This pull request is created by [cleanup][1] workflow.
|
||||
|
||||
[1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }}
|
||||
|
|
@ -5,7 +5,6 @@
|
|||
"db:matrix": "node scripts/commands/database/matrix.js",
|
||||
"db:update": "node scripts/commands/database/update.js",
|
||||
"db:export": "node scripts/commands/database/export.js",
|
||||
"db:cleanup": "node scripts/commands/database/cleanup.js",
|
||||
"cluster:load": "node scripts/commands/cluster/load.js",
|
||||
"playlist:validate": "node scripts/commands/playlist/validate.js",
|
||||
"playlist:generate": "node scripts/commands/playlist/generate.js",
|
||||
|
|
|
|||
|
|
@ -1,25 +0,0 @@
|
|||
const { db, logger } = require('../../core')
|
||||
const _ = require('lodash')
|
||||
|
||||
async function main() {
|
||||
logger.info(`loading streams...`)
|
||||
await db.streams.load()
|
||||
let streams = await db.streams.find({})
|
||||
|
||||
logger.info(`removing broken links...`)
|
||||
let removed = 0
|
||||
const failed = _.filter(streams, { status: 'error' })
|
||||
for (const stream of failed) {
|
||||
const hasDuplicate = _.find(streams, s => s.channel === stream.channel && s.status !== 'error')
|
||||
if (hasDuplicate) {
|
||||
await db.streams.remove({ _id: stream._id })
|
||||
removed++
|
||||
}
|
||||
}
|
||||
|
||||
db.streams.compact()
|
||||
|
||||
logger.info(`removed ${removed} links`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"2ST8btby3mmsgPF0"}
|
||||
{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"}
|
||||
{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":3,"_id":"WTbieV1ptnZVCIdn"}
|
||||
{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"blocked","cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"}
|
||||
{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","http_referrer":null,"user_agent":null,"status":"timeout","cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"2ST8btby3mmsgPF0"}
|
||||
{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"error","cluster_id":3,"_id":"3TbieV1ptnZVCIdn"}
|
||||
{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"}
|
||||
{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":3,"_id":"WTbieV1ptnZVCIdn"}
|
||||
{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"blocked","cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"}
|
||||
{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","http_referrer":null,"user_agent":null,"status":"timeout","cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
const { execSync } = require('child_process')
|
||||
const fs = require('fs-extra')
|
||||
const path = require('path')
|
||||
|
||||
beforeEach(() => {
|
||||
fs.emptyDirSync('tests/__data__/output')
|
||||
fs.copyFileSync(
|
||||
'tests/__data__/input/database/db_cleanup.streams.db',
|
||||
'tests/__data__/output/streams.db'
|
||||
)
|
||||
|
||||
const stdout = execSync('DB_DIR=tests/__data__/output npm run db:cleanup', {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
})
|
||||
|
||||
it('can remove broken links from database', () => {
|
||||
expect(content('tests/__data__/output/streams.db')).toEqual(
|
||||
content('tests/__data__/expected/database/db_cleanup.streams.db')
|
||||
)
|
||||
})
|
||||
|
||||
function content(filepath) {
|
||||
const data = fs.readFileSync(path.resolve(filepath), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
|
||||
return data
|
||||
.split('\n')
|
||||
.filter(l => l)
|
||||
.map(l => {
|
||||
return JSON.parse(l)
|
||||
})
|
||||
}
|
||||
Loading…
Reference in New Issue