From 2cb9914cf7ab32c793b38f6822217e28b920cccd Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 7 Feb 2022 07:12:33 +0300 Subject: [PATCH] Update cleanup-database.js --- scripts/commands/cleanup-database.js | 19 +++++---- .../expected/cleanup-database.streams.db | 5 +++ tests/commands/cleanup-database.test.js | 42 ++++++++++++------- 3 files changed, 42 insertions(+), 24 deletions(-) create mode 100644 tests/__data__/expected/cleanup-database.streams.db diff --git a/scripts/commands/cleanup-database.js b/scripts/commands/cleanup-database.js index d005c48078..2bc24f0d5b 100644 --- a/scripts/commands/cleanup-database.js +++ b/scripts/commands/cleanup-database.js @@ -1,24 +1,25 @@ const { db, logger } = require('../core') async function main() { - logger.info(`Loading database...`) - let streams = await db.find({}) + logger.info(`loading streams...`) + await db.streams.load() + let streams = await db.streams.find({}) - logger.info(`Removing broken links...`) + logger.info(`removing broken links...`) let removed = 0 - const buffer = [] + const buffer = {} for (const stream of streams) { - const duplicate = buffer.find(i => i.id === stream.id) + const duplicate = buffer[stream.channel_id] if (duplicate && ['offline', 'timeout'].includes(stream.status.code)) { - await db.remove({ _id: stream._id }) + await db.streams.remove({ _id: stream._id }) removed++ } else { - buffer.push(stream) + buffer[stream.channel_id] = stream } } - db.compact() + db.streams.compact() - logger.info(`Removed ${removed} links`) + logger.info(`removed ${removed} links`) } main() diff --git a/tests/__data__/expected/cleanup-database.streams.db b/tests/__data__/expected/cleanup-database.streams.db new file mode 100644 index 0000000000..ffee4256c6 --- /dev/null +++ b/tests/__data__/expected/cleanup-database.streams.db @@ -0,0 +1,5 @@ +{"channel_name":"ЛДПР ТВ","channel_id":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","resolution":{"height":1080,"width":null},"status":{"label":"","code":"online","level":1},"url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":false,"cluster_id":1,"_id":"2ST8btby3mmsgPF0"} +{"channel_name":"BBC News HD","channel_id":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","resolution":{"height":720,"width":null},"status":{"label":"Not 24/7","code":"not_247","level":3},"url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":false,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn"} +{"channel_name":"ATV","channel_id":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","resolution":{"height":720,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http":{"referrer":"","user-agent":""},"is_broken":true,"updated":false,"cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"} +{"channel_name":"Kayhan TV","channel_id":"KayhanTV.af","filepath":"channels/af.m3u","resolution":{"height":720,"width":null},"status":{"label":"Geo-blocked","code":"geo_blocked","level":2},"url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":false,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"} +{"channel_name":"Sharq","channel_id":"Sharq.af","filepath":"channels/af.m3u","resolution":{"height":576,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"http://51.210.199.50/hls/stream.m3u8","http":{"referrer":"","user-agent":""},"is_broken":true,"updated":false,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"} diff --git a/tests/commands/cleanup-database.test.js b/tests/commands/cleanup-database.test.js index bd541e8e7d..2d59d8521c 100644 --- a/tests/commands/cleanup-database.test.js +++ b/tests/commands/cleanup-database.test.js @@ -1,25 +1,37 @@ -const fs = require('fs') +const fs = require('fs-extra') const path = require('path') const { execSync } = require('child_process') beforeEach(() => { - fs.copyFileSync('tests/__data__/input/test.db', 'tests/__data__/temp/test.db') -}) + fs.emptyDirSync('tests/__data__/output') + fs.copyFileSync( + 'tests/__data__/input/database/cleanup-database.streams.db', + 'tests/__data__/output/streams.db' + ) -afterEach(() => { - fs.rmdirSync('tests/__data__/temp', { recursive: true }) - fs.mkdirSync('tests/__data__/temp') + const stdout = execSync( + 'DB_DIR=tests/__data__/output node scripts/commands/cleanup-database.js', + { + encoding: 'utf8' + } + ) }) it('can remove broken links from database', () => { - const result = execSync( - 'DB_FILEPATH=tests/__data__/temp/test.db node scripts/commands/cleanup-database.js', - { encoding: 'utf8' } - ) - - const database = fs.readFileSync('tests/__data__/temp/test.db', { encoding: 'utf8' }) - const lines = database.split('\n') - expect(lines[0]).toBe( - `{"name":"ЛДПР ТВ","id":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","src_country":{"name":"Russia","code":"RU","lang":"rus"},"tvg_country":"RU","countries":[{"name":"Russia","code":"RU","lang":"rus"}],"regions":[{"name":"Asia","code":"ASIA"},{"name":"Commonwealth of Independent States","code":"CIS"},{"name":"Europe, the Middle East and Africa","code":"EMEA"},{"name":"Europe","code":"EUR"}],"languages":[{"name":"Russian","code":"rus"}],"categories":[{"name":"General","slug":"general","nsfw":false}],"tvg_url":"","guides":["https://iptv-org.github.io/epg/guides/ru/tv.yandex.ru.epg.xml"],"logo":"https://iptvx.one/icn/ldpr-tv.png","resolution":{"height":1080,"width":null},"status":{"label":"","code":"online","level":1},"url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_nsfw":false,"is_broken":false,"updated":false,"cluster_id":1,"_id":"2ST8btby3mmsgPF0"}` + expect(content('tests/__data__/output/streams.db')).toEqual( + content('tests/__data__/expected/cleanup-database.streams.db') ) }) + +function content(filepath) { + const data = fs.readFileSync(path.resolve(filepath), { + encoding: 'utf8' + }) + + return data + .split('\n') + .filter(l => l) + .map(l => { + return JSON.parse(l) + }) +}