From fa4bad3dcad76fe5580e68cbfe9faada4eed3e85 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sat, 5 Feb 2022 06:59:34 +0300 Subject: [PATCH] Update save-results.js --- .github/workflows/auto-update.yml | 2 +- scripts/commands/save-results.js | 166 ++++++++++++ scripts/commands/update-database.js | 243 ------------------ .../expected/save-results.channels.db | 6 + .../cluster_1.log | 0 tests/__data__/input/save-results.channels.db | 6 + tests/__data__/input/update-database.test.db | 6 - tests/commands/save-results.test.js | 36 +++ tests/commands/update-database.test.js | 109 -------- 9 files changed, 215 insertions(+), 359 deletions(-) create mode 100644 scripts/commands/save-results.js delete mode 100644 scripts/commands/update-database.js create mode 100644 tests/__data__/expected/save-results.channels.db rename tests/__data__/input/logs/{check-streams => load-streams}/cluster_1.log (100%) create mode 100644 tests/__data__/input/save-results.channels.db delete mode 100644 tests/__data__/input/update-database.test.db create mode 100644 tests/commands/save-results.test.js delete mode 100644 tests/commands/update-database.test.js diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 641686c845..dc2fe12740 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -71,7 +71,7 @@ jobs: with: node-version: '14' - run: npm install - - run: node scripts/commands/update-database.js + - run: node scripts/commands/save-results.js - uses: actions/upload-artifact@v2 with: name: database diff --git a/scripts/commands/save-results.js b/scripts/commands/save-results.js new file mode 100644 index 0000000000..d3dbb9027c --- /dev/null +++ b/scripts/commands/save-results.js @@ -0,0 +1,166 @@ +const _ = require('lodash') +const statuses = require('../data/statuses') +const { db, store, parser, file, logger } = require('../core') + +let streams = [] +let results = {} +const origins = {} +const items = [] + +const LOGS_PATH = process.env.LOGS_PATH || 'scripts/logs/load-streams' + +async function main() { + await loadDatabase() + await loadResults() + await findStreamOrigins() + await updateStreams() + await updateDatabase() +} + +main() + +async function loadDatabase() { + logger.info('loading database...') + + streams = await db.find({}) + + logger.info(`found ${streams.length} streams`) +} + +async function loadResults() { + logger.info('loading results from logs/...') + + const files = await file.list(`${LOGS_PATH}/cluster_*.log`) + for (const filepath of files) { + const parsed = await parser.parseLogs(filepath) + for (const result of parsed) { + results[result._id] = result + } + } + + logger.info(`found ${Object.values(results).length} results`) +} + +async function findStreamOrigins() { + logger.info('searching for stream origins...') + + for (const { error, requests } of Object.values(results)) { + if (error || !Array.isArray(requests) || !requests.length) continue + + let origin = requests.shift() + origin = new URL(origin.url) + for (const request of requests) { + const curr = new URL(request.url) + const key = curr.href.replace(/(^\w+:|^)/, '') + if (!origins[key] && curr.host === origin.host) { + origins[key] = origin.href + } + } + } + + logger.info(`found ${_.uniq(Object.values(origins)).length} origins`) +} + +async function updateStreams() { + logger.info('updating streams...') + + let updated = 0 + for (const item of streams) { + const stream = store.create(item) + const result = results[item._id] + + if (result) { + const { error, streams, requests } = result + const resolution = parseResolution(streams) + const origin = findOrigin(requests) + let status = parseStatus(error) + + if (status) { + const prevStatus = item.status + if (prevStatus.code === 'not_247') + // not_247 -> * = not_247 + status = item.status + else if (prevStatus.code === 'geo_blocked') + // geo_blocked -> * = geo_blocked + status = item.status + else if (status.code === 'geo_blocked') + // * -> geo_blocked = * + status = item.status + else if (prevStatus.code === 'offline' && status.code === 'online') + // offline -> online = not_247 + status = statuses['not_247'] + + stream.set('status', { status }) + stream.set('is_broken', { status: stream.get('status') }) + } + + if (resolution) { + stream.set('resolution', { resolution }) + } + + if (origin) { + stream.set('url', { url: origin }) + } + } + + if (stream.changed) { + stream.set('updated', true) + items.push(stream.data()) + updated++ + } + } + + logger.info(`updated ${updated} items`) +} + +async function updateDatabase() { + logger.info('updating database...') + + for (const item of items) { + await db.update({ _id: item._id }, item) + } + db.compact() + + logger.info('done') +} + +function findOrigin(requests) { + if (origins && Array.isArray(requests)) { + requests = requests.map(r => r.url.replace(/(^\w+:|^)/, '')) + for (const url of requests) { + if (origins[url]) { + return origins[url] + } + } + } + + return null +} + +function parseResolution(streams) { + const resolution = streams + .filter(s => s.codec_type === 'video') + .reduce( + (acc, curr) => { + if (curr.height > acc.height) return { width: curr.width, height: curr.height } + return acc + }, + { width: 0, height: 0 } + ) + + if (resolution.width > 0 && resolution.height > 0) return resolution + return null +} + +function parseStatus(error) { + if (error) { + if (error.includes('timed out')) { + return statuses['timeout'] + } else if (error.includes('403')) { + return statuses['geo_blocked'] + } + return statuses['offline'] + } + + return statuses['online'] +} diff --git a/scripts/commands/update-database.js b/scripts/commands/update-database.js deleted file mode 100644 index 7afa20bd15..0000000000 --- a/scripts/commands/update-database.js +++ /dev/null @@ -1,243 +0,0 @@ -const _ = require('lodash') -const statuses = require('../data/statuses') -const languages = require('../data/languages') -const { db, store, parser, file, logger } = require('../core') - -let epgCodes = [] -let streams = [] -let checkResults = {} -const origins = {} -const items = [] - -const LOGS_PATH = process.env.LOGS_PATH || 'scripts/logs' -const EPG_CODES_FILEPATH = process.env.EPG_CODES_FILEPATH || 'scripts/data/codes.json' - -async function main() { - await setUp() - await loadDatabase() - await removeDuplicates() - await loadCheckResults() - await findStreamOrigins() - await updateStreams() - await updateDatabase() -} - -main() - -async function loadDatabase() { - logger.info('Loading database...') - - streams = await db.find({}) - - logger.info(`Found ${streams.length} streams`) -} - -async function removeDuplicates() { - logger.info('Removing duplicates...') - - const before = streams.length - streams = _.uniqBy(streams, 'id') - const after = streams.length - - logger.info(`Removed ${before - after} links`) -} - -async function loadCheckResults() { - logger.info('Loading check results from logs/...') - - const files = await file.list(`${LOGS_PATH}/check-streams/cluster_*.log`) - for (const filepath of files) { - const results = await parser.parseLogs(filepath) - for (const result of results) { - checkResults[result._id] = result - } - } - - logger.info(`Found ${Object.values(checkResults).length} results`) -} - -async function findStreamOrigins() { - logger.info('Searching for stream origins...') - - for (const { error, requests } of Object.values(checkResults)) { - if (error || !Array.isArray(requests) || !requests.length) continue - - let origin = requests.shift() - origin = new URL(origin.url) - for (const request of requests) { - const curr = new URL(request.url) - const key = curr.href.replace(/(^\w+:|^)/, '') - if (!origins[key] && curr.host === origin.host) { - origins[key] = origin.href - } - } - } - - logger.info(`Found ${_.uniq(Object.values(origins)).length} origins`) -} - -async function updateStreams() { - logger.info('Updating streams...') - - let updated = 0 - for (const item of streams) { - const stream = store.create(item) - const result = checkResults[item._id] - - if (result) { - const { error, streams, requests } = result - const resolution = parseResolution(streams) - const origin = findOrigin(requests) - let status = parseStatus(error) - - if (status) { - const prevStatus = item.status - if (prevStatus.code === 'not_247') // not_247 -> * = not_247 - status = item.status - else if (prevStatus.code === 'geo_blocked') // geo_blocked -> * = geo_blocked - status = item.status - else if (status.code === 'geo_blocked') // * -> geo_blocked = * - status = item.status - else if (prevStatus.code === 'offline' && status.code === 'online') // offline -> online = not_247 - status = statuses['not_247'] - - - stream.set('status', { status }) - stream.set('is_broken', { status: stream.get('status') }) - } - - if (resolution) { - stream.set('resolution', { resolution }) - } - - if (origin) { - stream.set('url', { url: origin }) - } - } - - if (!stream.has('logo')) { - const logo = findLogo(stream.get('id')) - stream.set('logo', { logo }) - } - - if (!stream.has('guides')) { - const guides = findGuides(stream.get('id')) - stream.set('guides', { guides }) - } - - if (!stream.has('countries') && stream.get('src_country')) { - const countries = [stream.get('src_country')] - stream.set('countries', { countries }) - } - - if (!stream.has('languages')) { - const languages = findLanguages(stream.get('countries'), stream.get('src_country')) - stream.set('languages', { languages }) - } - - if (stream.changed) { - stream.set('updated', true) - items.push(stream.data()) - updated++ - } - } - - logger.info(`Updated ${updated} items`) -} - -async function updateDatabase() { - logger.info('Updating database...') - - for (const item of items) { - await db.update({ _id: item._id }, item) - } - db.compact() - - logger.info('Done') -} - -async function setUp() { - try { - const codes = await file.read(EPG_CODES_FILEPATH) - epgCodes = JSON.parse(codes) - } catch (err) { - logger.error(err.message) - } -} - -function findLanguages(countries, src_country) { - if (countries && Array.isArray(countries)) { - let codes = countries.map(country => country.lang) - codes = _.uniq(codes) - - return codes.map(code => languages.find(l => l.code === code)).filter(l => l) - } - - if (src_country) { - const code = src_country.lang - const lang = languages.find(l => l.code === code) - - return lang ? [lang] : [] - } - - return [] -} - -function findOrigin(requests) { - if (origins && Array.isArray(requests)) { - requests = requests.map(r => r.url.replace(/(^\w+:|^)/, '')) - for (const url of requests) { - if (origins[url]) { - return origins[url] - } - } - } - - return null -} - -function parseResolution(streams) { - const resolution = streams - .filter(s => s.codec_type === 'video') - .reduce( - (acc, curr) => { - if (curr.height > acc.height) return { width: curr.width, height: curr.height } - return acc - }, - { width: 0, height: 0 } - ) - - if (resolution.width > 0 && resolution.height > 0) return resolution - return null -} - -function parseStatus(error) { - if (error) { - if (error.includes('timed out')) { - return statuses['timeout'] - } else if (error.includes('403')) { - return statuses['geo_blocked'] - } - return statuses['offline'] - } - - return statuses['online'] -} - -function findLogo(id) { - const item = epgCodes.find(i => i.tvg_id === id) - if (item && item.logo) { - return item.logo - } - - return null -} - -function findGuides(id) { - const item = epgCodes.find(i => i.tvg_id === id) - if (item && Array.isArray(item.guides)) { - return item.guides - } - - return [] -} diff --git a/tests/__data__/expected/save-results.channels.db b/tests/__data__/expected/save-results.channels.db new file mode 100644 index 0000000000..a7eca5bc13 --- /dev/null +++ b/tests/__data__/expected/save-results.channels.db @@ -0,0 +1,6 @@ +{"name":"ЛДПР ТВ","id":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","resolution":{"width":1920,"height":1080},"status":{"label":"","code":"online","level":1},"url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":true,"cluster_id":1,"_id":"2ST8btby3mmsgPF0"} +{"name":"BBC News HD","id":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","resolution":{"height":720,"width":null},"status":{"label":"Not 24/7","code":"not_247","level":3},"url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":false,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn"} +{"name":"ATV","id":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","resolution":{"height":720,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http":{"referrer":"","user-agent":""},"is_broken":true,"updated":false,"cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"} +{"name":"BBC News HD","id":"AndorraTV.ad","filepath":"tests/__data__/output/channels/uk.m3u","resolution":{"height":720,"width":null},"status":{"label":"Not 24/7","code":"not_247","level":3},"url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":false,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn"} +{"name":"Kayhan TV","id":"KayhanTV.af","filepath":"channels/af.m3u","resolution":{"height":720,"width":null},"status":{"label":"Geo-blocked","code":"geo_blocked","level":2},"url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":false,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"} +{"name":"Sharq","id":"Sharq.af","filepath":"channels/af.m3u","resolution":{"height":576,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"http://51.210.199.50/hls/stream.m3u8","http":{"referrer":"","user-agent":""},"is_broken":true,"updated":false,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"} diff --git a/tests/__data__/input/logs/check-streams/cluster_1.log b/tests/__data__/input/logs/load-streams/cluster_1.log similarity index 100% rename from tests/__data__/input/logs/check-streams/cluster_1.log rename to tests/__data__/input/logs/load-streams/cluster_1.log diff --git a/tests/__data__/input/save-results.channels.db b/tests/__data__/input/save-results.channels.db new file mode 100644 index 0000000000..27154ad3c9 --- /dev/null +++ b/tests/__data__/input/save-results.channels.db @@ -0,0 +1,6 @@ +{"name":"ЛДПР ТВ","id":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","resolution":{"height":1080,"width":null},"status":{"label":"","code":"online","level":1},"url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":false,"cluster_id":1,"_id":"2ST8btby3mmsgPF0"} +{"name":"BBC News HD","id":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","resolution":{"height":720,"width":null},"status":{"label":"Not 24/7","code":"not_247","level":3},"url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":false,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn"} +{"name":"ATV","id":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","resolution":{"height":720,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http":{"referrer":"","user-agent":""},"is_broken":true,"updated":false,"cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"} +{"name":"BBC News HD","id":"AndorraTV.ad","filepath":"tests/__data__/output/channels/uk.m3u","resolution":{"height":720,"width":null},"status":{"label":"Not 24/7","code":"not_247","level":3},"url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":false,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn"} +{"name":"Kayhan TV","id":"KayhanTV.af","filepath":"channels/af.m3u","resolution":{"height":720,"width":null},"status":{"label":"Geo-blocked","code":"geo_blocked","level":2},"url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":false,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"} +{"name":"Sharq","id":"Sharq.af","filepath":"channels/af.m3u","resolution":{"height":576,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"http://51.210.199.50/hls/stream.m3u8","http":{"referrer":"","user-agent":""},"is_broken":true,"updated":false,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"} diff --git a/tests/__data__/input/update-database.test.db b/tests/__data__/input/update-database.test.db deleted file mode 100644 index 33267072e2..0000000000 --- a/tests/__data__/input/update-database.test.db +++ /dev/null @@ -1,6 +0,0 @@ -{"name":"ЛДПР ТВ","id":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","src_country":{"name":"Russia","code":"RU","lang":"rus"},"tvg_country":"RU","countries":[{"name":"Russia","code":"RU","lang":"rus"}],"regions":[{"name":"Asia","code":"ASIA"},{"name":"Commonwealth of Independent States","code":"CIS"},{"name":"Europe, the Middle East and Africa","code":"EMEA"},{"name":"Europe","code":"EUR"}],"languages":[{"name":"Russian","code":"rus"}],"categories":[{"name":"General","slug":"general","nsfw":false}],"tvg_url":"","guides":["https://iptv-org.github.io/epg/guides/ru/tv.yandex.ru.epg.xml"],"logo":"https://iptvx.one/icn/ldpr-tv.png","resolution":{"height":1080,"width":null},"status":{"label":"","code":"online","level":1},"url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_nsfw":false,"is_broken":false,"updated":false,"cluster_id":1,"_id":"2ST8btby3mmsgPF0"} -{"name":"BBC News HD","id":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","src_country":{"name":"United Kingdom","code":"UK","lang":"eng"},"tvg_country":"UK","countries":[{"name":"United Kingdom","code":"UK","lang":"eng"}],"regions":[{"name":"Europe, the Middle East and Africa","code":"EMEA"},{"name":"Europe","code":"EUR"}],"languages":[{"name":"English","code":"eng"}],"categories":[{"name":"News","slug":"news","nsfw":false}],"tvg_url":"","guides":[],"logo":"https://i.imgur.com/eNPIQ9f.png","resolution":{"height":720,"width":null},"status":{"label":"Not 24/7","code":"not_247","level":3},"url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http":{"referrer":"","user-agent":""},"is_nsfw":false,"is_broken":false,"updated":false,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn"} -{"name":"ATV","id":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","src_country":{"name":"Andorra","code":"AD","lang":"cat"},"tvg_country":"AD","countries":[{"name":"Andorra","code":"AD","lang":"cat"}],"regions":[{"name":"Europe, the Middle East and Africa","code":"EMEA"},{"name":"Europe","code":"EUR"}],"languages":[{"name":"Catalan","code":"cat"}],"categories":[{"name":"General","slug":"general","nsfw":false}],"tvg_url":"","guides":[],"logo":"https://i.imgur.com/kJCjeQ4.png","resolution":{"height":720,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http":{"referrer":"","user-agent":""},"is_nsfw":false,"is_broken":true,"updated":false,"cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"} -{"name":"BBC News HD","id":"AndorraTV.ad","filepath":"tests/__data__/output/channels/uk.m3u","src_country":{"name":"United Kingdom","code":"UK","lang":"eng"},"tvg_country":"UK","countries":[{"name":"United Kingdom","code":"UK","lang":"eng"}],"regions":[{"name":"Europe, the Middle East and Africa","code":"EMEA"},{"name":"Europe","code":"EUR"}],"languages":[{"name":"English","code":"eng"}],"categories":[{"name":"News","slug":"news","nsfw":false}],"tvg_url":"","guides":[],"logo":"https://i.imgur.com/eNPIQ9f.png","resolution":{"height":720,"width":null},"status":{"label":"Not 24/7","code":"not_247","level":3},"url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http":{"referrer":"","user-agent":""},"is_nsfw":false,"is_broken":false,"updated":false,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn"} -{"name":"Kayhan TV","id":"KayhanTV.af","filepath":"channels/af.m3u","src_country":{"name":"Afghanistan","code":"AF","lang":"pus"},"tvg_country":"AF","countries":[{"name":"Afghanistan","code":"AF","lang":"pus"}],"regions":[{"name":"Asia-Pacific","code":"APAC"},{"name":"Asia","code":"ASIA"},{"name":"South Asia","code":"SAS"}],"languages":[{"name":"Pashto","code":"pus"}],"categories":[],"tvg_url":"","guides":[],"logo":"https://i.imgur.com/XpR1VvZ.png","resolution":{"height":720,"width":null},"status":{"label":"Geo-blocked","code":"geo_blocked","level":2},"url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_nsfw":false,"is_broken":false,"updated":false,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"} -{"name":"Sharq","id":"Sharq.af","filepath":"channels/af.m3u","src_country":{"name":"Afghanistan","code":"AF","lang":"pus"},"tvg_country":"AF","countries":[{"name":"Afghanistan","code":"AF","lang":"pus"}],"regions":[{"name":"Asia-Pacific","code":"APAC"},{"name":"Asia","code":"ASIA"},{"name":"South Asia","code":"SAS"}],"languages":[{"name":"Pashto","code":"pus"}],"categories":[{"name":"General","slug":"general","nsfw":false}],"tvg_url":"","guides":[],"logo":"https://ws.shoutcast.com/images/contacts/b/b9f8/b9f811c5-d210-4d0b-ae32-f467823a913e/radios/0d677ea5-46b4-4129-9359-9e5783fb6a37/0d677ea5-46b4-4129-9359-9e5783fb6a37.png","resolution":{"height":576,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"http://51.210.199.50/hls/stream.m3u8","http":{"referrer":"","user-agent":""},"is_nsfw":false,"is_broken":false,"updated":false,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"} diff --git a/tests/commands/save-results.test.js b/tests/commands/save-results.test.js new file mode 100644 index 0000000000..68943d6a87 --- /dev/null +++ b/tests/commands/save-results.test.js @@ -0,0 +1,36 @@ +const fs = require('fs-extra') +const path = require('path') +const { execSync } = require('child_process') + +beforeEach(() => { + fs.emptyDirSync('tests/__data__/temp') + fs.copyFileSync( + 'tests/__data__/input/save-results.channels.db', + 'tests/__data__/temp/channels.db' + ) + + const stdout = execSync( + 'DB_FILEPATH=tests/__data__/temp/channels.db LOGS_PATH=tests/__data__/input/logs/load-streams EPG_CODES_FILEPATH=tests/__data__/input/codes.json node scripts/commands/save-results.js', + { encoding: 'utf8' } + ) +}) + +it('can save results', () => { + const output = content('tests/__data__/temp/channels.db') + const expected = content('tests/__data__/expected/save-results.channels.db') + + expect(output).toEqual(expected) +}) + +function content(filepath) { + const data = fs.readFileSync(path.resolve(filepath), { + encoding: 'utf8' + }) + + return data + .split('\n') + .filter(l => l) + .map(l => { + return JSON.parse(l) + }) +} diff --git a/tests/commands/update-database.test.js b/tests/commands/update-database.test.js deleted file mode 100644 index 40e7d340f4..0000000000 --- a/tests/commands/update-database.test.js +++ /dev/null @@ -1,109 +0,0 @@ -const fs = require('fs') -const path = require('path') -const { execSync } = require('child_process') - -beforeEach(() => { - fs.rmdirSync('tests/__data__/temp', { recursive: true }) - fs.mkdirSync('tests/__data__/temp') - fs.copyFileSync('tests/__data__/input/update-database.test.db', 'tests/__data__/temp/test.db') -}) - -it('can update database', () => { - const result = execSync( - 'DB_FILEPATH=tests/__data__/temp/test.db LOGS_PATH=tests/__data__/input/logs EPG_CODES_FILEPATH=tests/__data__/input/codes.json node scripts/commands/update-database.js', - { encoding: 'utf8' } - ) - const database = fs.readFileSync('tests/__data__/temp/test.db', { encoding: 'utf8' }) - const lines = database.split('\n') - expect(JSON.parse(lines[0])).toMatchObject({ - name: 'ЛДПР ТВ', - id: 'LDPRTV.ru', - filepath: 'tests/__data__/output/channels/ru.m3u', - src_country: { name: 'Russia', code: 'RU', lang: 'rus' }, - tvg_country: 'RU', - countries: [{ name: 'Russia', code: 'RU', lang: 'rus' }], - regions: [ - { name: 'Asia', code: 'ASIA' }, - { name: 'Commonwealth of Independent States', code: 'CIS' }, - { name: 'Europe, the Middle East and Africa', code: 'EMEA' }, - { name: 'Europe', code: 'EUR' } - ], - languages: [{ name: 'Russian', code: 'rus' }], - categories: [{ name: 'General', slug: 'general', nsfw: false }], - tvg_url: '', - guides: ['https://iptv-org.github.io/epg/guides/ru/tv.yandex.ru.epg.xml'], - logo: 'https://iptvx.one/icn/ldpr-tv.png', - resolution: { height: 1080, width: 1920 }, - status: { label: '', code: 'online', level: 1 }, - url: 'http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8', - http: { referrer: '', 'user-agent': '' }, - is_nsfw: false, - is_broken: false, - updated: true, - cluster_id: 1, - _id: '2ST8btby3mmsgPF0' - }) - expect(JSON.parse(lines[1])).toMatchObject({ - name: 'BBC News HD', - id: 'BBCNews.uk', - filepath: 'tests/__data__/output/channels/uk.m3u', - src_country: { name: 'United Kingdom', code: 'UK', lang: 'eng' }, - tvg_country: 'UK', - countries: [{ name: 'United Kingdom', code: 'UK', lang: 'eng' }], - regions: [ - { name: 'Europe, the Middle East and Africa', code: 'EMEA' }, - { name: 'Europe', code: 'EUR' } - ], - languages: [{ name: 'English', code: 'eng' }], - categories: [{ name: 'News', slug: 'news', nsfw: false }], - tvg_url: '', - guides: [], - logo: 'https://i.imgur.com/eNPIQ9f.png', - resolution: { height: 720, width: null }, - status: { label: 'Not 24/7', code: 'not_247', level: 3 }, - url: 'http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8', - http: { referrer: '', 'user-agent': '' }, - is_nsfw: false, - is_broken: false, - updated: false, - cluster_id: 3, - _id: '3TbieV1ptnZVCIdn' - }) - expect(JSON.parse(lines[2])).toMatchObject({ - name: 'ATV', - id: 'AndorraTV.ad', - filepath: 'tests/__data__/output/channels/ad.m3u', - src_country: { name: 'Andorra', code: 'AD', lang: 'cat' }, - tvg_country: 'AD', - countries: [{ name: 'Andorra', code: 'AD', lang: 'cat' }], - regions: [ - { name: 'Europe, the Middle East and Africa', code: 'EMEA' }, - { name: 'Europe', code: 'EUR' } - ], - languages: [{ name: 'Catalan', code: 'cat' }], - categories: [{ name: 'General', slug: 'general', nsfw: false }], - tvg_url: '', - guides: ['https://iptv-org.github.io/epg/guides/ad/andorradifusio.ad.epg.xml'], - logo: 'https://i.imgur.com/kJCjeQ4.png', - resolution: { height: 720, width: null }, - status: { label: 'Offline', code: 'offline', level: 5 }, - url: 'https://iptv-all.lanesh4d0w.repl.co/andorra/atv', - http: { referrer: '', 'user-agent': '' }, - is_nsfw: false, - is_broken: true, - updated: true, - cluster_id: 1 - }) - expect(JSON.parse(lines[4])).toMatchObject({ - id: 'KayhanTV.af', - status: { label: 'Geo-blocked', code: 'geo_blocked', level: 2 }, - is_broken: false, - updated: false - }) - expect(JSON.parse(lines[5])).toMatchObject({ - id: 'Sharq.af', - status: { label: 'Offline', code: 'offline', level: 5 }, - is_broken: true, - updated: true - }) -})