diff --git a/scripts/commands/create-database.js b/scripts/commands/create-database.js index ecbbc05973..abb63a8aba 100644 --- a/scripts/commands/create-database.js +++ b/scripts/commands/create-database.js @@ -1,5 +1,4 @@ -const { db, file, parser, store, logger } = require('../core') -const transliteration = require('transliteration') +const { db, file, parser, store, logger, cid } = require('../core') const { program } = require('commander') const _ = require('lodash') @@ -17,19 +16,19 @@ const options = program const links = [] async function main() { - logger.info('Starting...') - logger.info(`Number of clusters: ${options.maxClusters}`) + logger.info('starting...') + logger.info(`number of clusters: ${options.maxClusters}`) await loadChannels() await saveToDatabase() - logger.info('Done') + logger.info('done') } main() async function loadChannels() { - logger.info(`Loading links...`) + logger.info(`loading links...`) const files = await file.list(`${options.inputDir}/**/*.m3u`) for (const filepath of files) { @@ -39,41 +38,33 @@ async function loadChannels() { links.push(item) } } - logger.info(`Found ${links.length} links`) + logger.info(`found ${links.length} links`) } async function saveToDatabase() { - logger.info('Saving to the database...') + logger.info('saving to the database...') await db.reset() const chunks = split(_.shuffle(links), options.maxClusters) for (const [i, chunk] of chunks.entries()) { for (const item of chunk) { const stream = store.create() - stream.set('name', { title: item.name }) stream.set('id', { id: item.tvg.id }) + stream.set('display_name', { display_name: item.name }) stream.set('filepath', { filepath: item.filepath }) - stream.set('src_country', { filepath: item.filepath }) - stream.set('tvg_country', { tvg_country: item.tvg.country }) - stream.set('countries', { tvg_country: item.tvg.country }) - stream.set('regions', { countries: stream.get('countries') }) - stream.set('languages', { tvg_language: item.tvg.language }) - stream.set('categories', { group_title: item.group.title }) - stream.set('tvg_url', { tvg_url: item.tvg.url }) - stream.set('guides', { tvg_url: item.tvg.url }) - stream.set('logo', { logo: item.tvg.logo }) stream.set('resolution', { title: item.name }) stream.set('status', { title: item.name }) stream.set('url', { url: item.url }) stream.set('http', { http: item.http }) - stream.set('is_nsfw', { categories: stream.get('categories') }) stream.set('is_broken', { status: stream.get('status') }) stream.set('updated', { updated: false }) stream.set('cluster_id', { cluster_id: i + 1 }) if (!stream.get('id')) { - const id = generateChannelId(stream.get('name'), stream.get('src_country')) + const id = cid.generate(item.name, item.filepath) + stream.set('id', { id }) + stream.set('updated', { updated: true }) } await db.insert(stream.data()) @@ -88,17 +79,3 @@ function split(arr, n) { } return result } - -function generateChannelId(name, src_country) { - if (name && src_country) { - const slug = transliteration - .transliterate(name) - .replace(/\+/gi, 'Plus') - .replace(/[^a-z\d]+/gi, '') - const code = src_country.code.toLowerCase() - - return `${slug}.${code}` - } - - return null -} diff --git a/scripts/store/setters/categories.js b/scripts/store/setters/categories.js deleted file mode 100644 index 8dc50974bb..0000000000 --- a/scripts/store/setters/categories.js +++ /dev/null @@ -1,8 +0,0 @@ -const categories = require('../../data/categories') - -module.exports = function ({ group_title }) { - return group_title - .split(';') - .map(i => categories[i.toLowerCase()]) - .filter(i => i) -} diff --git a/scripts/store/setters/countries.js b/scripts/store/setters/countries.js deleted file mode 100644 index b193911b8a..0000000000 --- a/scripts/store/setters/countries.js +++ /dev/null @@ -1,25 +0,0 @@ -const dataRegions = require('../../data/regions') -const dataCountries = require('../../data/countries') - -module.exports = function ({ tvg_country, countries = [] }) { - if (tvg_country) { - return tvg_country - .split(';') - .reduce((acc, curr) => { - const region = dataRegions[curr] - if (region) { - for (let code of region.country_codes) { - if (!acc.includes(code)) acc.push(code) - } - } else { - acc.push(curr) - } - - return acc - }, []) - .map(item => dataCountries[item]) - .filter(i => i) - } - - return countries -} diff --git a/scripts/store/setters/guides.js b/scripts/store/setters/guides.js deleted file mode 100644 index 0c0ff7296c..0000000000 --- a/scripts/store/setters/guides.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = function ({ tvg_url, guides = [] }) { - return tvg_url ? [tvg_url] : guides -} diff --git a/scripts/store/setters/index.js b/scripts/store/setters/index.js index 436bea49a6..4a5aaef82d 100644 --- a/scripts/store/setters/index.js +++ b/scripts/store/setters/index.js @@ -1,12 +1,4 @@ -exports.categories = require('./categories') -exports.countries = require('./countries') -exports.guides = require('./guides') exports.is_broken = require('./is_broken') -exports.is_nsfw = require('./is_nsfw') -exports.languages = require('./languages') -exports.name = require('./name') -exports.regions = require('./regions') exports.resolution = require('./resolution') -exports.src_country = require('./src_country') exports.status = require('./status') exports.url = require('./url') diff --git a/scripts/store/setters/is_nsfw.js b/scripts/store/setters/is_nsfw.js deleted file mode 100644 index 886b850a69..0000000000 --- a/scripts/store/setters/is_nsfw.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = function ({ categories }) { - return Array.isArray(categories) ? categories.filter(c => c.nsfw).length > 0 : false -} diff --git a/scripts/store/setters/languages.js b/scripts/store/setters/languages.js deleted file mode 100644 index a5ee56dcc3..0000000000 --- a/scripts/store/setters/languages.js +++ /dev/null @@ -1,12 +0,0 @@ -const langs = require('../../data/languages') - -module.exports = function ({ tvg_language, languages = [] }) { - if (tvg_language) { - return tvg_language - .split(';') - .map(name => langs.find(l => l.name === name)) - .filter(i => i) - } - - return languages -} diff --git a/scripts/store/setters/name.js b/scripts/store/setters/name.js deleted file mode 100644 index d566053371..0000000000 --- a/scripts/store/setters/name.js +++ /dev/null @@ -1,10 +0,0 @@ -module.exports = function ({ title }) { - return title - .trim() - .split(' ') - .map(s => s.trim()) - .filter(s => { - return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s) - }) - .join(' ') -} diff --git a/scripts/store/setters/regions.js b/scripts/store/setters/regions.js deleted file mode 100644 index 7016eecd1b..0000000000 --- a/scripts/store/setters/regions.js +++ /dev/null @@ -1,22 +0,0 @@ -const _ = require('lodash') - -let regions = require('../../data/regions') - -module.exports = function ({ countries }) { - if (!countries.length) return [] - - const output = [] - regions = Object.values(regions) - countries.forEach(country => { - regions - .filter(region => region.country_codes.includes(country.code)) - .forEach(found => { - output.push({ - name: found.name, - code: found.code - }) - }) - }) - - return _.uniqBy(output, 'code') -} diff --git a/tests/__data__/expected/channels.db b/tests/__data__/expected/channels.db new file mode 100644 index 0000000000..3aa99e6831 --- /dev/null +++ b/tests/__data__/expected/channels.db @@ -0,0 +1,2 @@ +{"display_name":"ATV (720p) [Offline]","id":"ATV.ad","filepath":"tests/__data__/input/channels/ad.m3u","resolution":{"height":720,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http":{"referrer":"","user-agent":""},"is_broken":true,"updated":false,"cluster_id":1,"_id":"verufR2ehwdsfou3"} +{"display_name":"Fox Sports 2 Asia (Thai) (720p)","id":"FoxSports2AsiaThai.us","filepath":"tests/__data__/input/channels/us_blocked.m3u","resolution":{"height":720,"width":null},"status":{"label":"","code":"online","level":1},"url":"https://example.com/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":true,"cluster_id":1,"_id":"sLG04kZhqlEcYc25"} diff --git a/tests/__data__/input/channels/ad_example.m3u b/tests/__data__/input/channels/ad.m3u similarity index 100% rename from tests/__data__/input/channels/ad_example.m3u rename to tests/__data__/input/channels/ad.m3u diff --git a/tests/__data__/input/channels/us_blocked.m3u b/tests/__data__/input/channels/us_blocked.m3u index f08f25184e..0a1d4b7613 100644 --- a/tests/__data__/input/channels/us_blocked.m3u +++ b/tests/__data__/input/channels/us_blocked.m3u @@ -1,3 +1,3 @@ #EXTM3U -#EXTINF:-1 tvg-id="FoxSports2Asia.us" tvg-country="TH" tvg-language="Thai" tvg-logo="" group-title="Sports",Fox Sports 2 Asia (Thai) (720p) +#EXTINF:-1 tvg-id="" tvg-country="TH" tvg-language="Thai" tvg-logo="" group-title="Sports",Fox Sports 2 Asia (Thai) (720p) https://example.com/playlist.m3u8 diff --git a/tests/commands/create-database.test.js b/tests/commands/create-database.test.js index f50a9fdd06..a09781b9e9 100644 --- a/tests/commands/create-database.test.js +++ b/tests/commands/create-database.test.js @@ -5,42 +5,43 @@ const { execSync } = require('child_process') beforeEach(() => { fs.rmdirSync('tests/__data__/output', { recursive: true }) fs.mkdirSync('tests/__data__/output') + + const stdout = execSync( + 'DB_FILEPATH=tests/__data__/output/channels.db node scripts/commands/create-database.js --input-dir=tests/__data__/input/channels --max-clusters=1', + { encoding: 'utf8' } + ) }) it('can create database', () => { - execSync( - 'DB_FILEPATH=tests/__data__/output/test.db node scripts/commands/create-database.js --input-dir=tests/__data__/input/channels --max-clusters=1', - { encoding: 'utf8' } - ) + let output = content('tests/__data__/output/channels.db') + let expected = content('tests/__data__/expected/channels.db') - const database = fs.readFileSync(path.resolve('tests/__data__/output/test.db'), { + output = output.map(i => { + i._id = null + return i + }) + expected = expected.map(i => { + i._id = null + return i + }) + + expect(output).toEqual( + expect.arrayContaining([ + expect.objectContaining(expected[0]), + expect.objectContaining(expected[1]) + ]) + ) +}) + +function content(filepath) { + const data = fs.readFileSync(path.resolve(filepath), { encoding: 'utf8' }) - const item = database.split('\n').find(i => i.includes('ATV.ad')) - expect(JSON.parse(item)).toMatchObject({ - name: 'ATV', - id: 'ATV.ad', - filepath: 'tests/__data__/input/channels/ad_example.m3u', - src_country: { name: 'Andorra', code: 'AD', lang: 'cat' }, - tvg_country: 'AD', - countries: [{ name: 'Andorra', code: 'AD', lang: 'cat' }], - regions: [ - { name: 'Europe, the Middle East and Africa', code: 'EMEA' }, - { name: 'Europe', code: 'EUR' }, - { name: 'Worldwide', code: 'INT' } - ], - languages: [{ name: 'Catalan', code: 'cat' }], - categories: [{ name: 'General', slug: 'general', nsfw: false }], - tvg_url: '', - guides: [], - logo: 'https://i.imgur.com/kJCjeQ4.png', - resolution: { height: 720, width: null }, - status: { label: 'Offline', code: 'offline', level: 5 }, - url: 'https://iptv-all.lanesh4d0w.repl.co/andorra/atv', - http: { referrer: '', 'user-agent': '' }, - is_nsfw: false, - is_broken: true, - updated: false, - cluster_id: 1 - }) -}) + + return data + .split('\n') + .filter(l => l) + .map(l => { + return JSON.parse(l) + }) +}