Update create-database.js

scripts/commands/create-database.js

@@ -1,5 +1,4 @@
-const { db, file, parser, store, logger } = require('../core')
+const { db, file, parser, store, logger, cid } = require('../core')
-const transliteration = require('transliteration')
 const { program } = require('commander')
 const _ = require('lodash')
 
@@ -17,19 +16,19 @@ const options = program
 const links = []
 
 async function main() {
-  logger.info('Starting...')
+  logger.info('starting...')
-  logger.info(`Number of clusters: ${options.maxClusters}`)
+  logger.info(`number of clusters: ${options.maxClusters}`)
 
   await loadChannels()
   await saveToDatabase()
 
-  logger.info('Done')
+  logger.info('done')
 }
 
 main()
 
 async function loadChannels() {
-  logger.info(`Loading links...`)
+  logger.info(`loading links...`)
 
   const files = await file.list(`${options.inputDir}/**/*.m3u`)
   for (const filepath of files) {
@@ -39,41 +38,33 @@ async function loadChannels() {
       links.push(item)
     }
   }
-  logger.info(`Found ${links.length} links`)
+  logger.info(`found ${links.length} links`)
 }
 
 async function saveToDatabase() {
-  logger.info('Saving to the database...')
+  logger.info('saving to the database...')
 
   await db.reset()
   const chunks = split(_.shuffle(links), options.maxClusters)
   for (const [i, chunk] of chunks.entries()) {
     for (const item of chunk) {
       const stream = store.create()
-      stream.set('name', { title: item.name })
       stream.set('id', { id: item.tvg.id })
+      stream.set('display_name', { display_name: item.name })
       stream.set('filepath', { filepath: item.filepath })
-      stream.set('src_country', { filepath: item.filepath })
-      stream.set('tvg_country', { tvg_country: item.tvg.country })
-      stream.set('countries', { tvg_country: item.tvg.country })
-      stream.set('regions', { countries: stream.get('countries') })
-      stream.set('languages', { tvg_language: item.tvg.language })
-      stream.set('categories', { group_title: item.group.title })
-      stream.set('tvg_url', { tvg_url: item.tvg.url })
-      stream.set('guides', { tvg_url: item.tvg.url })
-      stream.set('logo', { logo: item.tvg.logo })
       stream.set('resolution', { title: item.name })
       stream.set('status', { title: item.name })
       stream.set('url', { url: item.url })
       stream.set('http', { http: item.http })
-      stream.set('is_nsfw', { categories: stream.get('categories') })
       stream.set('is_broken', { status: stream.get('status') })
       stream.set('updated', { updated: false })
       stream.set('cluster_id', { cluster_id: i + 1 })
 
       if (!stream.get('id')) {
-        const id = generateChannelId(stream.get('name'), stream.get('src_country'))
+        const id = cid.generate(item.name, item.filepath)
+
         stream.set('id', { id })
+        stream.set('updated', { updated: true })
       }
 
       await db.insert(stream.data())
@@ -88,17 +79,3 @@ function split(arr, n) {
   }
   return result
 }
-
-function generateChannelId(name, src_country) {
-  if (name && src_country) {
-    const slug = transliteration
-      .transliterate(name)
-      .replace(/\+/gi, 'Plus')
-      .replace(/[^a-z\d]+/gi, '')
-    const code = src_country.code.toLowerCase()
-
-    return `${slug}.${code}`
-  }
-
-  return null
-}

scripts/store/setters/categories.js

@@ -1,8 +0,0 @@
-const categories = require('../../data/categories')
-
-module.exports = function ({ group_title }) {
-  return group_title
-    .split(';')
-    .map(i => categories[i.toLowerCase()])
-    .filter(i => i)
-}

scripts/store/setters/countries.js

@@ -1,25 +0,0 @@
-const dataRegions = require('../../data/regions')
-const dataCountries = require('../../data/countries')
-
-module.exports = function ({ tvg_country, countries = [] }) {
-  if (tvg_country) {
-    return tvg_country
-      .split(';')
-      .reduce((acc, curr) => {
-        const region = dataRegions[curr]
-        if (region) {
-          for (let code of region.country_codes) {
-            if (!acc.includes(code)) acc.push(code)
-          }
-        } else {
-          acc.push(curr)
-        }
-
-        return acc
-      }, [])
-      .map(item => dataCountries[item])
-      .filter(i => i)
-  }
-
-  return countries
-}

scripts/store/setters/guides.js

@@ -1,3 +0,0 @@
-module.exports = function ({ tvg_url, guides = [] }) {
-  return tvg_url ? [tvg_url] : guides
-}

scripts/store/setters/index.js

@@ -1,12 +1,4 @@
-exports.categories = require('./categories')
-exports.countries = require('./countries')
-exports.guides = require('./guides')
 exports.is_broken = require('./is_broken')
-exports.is_nsfw = require('./is_nsfw')
-exports.languages = require('./languages')
-exports.name = require('./name')
-exports.regions = require('./regions')
 exports.resolution = require('./resolution')
-exports.src_country = require('./src_country')
 exports.status = require('./status')
 exports.url = require('./url')

scripts/store/setters/is_nsfw.js

@@ -1,3 +0,0 @@
-module.exports = function ({ categories }) {
-  return Array.isArray(categories) ? categories.filter(c => c.nsfw).length > 0 : false
-}

scripts/store/setters/languages.js

@@ -1,12 +0,0 @@
-const langs = require('../../data/languages')
-
-module.exports = function ({ tvg_language, languages = [] }) {
-  if (tvg_language) {
-    return tvg_language
-      .split(';')
-      .map(name => langs.find(l => l.name === name))
-      .filter(i => i)
-  }
-
-  return languages
-}

scripts/store/setters/name.js

@@ -1,10 +0,0 @@
-module.exports = function ({ title }) {
-  return title
-    .trim()
-    .split(' ')
-    .map(s => s.trim())
-    .filter(s => {
-      return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s)
-    })
-    .join(' ')
-}

scripts/store/setters/regions.js

@@ -1,22 +0,0 @@
-const _ = require('lodash')
-
-let regions = require('../../data/regions')
-
-module.exports = function ({ countries }) {
-  if (!countries.length) return []
-
-  const output = []
-  regions = Object.values(regions)
-  countries.forEach(country => {
-    regions
-      .filter(region => region.country_codes.includes(country.code))
-      .forEach(found => {
-        output.push({
-          name: found.name,
-          code: found.code
-        })
-      })
-  })
-
-  return _.uniqBy(output, 'code')
-}

tests/__data__/expected/channels.db

@@ -0,0 +1,2 @@
+{"display_name":"ATV (720p) [Offline]","id":"ATV.ad","filepath":"tests/__data__/input/channels/ad.m3u","resolution":{"height":720,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http":{"referrer":"","user-agent":""},"is_broken":true,"updated":false,"cluster_id":1,"_id":"verufR2ehwdsfou3"}
+{"display_name":"Fox Sports 2 Asia (Thai) (720p)","id":"FoxSports2AsiaThai.us","filepath":"tests/__data__/input/channels/us_blocked.m3u","resolution":{"height":720,"width":null},"status":{"label":"","code":"online","level":1},"url":"https://example.com/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":true,"cluster_id":1,"_id":"sLG04kZhqlEcYc25"}

tests/__data__/input/channels/us_blocked.m3u

@@ -1,3 +1,3 @@
 #EXTM3U
-#EXTINF:-1 tvg-id="FoxSports2Asia.us" tvg-country="TH" tvg-language="Thai" tvg-logo="" group-title="Sports",Fox Sports 2 Asia (Thai) (720p)
+#EXTINF:-1 tvg-id="" tvg-country="TH" tvg-language="Thai" tvg-logo="" group-title="Sports",Fox Sports 2 Asia (Thai) (720p)
 https://example.com/playlist.m3u8

tests/commands/create-database.test.js

@@ -5,42 +5,43 @@ const { execSync } = require('child_process')
 beforeEach(() => {
   fs.rmdirSync('tests/__data__/output', { recursive: true })
   fs.mkdirSync('tests/__data__/output')
+
+  const stdout = execSync(
+    'DB_FILEPATH=tests/__data__/output/channels.db node scripts/commands/create-database.js --input-dir=tests/__data__/input/channels --max-clusters=1',
+    { encoding: 'utf8' }
+  )
 })
 
 it('can create database', () => {
-  execSync(
+  let output = content('tests/__data__/output/channels.db')
-    'DB_FILEPATH=tests/__data__/output/test.db node scripts/commands/create-database.js --input-dir=tests/__data__/input/channels --max-clusters=1',
+  let expected = content('tests/__data__/expected/channels.db')
-    { encoding: 'utf8' }
-  )
 
-  const database = fs.readFileSync(path.resolve('tests/__data__/output/test.db'), {
+  output = output.map(i => {
+    i._id = null
+    return i
+  })
+  expected = expected.map(i => {
+    i._id = null
+    return i
+  })
+
+  expect(output).toEqual(
+    expect.arrayContaining([
+      expect.objectContaining(expected[0]),
+      expect.objectContaining(expected[1])
+    ])
+  )
+})
+
+function content(filepath) {
+  const data = fs.readFileSync(path.resolve(filepath), {
     encoding: 'utf8'
   })
-  const item = database.split('\n').find(i => i.includes('ATV.ad'))
+
-  expect(JSON.parse(item)).toMatchObject({
+  return data
-    name: 'ATV',
+    .split('\n')
-    id: 'ATV.ad',
+    .filter(l => l)
-    filepath: 'tests/__data__/input/channels/ad_example.m3u',
+    .map(l => {
-    src_country: { name: 'Andorra', code: 'AD', lang: 'cat' },
+      return JSON.parse(l)
-    tvg_country: 'AD',
-    countries: [{ name: 'Andorra', code: 'AD', lang: 'cat' }],
-    regions: [
-      { name: 'Europe, the Middle East and Africa', code: 'EMEA' },
-      { name: 'Europe', code: 'EUR' },
-      { name: 'Worldwide', code: 'INT' }
-    ],
-    languages: [{ name: 'Catalan', code: 'cat' }],
-    categories: [{ name: 'General', slug: 'general', nsfw: false }],
-    tvg_url: '',
-    guides: [],
-    logo: 'https://i.imgur.com/kJCjeQ4.png',
-    resolution: { height: 720, width: null },
-    status: { label: 'Offline', code: 'offline', level: 5 },
-    url: 'https://iptv-all.lanesh4d0w.repl.co/andorra/atv',
-    http: { referrer: '', 'user-agent': '' },
-    is_nsfw: false,
-    is_broken: true,
-    updated: false,
-    cluster_id: 1
-  })
     })
+}