From 411171e3f082c17d474c70548e8038c9c2f952de Mon Sep 17 00:00:00 2001
From: Aleksandr Statciuk <github@oiya.xyz>
Date: Sat, 5 Feb 2022 02:48:48 +0300
Subject: [PATCH] Update create-database.js

---
 scripts/commands/create-database.js           | 45 +++----------
 scripts/store/setters/categories.js           |  8 ---
 scripts/store/setters/countries.js            | 25 -------
 scripts/store/setters/guides.js               |  3 -
 scripts/store/setters/index.js                |  8 ---
 scripts/store/setters/is_nsfw.js              |  3 -
 scripts/store/setters/languages.js            | 12 ----
 scripts/store/setters/name.js                 | 10 ---
 scripts/store/setters/regions.js              | 22 ------
 tests/__data__/expected/channels.db           |  2 +
 .../input/channels/{ad_example.m3u => ad.m3u} |  0
 tests/__data__/input/channels/us_blocked.m3u  |  2 +-
 tests/commands/create-database.test.js        | 67 ++++++++++---------
 13 files changed, 48 insertions(+), 159 deletions(-)
 delete mode 100644 scripts/store/setters/categories.js
 delete mode 100644 scripts/store/setters/countries.js
 delete mode 100644 scripts/store/setters/guides.js
 delete mode 100644 scripts/store/setters/is_nsfw.js
 delete mode 100644 scripts/store/setters/languages.js
 delete mode 100644 scripts/store/setters/name.js
 delete mode 100644 scripts/store/setters/regions.js
 create mode 100644 tests/__data__/expected/channels.db
 rename tests/__data__/input/channels/{ad_example.m3u => ad.m3u} (100%)

diff --git a/scripts/commands/create-database.js b/scripts/commands/create-database.js
index ecbbc05973..abb63a8aba 100644
--- a/scripts/commands/create-database.js
+++ b/scripts/commands/create-database.js
@@ -1,5 +1,4 @@
-const { db, file, parser, store, logger } = require('../core')
-const transliteration = require('transliteration')
+const { db, file, parser, store, logger, cid } = require('../core')
 const { program } = require('commander')
 const _ = require('lodash')
 
@@ -17,19 +16,19 @@ const options = program
 const links = []
 
 async function main() {
-  logger.info('Starting...')
-  logger.info(`Number of clusters: ${options.maxClusters}`)
+  logger.info('starting...')
+  logger.info(`number of clusters: ${options.maxClusters}`)
 
   await loadChannels()
   await saveToDatabase()
 
-  logger.info('Done')
+  logger.info('done')
 }
 
 main()
 
 async function loadChannels() {
-  logger.info(`Loading links...`)
+  logger.info(`loading links...`)
 
   const files = await file.list(`${options.inputDir}/**/*.m3u`)
   for (const filepath of files) {
@@ -39,41 +38,33 @@ async function loadChannels() {
       links.push(item)
     }
   }
-  logger.info(`Found ${links.length} links`)
+  logger.info(`found ${links.length} links`)
 }
 
 async function saveToDatabase() {
-  logger.info('Saving to the database...')
+  logger.info('saving to the database...')
 
   await db.reset()
   const chunks = split(_.shuffle(links), options.maxClusters)
   for (const [i, chunk] of chunks.entries()) {
     for (const item of chunk) {
       const stream = store.create()
-      stream.set('name', { title: item.name })
       stream.set('id', { id: item.tvg.id })
+      stream.set('display_name', { display_name: item.name })
       stream.set('filepath', { filepath: item.filepath })
-      stream.set('src_country', { filepath: item.filepath })
-      stream.set('tvg_country', { tvg_country: item.tvg.country })
-      stream.set('countries', { tvg_country: item.tvg.country })
-      stream.set('regions', { countries: stream.get('countries') })
-      stream.set('languages', { tvg_language: item.tvg.language })
-      stream.set('categories', { group_title: item.group.title })
-      stream.set('tvg_url', { tvg_url: item.tvg.url })
-      stream.set('guides', { tvg_url: item.tvg.url })
-      stream.set('logo', { logo: item.tvg.logo })
       stream.set('resolution', { title: item.name })
       stream.set('status', { title: item.name })
       stream.set('url', { url: item.url })
       stream.set('http', { http: item.http })
-      stream.set('is_nsfw', { categories: stream.get('categories') })
       stream.set('is_broken', { status: stream.get('status') })
       stream.set('updated', { updated: false })
       stream.set('cluster_id', { cluster_id: i + 1 })
 
       if (!stream.get('id')) {
-        const id = generateChannelId(stream.get('name'), stream.get('src_country'))
+        const id = cid.generate(item.name, item.filepath)
+
         stream.set('id', { id })
+        stream.set('updated', { updated: true })
       }
 
       await db.insert(stream.data())
@@ -88,17 +79,3 @@ function split(arr, n) {
   }
   return result
 }
-
-function generateChannelId(name, src_country) {
-  if (name && src_country) {
-    const slug = transliteration
-      .transliterate(name)
-      .replace(/\+/gi, 'Plus')
-      .replace(/[^a-z\d]+/gi, '')
-    const code = src_country.code.toLowerCase()
-
-    return `${slug}.${code}`
-  }
-
-  return null
-}
diff --git a/scripts/store/setters/categories.js b/scripts/store/setters/categories.js
deleted file mode 100644
index 8dc50974bb..0000000000
--- a/scripts/store/setters/categories.js
+++ /dev/null
@@ -1,8 +0,0 @@
-const categories = require('../../data/categories')
-
-module.exports = function ({ group_title }) {
-  return group_title
-    .split(';')
-    .map(i => categories[i.toLowerCase()])
-    .filter(i => i)
-}
diff --git a/scripts/store/setters/countries.js b/scripts/store/setters/countries.js
deleted file mode 100644
index b193911b8a..0000000000
--- a/scripts/store/setters/countries.js
+++ /dev/null
@@ -1,25 +0,0 @@
-const dataRegions = require('../../data/regions')
-const dataCountries = require('../../data/countries')
-
-module.exports = function ({ tvg_country, countries = [] }) {
-  if (tvg_country) {
-    return tvg_country
-      .split(';')
-      .reduce((acc, curr) => {
-        const region = dataRegions[curr]
-        if (region) {
-          for (let code of region.country_codes) {
-            if (!acc.includes(code)) acc.push(code)
-          }
-        } else {
-          acc.push(curr)
-        }
-
-        return acc
-      }, [])
-      .map(item => dataCountries[item])
-      .filter(i => i)
-  }
-
-  return countries
-}
diff --git a/scripts/store/setters/guides.js b/scripts/store/setters/guides.js
deleted file mode 100644
index 0c0ff7296c..0000000000
--- a/scripts/store/setters/guides.js
+++ /dev/null
@@ -1,3 +0,0 @@
-module.exports = function ({ tvg_url, guides = [] }) {
-  return tvg_url ? [tvg_url] : guides
-}
diff --git a/scripts/store/setters/index.js b/scripts/store/setters/index.js
index 436bea49a6..4a5aaef82d 100644
--- a/scripts/store/setters/index.js
+++ b/scripts/store/setters/index.js
@@ -1,12 +1,4 @@
-exports.categories = require('./categories')
-exports.countries = require('./countries')
-exports.guides = require('./guides')
 exports.is_broken = require('./is_broken')
-exports.is_nsfw = require('./is_nsfw')
-exports.languages = require('./languages')
-exports.name = require('./name')
-exports.regions = require('./regions')
 exports.resolution = require('./resolution')
-exports.src_country = require('./src_country')
 exports.status = require('./status')
 exports.url = require('./url')
diff --git a/scripts/store/setters/is_nsfw.js b/scripts/store/setters/is_nsfw.js
deleted file mode 100644
index 886b850a69..0000000000
--- a/scripts/store/setters/is_nsfw.js
+++ /dev/null
@@ -1,3 +0,0 @@
-module.exports = function ({ categories }) {
-  return Array.isArray(categories) ? categories.filter(c => c.nsfw).length > 0 : false
-}
diff --git a/scripts/store/setters/languages.js b/scripts/store/setters/languages.js
deleted file mode 100644
index a5ee56dcc3..0000000000
--- a/scripts/store/setters/languages.js
+++ /dev/null
@@ -1,12 +0,0 @@
-const langs = require('../../data/languages')
-
-module.exports = function ({ tvg_language, languages = [] }) {
-  if (tvg_language) {
-    return tvg_language
-      .split(';')
-      .map(name => langs.find(l => l.name === name))
-      .filter(i => i)
-  }
-
-  return languages
-}
diff --git a/scripts/store/setters/name.js b/scripts/store/setters/name.js
deleted file mode 100644
index d566053371..0000000000
--- a/scripts/store/setters/name.js
+++ /dev/null
@@ -1,10 +0,0 @@
-module.exports = function ({ title }) {
-  return title
-    .trim()
-    .split(' ')
-    .map(s => s.trim())
-    .filter(s => {
-      return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s)
-    })
-    .join(' ')
-}
diff --git a/scripts/store/setters/regions.js b/scripts/store/setters/regions.js
deleted file mode 100644
index 7016eecd1b..0000000000
--- a/scripts/store/setters/regions.js
+++ /dev/null
@@ -1,22 +0,0 @@
-const _ = require('lodash')
-
-let regions = require('../../data/regions')
-
-module.exports = function ({ countries }) {
-  if (!countries.length) return []
-
-  const output = []
-  regions = Object.values(regions)
-  countries.forEach(country => {
-    regions
-      .filter(region => region.country_codes.includes(country.code))
-      .forEach(found => {
-        output.push({
-          name: found.name,
-          code: found.code
-        })
-      })
-  })
-
-  return _.uniqBy(output, 'code')
-}
diff --git a/tests/__data__/expected/channels.db b/tests/__data__/expected/channels.db
new file mode 100644
index 0000000000..3aa99e6831
--- /dev/null
+++ b/tests/__data__/expected/channels.db
@@ -0,0 +1,2 @@
+{"display_name":"ATV (720p) [Offline]","id":"ATV.ad","filepath":"tests/__data__/input/channels/ad.m3u","resolution":{"height":720,"width":null},"status":{"label":"Offline","code":"offline","level":5},"url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http":{"referrer":"","user-agent":""},"is_broken":true,"updated":false,"cluster_id":1,"_id":"verufR2ehwdsfou3"}
+{"display_name":"Fox Sports 2 Asia (Thai) (720p)","id":"FoxSports2AsiaThai.us","filepath":"tests/__data__/input/channels/us_blocked.m3u","resolution":{"height":720,"width":null},"status":{"label":"","code":"online","level":1},"url":"https://example.com/playlist.m3u8","http":{"referrer":"","user-agent":""},"is_broken":false,"updated":true,"cluster_id":1,"_id":"sLG04kZhqlEcYc25"}
diff --git a/tests/__data__/input/channels/ad_example.m3u b/tests/__data__/input/channels/ad.m3u
similarity index 100%
rename from tests/__data__/input/channels/ad_example.m3u
rename to tests/__data__/input/channels/ad.m3u
diff --git a/tests/__data__/input/channels/us_blocked.m3u b/tests/__data__/input/channels/us_blocked.m3u
index f08f25184e..0a1d4b7613 100644
--- a/tests/__data__/input/channels/us_blocked.m3u
+++ b/tests/__data__/input/channels/us_blocked.m3u
@@ -1,3 +1,3 @@
 #EXTM3U
-#EXTINF:-1 tvg-id="FoxSports2Asia.us" tvg-country="TH" tvg-language="Thai" tvg-logo="" group-title="Sports",Fox Sports 2 Asia (Thai) (720p)
+#EXTINF:-1 tvg-id="" tvg-country="TH" tvg-language="Thai" tvg-logo="" group-title="Sports",Fox Sports 2 Asia (Thai) (720p)
 https://example.com/playlist.m3u8
diff --git a/tests/commands/create-database.test.js b/tests/commands/create-database.test.js
index f50a9fdd06..a09781b9e9 100644
--- a/tests/commands/create-database.test.js
+++ b/tests/commands/create-database.test.js
@@ -5,42 +5,43 @@ const { execSync } = require('child_process')
 beforeEach(() => {
   fs.rmdirSync('tests/__data__/output', { recursive: true })
   fs.mkdirSync('tests/__data__/output')
+
+  const stdout = execSync(
+    'DB_FILEPATH=tests/__data__/output/channels.db node scripts/commands/create-database.js --input-dir=tests/__data__/input/channels --max-clusters=1',
+    { encoding: 'utf8' }
+  )
 })
 
 it('can create database', () => {
-  execSync(
-    'DB_FILEPATH=tests/__data__/output/test.db node scripts/commands/create-database.js --input-dir=tests/__data__/input/channels --max-clusters=1',
-    { encoding: 'utf8' }
-  )
+  let output = content('tests/__data__/output/channels.db')
+  let expected = content('tests/__data__/expected/channels.db')
 
-  const database = fs.readFileSync(path.resolve('tests/__data__/output/test.db'), {
+  output = output.map(i => {
+    i._id = null
+    return i
+  })
+  expected = expected.map(i => {
+    i._id = null
+    return i
+  })
+
+  expect(output).toEqual(
+    expect.arrayContaining([
+      expect.objectContaining(expected[0]),
+      expect.objectContaining(expected[1])
+    ])
+  )
+})
+
+function content(filepath) {
+  const data = fs.readFileSync(path.resolve(filepath), {
     encoding: 'utf8'
   })
-  const item = database.split('\n').find(i => i.includes('ATV.ad'))
-  expect(JSON.parse(item)).toMatchObject({
-    name: 'ATV',
-    id: 'ATV.ad',
-    filepath: 'tests/__data__/input/channels/ad_example.m3u',
-    src_country: { name: 'Andorra', code: 'AD', lang: 'cat' },
-    tvg_country: 'AD',
-    countries: [{ name: 'Andorra', code: 'AD', lang: 'cat' }],
-    regions: [
-      { name: 'Europe, the Middle East and Africa', code: 'EMEA' },
-      { name: 'Europe', code: 'EUR' },
-      { name: 'Worldwide', code: 'INT' }
-    ],
-    languages: [{ name: 'Catalan', code: 'cat' }],
-    categories: [{ name: 'General', slug: 'general', nsfw: false }],
-    tvg_url: '',
-    guides: [],
-    logo: 'https://i.imgur.com/kJCjeQ4.png',
-    resolution: { height: 720, width: null },
-    status: { label: 'Offline', code: 'offline', level: 5 },
-    url: 'https://iptv-all.lanesh4d0w.repl.co/andorra/atv',
-    http: { referrer: '', 'user-agent': '' },
-    is_nsfw: false,
-    is_broken: true,
-    updated: false,
-    cluster_id: 1
-  })
-})
+
+  return data
+    .split('\n')
+    .filter(l => l)
+    .map(l => {
+      return JSON.parse(l)
+    })
+}