Merge branch 'master' into create-clean-script

scripts/format.js

@@ -11,7 +11,7 @@ program
   .option('-d, --debug', 'Debug mode')
   .option('-c, --country <country>', 'Comma-separated list of country codes', '')
   .option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
-  .option('--delay <delay>', 'Delay between parser requests', 0)
+  .option('--delay <delay>', 'Delay between parser requests', 1000)
   .option('--timeout <timeout>', 'Set timeout for each request', 5000)
   .parse(process.argv)
 
@@ -19,21 +19,18 @@ const config = program.opts()
 
 const instance = axios.create({
   timeout: config.timeout,
-  maxContentLength: 20000,
+  maxContentLength: 200000,
   httpsAgent: new https.Agent({
     rejectUnauthorized: false
   })
 })
 
-let globalBuffer = []
-
 async function main() {
   const playlists = parseIndex()
 
   for (const playlist of playlists) {
     await loadPlaylist(playlist.url)
       .then(addToBuffer)
-      .then(removeDuplicates)
       .then(sortChannels)
       .then(filterChannels)
       .then(detectResolution)
@@ -41,15 +38,6 @@ async function main() {
       .then(done)
   }
 
-  if (playlists.length) {
-    await loadPlaylist('channels/unsorted.m3u')
-      .then(removeUnsortedDuplicates)
-      .then(filterChannels)
-      .then(sortChannels)
-      .then(savePlaylist)
-      .then(done)
-  }
-
   finish()
 }
 
@@ -69,13 +57,6 @@ async function loadPlaylist(url) {
   return parser.parsePlaylist(url)
 }
 
-async function addToBuffer(playlist) {
-  if (playlist.url === 'channels/unsorted.m3u') return playlist
-  globalBuffer = globalBuffer.concat(playlist.channels)
-
-  return playlist
-}
-
 async function sortChannels(playlist) {
   console.info(`  Sorting channels...`)
   playlist.channels = utils.sortBy(playlist.channels, ['name', 'url'])
@@ -93,26 +74,9 @@ async function filterChannels(playlist) {
   return playlist
 }
 
-async function removeDuplicates(playlist) {
-  console.info(`  Looking for duplicates...`)
-  let buffer = {}
-  const channels = playlist.channels.filter(i => {
-    const url = utils.removeProtocol(i.url)
-    const result = typeof buffer[url] === 'undefined'
-    if (result) {
-      buffer[url] = true
-    }
-
-    return result
-  })
-
-  playlist.channels = channels
-
-  return playlist
-}
-
 async function detectResolution(playlist) {
-  const bar = new ProgressBar('  Detecting resolution: [:bar] :current/:total (:percent) ', {
+  console.log('  Detecting resolution...')
+  const bar = new ProgressBar('    Progress: [:bar] :current/:total (:percent) ', {
     total: playlist.channels.length
   })
   const results = []
@@ -157,27 +121,6 @@ function parseResolution(string) {
     : undefined
 }
 
-async function removeUnsortedDuplicates(playlist) {
-  console.info(`  Looking for duplicates...`)
-  // locally
-  let buffer = {}
-  let channels = playlist.channels.filter(i => {
-    const url = utils.removeProtocol(i.url)
-    const result = typeof buffer[url] === 'undefined'
-    if (result) buffer[url] = true
-
-    return result
-  })
-  // globally
-  const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
-  channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
-  if (channels.length === playlist.channels.length) return playlist
-
-  playlist.channels = channels
-
-  return playlist
-}
-
 async function savePlaylist(playlist) {
   const original = utils.readFile(playlist.url)
   const output = playlist.toString()

scripts/remove-duplicates.js

@@ -0,0 +1,110 @@
+const parser = require('./parser')
+const utils = require('./utils')
+
+let globalBuffer = []
+
+async function main() {
+  const playlists = parseIndex()
+
+  for (const playlist of playlists) {
+    await loadPlaylist(playlist.url)
+      .then(addToBuffer)
+      .then(removeDuplicates)
+      .then(savePlaylist)
+      .then(done)
+  }
+
+  if (playlists.length) {
+    await loadPlaylist('channels/unsorted.m3u')
+      .then(removeUnsortedDuplicates)
+      .then(savePlaylist)
+      .then(done)
+  }
+
+  finish()
+}
+
+function parseIndex() {
+  console.info(`Parsing 'index.m3u'...`)
+  let playlists = parser.parseIndex()
+  playlists = playlists.filter(i => i.url !== 'channels/unsorted.m3u')
+  console.info(`Found ${playlists.length} playlist(s)\n`)
+
+  return playlists
+}
+
+async function loadPlaylist(url) {
+  console.info(`Processing '${url}'...`)
+  return parser.parsePlaylist(url)
+}
+
+async function addToBuffer(playlist) {
+  if (playlist.url === 'channels/unsorted.m3u') return playlist
+  globalBuffer = globalBuffer.concat(playlist.channels)
+
+  return playlist
+}
+
+async function removeDuplicates(playlist) {
+  console.info(`  Looking for duplicates...`)
+  let buffer = {}
+  const channels = playlist.channels.filter(i => {
+    const url = utils.removeProtocol(i.url)
+    const result = typeof buffer[url] === 'undefined'
+    if (result) {
+      buffer[url] = true
+    }
+
+    return result
+  })
+
+  playlist.channels = channels
+
+  return playlist
+}
+
+async function removeUnsortedDuplicates(playlist) {
+  console.info(`  Looking for duplicates...`)
+  // locally
+  let buffer = {}
+  let channels = playlist.channels.filter(i => {
+    const url = utils.removeProtocol(i.url)
+    const result = typeof buffer[url] === 'undefined'
+    if (result) buffer[url] = true
+
+    return result
+  })
+  // globally
+  const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
+  channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
+  if (channels.length === playlist.channels.length) return playlist
+
+  playlist.channels = channels
+
+  return playlist
+}
+
+async function savePlaylist(playlist) {
+  const original = utils.readFile(playlist.url)
+  const output = playlist.toString({ raw: true })
+
+  if (original === output) {
+    console.info(`No changes have been made.`)
+    return false
+  } else {
+    utils.createFile(playlist.url, output)
+    console.info(`Playlist has been updated.`)
+  }
+
+  return true
+}
+
+async function done() {
+  console.info(` `)
+}
+
+function finish() {
+  console.info('Done.')
+}
+
+main()