From 11f7e4cf45f1b1b9f9cc4a60901416bd2e6db78d Mon Sep 17 00:00:00 2001
From: freearhey <free.arhey@gmail.com>
Date: Sun, 3 Nov 2019 16:03:35 +0300
Subject: [PATCH] wip

---
 scripts/format.js | 268 ++++++++++++++++++++--------------------------
 scripts/helper.js |  23 +---
 2 files changed, 120 insertions(+), 171 deletions(-)

diff --git a/scripts/format.js b/scripts/format.js
index 9a9862a084..21c09b22d4 100644
--- a/scripts/format.js
+++ b/scripts/format.js
@@ -1,21 +1,37 @@
 const helper = require('./helper')
 
 const config = {
-  debug: false,
+  debug: process.env.npm_config_debug || false,
   country: process.env.npm_config_country,
   exclude: process.env.npm_config_exclude,
   epg: process.env.npm_config_epg || false
 }
 
-// let stats = {
-//   total: 0,
-//   updated: 0,
-//   duplicates: 0,
-//   removed: 0
-// }
-// let buffer = {}
-
 let playlists = {}
+let isChanged = false
+
+async function main() {
+  console.log(`Parsing index...`)
+  parseIndex()
+  console.log(`Sorting channels...`)
+  sortChannels()
+  console.log(`Removing duplicates...`)
+  removeDuplicates()
+
+  if(config.epg) {
+    console.log('Adding the missing data from EPG files...')
+    await addMissingData()
+  }
+
+  if(isChanged) {
+    console.log('Updating files...')
+    updateFiles()
+  } else {
+    console.log('Nothing is changed.')
+  }
+
+  console.log('Done.\n')
+}
 
 function parseIndex() {
   const root = helper.parsePlaylist('index.m3u')
@@ -23,162 +39,110 @@ function parseIndex() {
   
   for(let rootItem of rootItems) {
     const playlist = helper.parsePlaylist(rootItem.url)
-    playlists[rootItem.url] = {}
-    for(let item of playlist.items) {
-      playlists[rootItem.url][item.url] = helper.createChannel(item)
+    const tvgUrl = playlist.header.attrs['x-tvg-url']
+
+    playlists[rootItem.url] = playlist
+    playlists[rootItem.url].items = playlist.items.map(item => {
+      let channel = helper.createChannel(item)
+      channel.epg = tvgUrl
+      
+      return channel
+    })
+  }
+}
+
+function sortChannels() {
+  for(let pid in playlists) {
+    const channels = playlists[pid].items
+    playlists[pid].items = helper.sortBy(channels, ['title', 'url'])
+    if(channels !== playlists[pid].items) { 
+      playlists[pid].changed = true 
+      isChanged = true
     }
   }
 }
 
-function main() {
-  console.log(`Parsing index...`)
-  parseIndex()
-  console.log(playlists)
-  // add missing data from epg (if exists)
-  // sort channels
-  // save all playlists back
-  // display stats
+function removeDuplicates() {
+  let buffer = {}
+  for(let pid in playlists) {
+    const channels = playlists[pid].items
+    playlists[pid].items = channels.filter(i => {
+      let result = typeof buffer[i.url] === 'undefined'
+      
+      if(result) {
+        buffer[i.url] = true
+      } else {
+        if(config.debug) {
+          console.log(`Duplicate of '${i.title}' has been removed from '${pid}'`)
+        }
+      }
+      
+      return result
+    })
 
-  // for(let country of countries) {
+    if(channels.length !== playlists[pid].items.length) { 
+      playlists[pid].changed = true 
+      isChanged = true
+    }
+  }
+}
 
-  //   if (helper.skipPlaylist(country.url)) {
-  //     continue
-  //   }
-    
-  //   if(verbose) {
-  //     console.log(`Clear cache...`)
-  //   }
-  //   helper.clearCache()
+function updateFiles() {
+  for(let pid in playlists) {
+    let playlist = playlists[pid]
+    if(playlist.changed) {
+      helper.createFile(pid, playlist.getHeader())
+      for(let channel of playlist.items) {
+        helper.appendToFile(pid, channel.toShortString())
+      }
 
-  //   console.log(`Parsing '${country.url}'...`)
-  //   const playlist = helper.parsePlaylist(country.url)
+      if(config.debug) {
+        console.log(`File '${pid}' has been updated`)
+      }
+    }
+  }
+}
 
-  //   if(verbose) {
-  //     console.log(`Creating channels list...`)
-  //   }
-  //   let channels = []
-  //   for(let item of playlist.items) {
-  //     let channel = helper.createChannel(item)
+async function addMissingData() {
+  let guides = {}
+  for(let playlist of Object.values(playlists)) {
+    for(let item of playlist.items) {
+      if(!item.epg) continue
+      try {
+        const guide = guides[item.epg] || await helper.parseEPG(item.epg)
+        guides[item.epg] = guide
 
-  //     if(helper.checkCache(channel.url)) {
-  //       stats.duplicates++
-  //     } else if(!helper.validateUrl(channel.url)) {
-  //       stats.unvalid++
-  //     } else {
-  //       channels.push(channel)
-  //       helper.addToCache(channel.url)
-  //     }
-
-  //     if(unsorted[channel.url]) {
-  //       if(verbose) {
-  //         console.log(`Removed '${channel.url}' from 'channels/unsorted.m3u'...`)
-  //       }
-  //       delete unsorted[channel.url]
-  //       stats.removed++
-  //       stats.duplicates++
-  //     }
-  //   }
-
-  //   const epgUrl = playlist.header.attrs['x-tvg-url']
-  //   if(epgUrl && !buffer[epgUrl] && parseEpg) {
-  //     try {
-  //       console.log(`Loading '${epgUrl}'...`)
-  //       const epg = await helper.loadEPG(epgUrl)
-  //       console.log(`Adding '${epgUrl}' to buffer...`)
-  //       buffer[epgUrl] = epg
-  //     } catch(e) {
-  //       console.log(`Could not load '${epgUrl}'`)
-  //     }
-  //   }
-
-  //   if(buffer[epgUrl]) {
-  //     console.log('Add missing tvg-id from EPG by channel title...')
-  //     for(let channel of channels) {
-  //       for(let channelId in buffer[epgUrl].channels) {
-  //         let c = buffer[epgUrl].channels[channelId]
-  //         for(let epgName of c.name) {
-  //           if(epgName.value) {
-  //             let escaped = helper.escapeStringRegexp(epgName.value)
-  //             channelTitle = channel.title.replace(/(fhd|hd|sd|高清)$/i, '').trim()
-  //             let regexp = new RegExp(`^${escaped}$`, 'i')
-  //             if(regexp.test(channelTitle)) {
-  //               if(!channel.id) {
-  //                 channel.id = c.id
-  //                 continue
-  //               }
-  //             }
-  //           }
-  //         }
-  //       }
-  //     }
-  //   }
-
-  //   if(buffer[epgUrl]) {
-  //     console.log(`Fills in missing channel's data...`)
-  //     for(let channel of channels) {
-  //       let channelId = channel.id
-  //       if(!channelId) continue
-  //       let c = buffer[epgUrl].channels[channelId]
-  //       if(!c) continue
-  //       let updated = false
+        if(!item.id) continue
         
-  //       if(!channel.name && c.name.length) {
-  //         channel.name = c.name[0].value
-  //         updated = true
-  //         if(verbose) {
-  //           console.log(`Added name '${c.name[0].value}' to '${channel.id}'`)
-  //         }
-  //       }
+        const channel = guide.channels[item.id]
+        if(!channel) continue
 
-  //       if(!channel.language && c.name.length && c.name[0].lang) {
-  //         let language = helper.getISO6391Name(c.name[0].lang)
-  //         channel.language = language
-  //         updated = true
-  //         if(verbose) {
-  //           console.log(`Added language '${language}' to '${channel.id}'`)
-  //         }
-  //       }
+        if(!item.name && channel.name.length) {
+          item.name = channel.name[0].value
+          if(config.debug) {
+            console.log(`Added tvg-name '${item.name}' to '${item.id}'`)
+          }
+        }
 
-  //       if(!channel.logo && c.icon.length) {
-  //         const icon = c.icon[0].split('|')[0]
-  //         channel.logo = icon
-  //         updated = true
-  //         if(verbose) {
-  //           console.log(`Added logo '${icon}' to '${channel.id}'`)
-  //         }
-  //       }
+        if(!item.language && channel.name.length && channel.name[0].lang) {
+          item.language = channel.name[0].lang
+          if(config.debug) {
+            console.log(`Added tvg-language '${item.language}' to '${item.id}'`)
+          }
+        }
 
-  //       if(updated) {
-  //         stats.updated++
-  //       }
-  //     }
-  //   }
-    
-  //   if(verbose) {
-  //     console.log(`Sorting channels...`)
-  //   }
-  //   channels = helper.sortBy(channels, ['title', 'url'])
-
-  //   if(!debug) {
-  //     console.log(`Updating '${country.url}'...`)
-  //     helper.createFile(country.url, playlist.getHeader())
-  //     channels.forEach(channel => {
-  //       helper.appendToFile(country.url, channel.toShortString())
-  //     })
-  //   }
-
-  //   stats.total += channels.length
-  // }
-
-  // if(!debug & stats.removed > 0) {
-  //   console.log(`Updating 'channels/unsorted.m3u'...`)
-  //   helper.createFile('channels/unsorted.m3u', playlist.getHeader())
-  //   Object.values(unsorted).forEach(channel => {
-  //     helper.appendToFile('channels/unsorted.m3u', channel.toShortString())
-  //   })
-  // }
-
-  // console.log(`Total: ${stats.total}. Duplicates: ${stats.duplicates}. Unvalid: ${stats.unvalid}. Updated: ${stats.updated}.`)
+        if(!item.logo && channel.icon.length) {
+          item.logo = channel.icon[0]
+          if(config.debug) {
+            console.log(`Added tvg-logo '${item.logo}' to '${item.id}'`)
+          }
+        }
+      } catch(err) {
+        console.error(`Could not load '${item.epg}'`)
+        continue
+      }
+    }
+  }
 }
 
 main()
diff --git a/scripts/helper.js b/scripts/helper.js
index 1474ac296d..dc09312ebd 100644
--- a/scripts/helper.js
+++ b/scripts/helper.js
@@ -51,8 +51,8 @@ helper.parsePlaylist = function(filename) {
   return new Playlist(result)
 }
 
-helper.loadEPG = async function(url) {
-  const content = await getEPGFile(url)
+helper.parseEPG = async function(url) {
+  const content = await this.getEPG(url)
   const result = epgParser.parse(content)
   const channels = {}
   for(let channel of result.channels) {
@@ -65,7 +65,7 @@ helper.loadEPG = async function(url) {
   })
 }
 
-helper.getEPGFile = function(url) {
+helper.getEPG = function(url) {
   return new Promise((resolve, reject) => {
     var buffer = []
     axios({
@@ -111,22 +111,6 @@ helper.getBasename = function(filename) {
   return path.basename(filename, path.extname(filename))
 }
 
-helper.addToCache = function(url) {
-  let id = this.getUrlPath(url)
-
-  cache[id] = true
-}
-
-helper.checkCache = function(url) {
-  let id = this.getUrlPath(url)
-
-  return cache.hasOwnProperty(id)
-}
-
-helper.clearCache = function() {
-  cache = {}
-}
-
 helper.getUrlPath = function(u) {
   let parsed = urlParser.parse(u)
   let searchQuery = parsed.search || ''
@@ -220,6 +204,7 @@ class Playlist {
   constructor(data) {
     this.header = data.header
     this.items = data.items
+    this.changed = false
   }
 
   getHeader() {