Merge pull request #4055 from iptv-org/update-remove-duplicates-js

Update remove-duplicates.js

scripts/helpers/Channel.js

@@ -20,6 +20,11 @@ module.exports = class Channel {
     this.category = this.parseCategory(data.group.title)
     this.countries = this.parseCountries(data.tvg.country)
     this.languages = this.parseLanguages(data.tvg.language)
+    this.hash = this.generateHash()
+  }
+
+  generateHash() {
+    return `${this.tvg.id}:${this.tvg.name}:${this.tvg.country}:${this.tvg.language}:${this.logo}:${this.group.title}:${this.name}`.toLowerCase()
   }
 
   updateUrl(url) {

scripts/remove-duplicates.js

@@ -13,7 +13,7 @@ async function main() {
     log.print(`\nProcessing '${playlist.url}'...`)
     await parser
       .parsePlaylist(playlist.url)
-      .then(addToBuffer)
+      .then(addToGlobalBuffer)
       .then(removeDuplicates)
       .then(p => p.save())
   }
@@ -22,7 +22,8 @@ async function main() {
     log.print(`\nProcessing 'channels/unsorted.m3u'...`)
     await parser
       .parsePlaylist('channels/unsorted.m3u')
-      .then(removeUnsortedDuplicates)
+      .then(removeDuplicates)
+      .then(removeGlobalDuplicates)
       .then(p => p.save())
   }
 
@@ -30,22 +31,27 @@ async function main() {
   log.finish()
 }
 
-async function addToBuffer(playlist) {
-  globalBuffer = globalBuffer.concat(playlist.channels)
+async function addToGlobalBuffer(playlist) {
+  playlist.channels.forEach(channel => {
+    const url = utils.removeProtocol(channel.url)
+    globalBuffer.push(url)
+  })
 
   return playlist
 }
 
 async function removeDuplicates(playlist) {
-  let buffer = {}
-  const channels = playlist.channels.filter(i => {
-    const url = utils.removeProtocol(i.url)
-    const result = typeof buffer[url] === 'undefined'
-    if (result) {
-      buffer[url] = true
-    }
+  const buffer = []
+  const channels = playlist.channels.filter(channel => {
+    const sameUrl = buffer.find(item => {
+      return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url)
+    })
+    if (sameUrl) return false
+    const sameHash = buffer.find(item => item.hash === channel.hash)
+    if (sameHash && channel.status === 'Offline') return false
 
-    return result
+    buffer.push(channel)
+    return true
   })
 
   if (playlist.channels.length !== channels.length) {
@@ -57,21 +63,12 @@ async function removeDuplicates(playlist) {
   return playlist
 }
 
-async function removeUnsortedDuplicates(playlist) {
-  // locally
-  let buffer = {}
-  let channels = playlist.channels.filter(i => {
-    const url = utils.removeProtocol(i.url)
-    const result = typeof buffer[url] === 'undefined'
-    if (result) buffer[url] = true
-
-    return result
+async function removeGlobalDuplicates(playlist) {
+  const channels = playlist.channels.filter(channel => {
+    const url = utils.removeProtocol(channel.url)
+    return !globalBuffer.includes(url)
   })
 
-  // globally
-  const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
-  channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
-
   if (channels.length !== playlist.channels.length) {
     log.print('updated')
     playlist.channels = channels