From b81c8ae9f53963c9474df53a07ad8c4ac58773b0 Mon Sep 17 00:00:00 2001
From: Aleksandr Statciuk <free.arhey@gmail.com>
Date: Thu, 12 Aug 2021 18:42:30 +0300
Subject: [PATCH] Update remove-duplicates.js

---
 scripts/remove-duplicates.js | 47 +++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 25 deletions(-)

diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js
index 650fbab472..19be9df4ff 100644
--- a/scripts/remove-duplicates.js
+++ b/scripts/remove-duplicates.js
@@ -13,7 +13,7 @@ async function main() {
     log.print(`\nProcessing '${playlist.url}'...`)
     await parser
       .parsePlaylist(playlist.url)
-      .then(addToBuffer)
+      .then(addToGlobalBuffer)
       .then(removeDuplicates)
       .then(p => p.save())
   }
@@ -22,7 +22,8 @@ async function main() {
     log.print(`\nProcessing 'channels/unsorted.m3u'...`)
     await parser
       .parsePlaylist('channels/unsorted.m3u')
-      .then(removeUnsortedDuplicates)
+      .then(removeDuplicates)
+      .then(removeGlobalDuplicates)
       .then(p => p.save())
   }
 
@@ -30,22 +31,27 @@ async function main() {
   log.finish()
 }
 
-async function addToBuffer(playlist) {
-  globalBuffer = globalBuffer.concat(playlist.channels)
+async function addToGlobalBuffer(playlist) {
+  playlist.channels.forEach(channel => {
+    const url = utils.removeProtocol(channel.url)
+    globalBuffer.push(url)
+  })
 
   return playlist
 }
 
 async function removeDuplicates(playlist) {
-  let buffer = {}
-  const channels = playlist.channels.filter(i => {
-    const url = utils.removeProtocol(i.url)
-    const result = typeof buffer[url] === 'undefined'
-    if (result) {
-      buffer[url] = true
-    }
+  const buffer = []
+  const channels = playlist.channels.filter(channel => {
+    const sameUrl = buffer.find(item => {
+      return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url)
+    })
+    if (sameUrl) return false
+    const sameHash = buffer.find(item => item.hash === channel.hash)
+    if (sameHash && channel.status === 'Offline') return false
 
-    return result
+    buffer.push(channel)
+    return true
   })
 
   if (playlist.channels.length !== channels.length) {
@@ -57,21 +63,12 @@ async function removeDuplicates(playlist) {
   return playlist
 }
 
-async function removeUnsortedDuplicates(playlist) {
-  // locally
-  let buffer = {}
-  let channels = playlist.channels.filter(i => {
-    const url = utils.removeProtocol(i.url)
-    const result = typeof buffer[url] === 'undefined'
-    if (result) buffer[url] = true
-
-    return result
+async function removeGlobalDuplicates(playlist) {
+  const channels = playlist.channels.filter(channel => {
+    const url = utils.removeProtocol(channel.url)
+    return !globalBuffer.includes(url)
   })
 
-  // globally
-  const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
-  channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
-
   if (channels.length !== playlist.channels.length) {
     log.print('updated')
     playlist.channels = channels