From bd7c6dae4960730a7d5905e0b3f54004ac5a886a Mon Sep 17 00:00:00 2001
From: freearhey <free.arhey@gmail.com>
Date: Sat, 27 Mar 2021 15:09:13 +0300
Subject: [PATCH] Update format.js

Remove duplicates inside unsorted.m3u itself
---
 scripts/format.js | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/scripts/format.js b/scripts/format.js
index c9e76a55e1..4d7b615c7e 100644
--- a/scripts/format.js
+++ b/scripts/format.js
@@ -98,7 +98,7 @@ async function removeDuplicates(playlist) {
   console.info(`  Looking for duplicates...`)
   let buffer = {}
   const channels = playlist.channels.filter(i => {
-    const url = i.url.replace(/(^\w+:|^)\/\//, '')
+    const url = utils.removeProtocol(i.url)
     const result = typeof buffer[url] === 'undefined'
     if (result) {
       buffer[url] = true
@@ -161,9 +161,20 @@ function parseResolution(string) {
 
 async function removeUnsortedDuplicates(playlist) {
   console.info(`  Looking for duplicates...`)
-  const urls = globalBuffer.map(i => i.url.replace(/(^\w+:|^)\/\//, ''))
-  const channels = playlist.channels.filter(i => !urls.includes(i.url.replace(/(^\w+:|^)\/\//, '')))
+  // locally
+  let buffer = {}
+  let channels = playlist.channels.filter(i => {
+    const url = utils.removeProtocol(i.url)
+    const result = typeof buffer[url] === 'undefined'
+    if (result) buffer[url] = true
+
+    return result
+  })
+  // globally
+  const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
+  channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
   if (channels.length === playlist.channels.length) return playlist
+
   playlist.channels = channels
 
   return playlist