From 86906f3cc47c7afa51a6e24d1d4130df432341b0 Mon Sep 17 00:00:00 2001
From: freearhey <free.arhey@gmail.com>
Date: Thu, 6 May 2021 16:25:02 +0300
Subject: [PATCH] Create remove-duplicates.js

---
 scripts/remove-duplicates.js | 110 +++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 scripts/remove-duplicates.js

diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js
new file mode 100644
index 0000000000..f7fbd94327
--- /dev/null
+++ b/scripts/remove-duplicates.js
@@ -0,0 +1,110 @@
+const parser = require('./parser')
+const utils = require('./utils')
+
+let globalBuffer = []
+
+async function main() {
+  const playlists = parseIndex()
+
+  for (const playlist of playlists) {
+    await loadPlaylist(playlist.url)
+      .then(addToBuffer)
+      .then(removeDuplicates)
+      .then(savePlaylist)
+      .then(done)
+  }
+
+  if (playlists.length) {
+    await loadPlaylist('channels/unsorted.m3u')
+      .then(removeUnsortedDuplicates)
+      .then(savePlaylist)
+      .then(done)
+  }
+
+  finish()
+}
+
+function parseIndex() {
+  console.info(`Parsing 'index.m3u'...`)
+  let playlists = parser.parseIndex()
+  playlists = playlists.filter(i => i.url !== 'channels/unsorted.m3u')
+  console.info(`Found ${playlists.length} playlist(s)\n`)
+
+  return playlists
+}
+
+async function loadPlaylist(url) {
+  console.info(`Processing '${url}'...`)
+  return parser.parsePlaylist(url)
+}
+
+async function addToBuffer(playlist) {
+  if (playlist.url === 'channels/unsorted.m3u') return playlist
+  globalBuffer = globalBuffer.concat(playlist.channels)
+
+  return playlist
+}
+
+async function removeDuplicates(playlist) {
+  console.info(`  Looking for duplicates...`)
+  let buffer = {}
+  const channels = playlist.channels.filter(i => {
+    const url = utils.removeProtocol(i.url)
+    const result = typeof buffer[url] === 'undefined'
+    if (result) {
+      buffer[url] = true
+    }
+
+    return result
+  })
+
+  playlist.channels = channels
+
+  return playlist
+}
+
+async function removeUnsortedDuplicates(playlist) {
+  console.info(`  Looking for duplicates...`)
+  // locally
+  let buffer = {}
+  let channels = playlist.channels.filter(i => {
+    const url = utils.removeProtocol(i.url)
+    const result = typeof buffer[url] === 'undefined'
+    if (result) buffer[url] = true
+
+    return result
+  })
+  // globally
+  const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
+  channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
+  if (channels.length === playlist.channels.length) return playlist
+
+  playlist.channels = channels
+
+  return playlist
+}
+
+async function savePlaylist(playlist) {
+  const original = utils.readFile(playlist.url)
+  const output = playlist.toString({ raw: true })
+
+  if (original === output) {
+    console.info(`No changes have been made.`)
+    return false
+  } else {
+    utils.createFile(playlist.url, output)
+    console.info(`Playlist has been updated.`)
+  }
+
+  return true
+}
+
+async function done() {
+  console.info(` `)
+}
+
+function finish() {
+  console.info('Done.')
+}
+
+main()