iptv/scripts/remove-duplicates.js

83 lines
1.9 KiB
JavaScript
Raw Normal View History

2021-08-01 18:56:02 +02:00
const parser = require('./helpers/parser')
const utils = require('./helpers/utils')
const log = require('./helpers/log')
2021-05-06 15:25:02 +02:00
let globalBuffer = []
async function main() {
2021-08-01 07:04:03 +02:00
log.start()
2021-05-06 15:25:02 +02:00
2021-08-01 07:04:03 +02:00
log.print(`Parsing 'index.m3u'...`)
const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
2021-05-06 15:25:02 +02:00
for (const playlist of playlists) {
2021-08-01 07:04:03 +02:00
log.print(`\nProcessing '${playlist.url}'...`)
await parser
.parsePlaylist(playlist.url)
.then(addToBuffer)
.then(removeDuplicates)
2021-08-01 18:56:02 +02:00
.then(p => p.save())
2021-05-06 15:25:02 +02:00
}
if (playlists.length) {
2021-08-01 07:04:03 +02:00
log.print(`\nProcessing 'channels/unsorted.m3u'...`)
await parser
.parsePlaylist('channels/unsorted.m3u')
.then(removeUnsortedDuplicates)
2021-08-01 18:56:02 +02:00
.then(p => p.save())
2021-05-06 15:25:02 +02:00
}
2021-08-01 18:56:02 +02:00
log.print('\n')
2021-08-01 07:04:03 +02:00
log.finish()
2021-05-06 15:25:02 +02:00
}
async function addToBuffer(playlist) {
globalBuffer = globalBuffer.concat(playlist.channels)
return playlist
}
async function removeDuplicates(playlist) {
let buffer = {}
2021-08-01 07:04:03 +02:00
const channels = playlist.channels.filter(i => {
2021-05-06 15:25:02 +02:00
const url = utils.removeProtocol(i.url)
const result = typeof buffer[url] === 'undefined'
if (result) {
buffer[url] = true
}
return result
})
2021-08-01 07:04:03 +02:00
if (playlist.channels.length !== channels.length) {
log.print('updated')
playlist.channels = channels
}
2021-05-06 15:25:02 +02:00
return playlist
}
async function removeUnsortedDuplicates(playlist) {
// locally
let buffer = {}
let channels = playlist.channels.filter(i => {
const url = utils.removeProtocol(i.url)
const result = typeof buffer[url] === 'undefined'
if (result) buffer[url] = true
return result
})
2021-08-01 07:04:03 +02:00
2021-05-06 15:25:02 +02:00
// globally
const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
2021-08-01 07:04:03 +02:00
if (channels.length !== playlist.channels.length) {
log.print('updated')
playlist.channels = channels
2021-05-06 15:25:02 +02:00
}
2021-08-01 07:04:03 +02:00
return playlist
2021-05-06 15:25:02 +02:00
}
main()