Merge pull request #4055 from iptv-org/update-remove-duplicates-js
Update remove-duplicates.js
This commit is contained in:
commit
30e1b556fd
|
@ -20,6 +20,11 @@ module.exports = class Channel {
|
||||||
this.category = this.parseCategory(data.group.title)
|
this.category = this.parseCategory(data.group.title)
|
||||||
this.countries = this.parseCountries(data.tvg.country)
|
this.countries = this.parseCountries(data.tvg.country)
|
||||||
this.languages = this.parseLanguages(data.tvg.language)
|
this.languages = this.parseLanguages(data.tvg.language)
|
||||||
|
this.hash = this.generateHash()
|
||||||
|
}
|
||||||
|
|
||||||
|
generateHash() {
|
||||||
|
return `${this.tvg.id}:${this.tvg.name}:${this.tvg.country}:${this.tvg.language}:${this.logo}:${this.group.title}:${this.name}`.toLowerCase()
|
||||||
}
|
}
|
||||||
|
|
||||||
updateUrl(url) {
|
updateUrl(url) {
|
||||||
|
|
|
@ -13,7 +13,7 @@ async function main() {
|
||||||
log.print(`\nProcessing '${playlist.url}'...`)
|
log.print(`\nProcessing '${playlist.url}'...`)
|
||||||
await parser
|
await parser
|
||||||
.parsePlaylist(playlist.url)
|
.parsePlaylist(playlist.url)
|
||||||
.then(addToBuffer)
|
.then(addToGlobalBuffer)
|
||||||
.then(removeDuplicates)
|
.then(removeDuplicates)
|
||||||
.then(p => p.save())
|
.then(p => p.save())
|
||||||
}
|
}
|
||||||
|
@ -22,7 +22,8 @@ async function main() {
|
||||||
log.print(`\nProcessing 'channels/unsorted.m3u'...`)
|
log.print(`\nProcessing 'channels/unsorted.m3u'...`)
|
||||||
await parser
|
await parser
|
||||||
.parsePlaylist('channels/unsorted.m3u')
|
.parsePlaylist('channels/unsorted.m3u')
|
||||||
.then(removeUnsortedDuplicates)
|
.then(removeDuplicates)
|
||||||
|
.then(removeGlobalDuplicates)
|
||||||
.then(p => p.save())
|
.then(p => p.save())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -30,22 +31,27 @@ async function main() {
|
||||||
log.finish()
|
log.finish()
|
||||||
}
|
}
|
||||||
|
|
||||||
async function addToBuffer(playlist) {
|
async function addToGlobalBuffer(playlist) {
|
||||||
globalBuffer = globalBuffer.concat(playlist.channels)
|
playlist.channels.forEach(channel => {
|
||||||
|
const url = utils.removeProtocol(channel.url)
|
||||||
|
globalBuffer.push(url)
|
||||||
|
})
|
||||||
|
|
||||||
return playlist
|
return playlist
|
||||||
}
|
}
|
||||||
|
|
||||||
async function removeDuplicates(playlist) {
|
async function removeDuplicates(playlist) {
|
||||||
let buffer = {}
|
const buffer = []
|
||||||
const channels = playlist.channels.filter(i => {
|
const channels = playlist.channels.filter(channel => {
|
||||||
const url = utils.removeProtocol(i.url)
|
const sameUrl = buffer.find(item => {
|
||||||
const result = typeof buffer[url] === 'undefined'
|
return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url)
|
||||||
if (result) {
|
})
|
||||||
buffer[url] = true
|
if (sameUrl) return false
|
||||||
}
|
const sameHash = buffer.find(item => item.hash === channel.hash)
|
||||||
|
if (sameHash && channel.status === 'Offline') return false
|
||||||
|
|
||||||
return result
|
buffer.push(channel)
|
||||||
|
return true
|
||||||
})
|
})
|
||||||
|
|
||||||
if (playlist.channels.length !== channels.length) {
|
if (playlist.channels.length !== channels.length) {
|
||||||
|
@ -57,21 +63,12 @@ async function removeDuplicates(playlist) {
|
||||||
return playlist
|
return playlist
|
||||||
}
|
}
|
||||||
|
|
||||||
async function removeUnsortedDuplicates(playlist) {
|
async function removeGlobalDuplicates(playlist) {
|
||||||
// locally
|
const channels = playlist.channels.filter(channel => {
|
||||||
let buffer = {}
|
const url = utils.removeProtocol(channel.url)
|
||||||
let channels = playlist.channels.filter(i => {
|
return !globalBuffer.includes(url)
|
||||||
const url = utils.removeProtocol(i.url)
|
|
||||||
const result = typeof buffer[url] === 'undefined'
|
|
||||||
if (result) buffer[url] = true
|
|
||||||
|
|
||||||
return result
|
|
||||||
})
|
})
|
||||||
|
|
||||||
// globally
|
|
||||||
const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
|
|
||||||
channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
|
|
||||||
|
|
||||||
if (channels.length !== playlist.channels.length) {
|
if (channels.length !== playlist.channels.length) {
|
||||||
log.print('updated')
|
log.print('updated')
|
||||||
playlist.channels = channels
|
playlist.channels = channels
|
||||||
|
|
Loading…
Reference in New Issue