ixixy
/
iptv
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #4055 from iptv-org/update-remove-duplicates-js 

Update remove-duplicates.js
This commit is contained in:
Dum4G 2021-08-13 02:52:04 +03:00 committed by GitHub
parent ae5fbce25e b81c8ae9f5
commit 30e1b556fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/helpers/Channel.js
View File

@ -20,6 +20,11 @@ module.exports = class Channel {
this.category = this.parseCategory(data.group.title) this.category = this.parseCategory(data.group.title)
this.countries = this.parseCountries(data.tvg.country) this.countries = this.parseCountries(data.tvg.country)
this.languages = this.parseLanguages(data.tvg.language) this.languages = this.parseLanguages(data.tvg.language)
this.hash = this.generateHash()
}
generateHash() {
return `${this.tvg.id}:${this.tvg.name}:${this.tvg.country}:${this.tvg.language}:${this.logo}:${this.group.title}:${this.name}`.toLowerCase()
} }
updateUrl(url) { updateUrl(url) {

47
scripts/remove-duplicates.js
View File

@ -13,7 +13,7 @@ async function main() {
log.print(`\nProcessing '${playlist.url}'...`) log.print(`\nProcessing '${playlist.url}'...`)
await parser await parser
.parsePlaylist(playlist.url) .parsePlaylist(playlist.url)
.then(addToBuffer) .then(addToGlobalBuffer)
.then(removeDuplicates) .then(removeDuplicates)
.then(p => p.save()) .then(p => p.save())
} }
@ -22,7 +22,8 @@ async function main() {
log.print(`\nProcessing 'channels/unsorted.m3u'...`) log.print(`\nProcessing 'channels/unsorted.m3u'...`)
await parser await parser
.parsePlaylist('channels/unsorted.m3u') .parsePlaylist('channels/unsorted.m3u')
.then(removeUnsortedDuplicates) .then(removeDuplicates)
.then(removeGlobalDuplicates)
.then(p => p.save()) .then(p => p.save())
} }
@ -30,22 +31,27 @@ async function main() {
log.finish() log.finish()
} }
async function addToBuffer(playlist) { async function addToGlobalBuffer(playlist) {
globalBuffer = globalBuffer.concat(playlist.channels) playlist.channels.forEach(channel => {
const url = utils.removeProtocol(channel.url)
globalBuffer.push(url)
})
return playlist return playlist
} }
async function removeDuplicates(playlist) { async function removeDuplicates(playlist) {
let buffer = {} const buffer = []
const channels = playlist.channels.filter(i => { const channels = playlist.channels.filter(channel => {
const url = utils.removeProtocol(i.url) const sameUrl = buffer.find(item => {
const result = typeof buffer[url] === 'undefined' return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url)
if (result) { })
buffer[url] = true if (sameUrl) return false
} const sameHash = buffer.find(item => item.hash === channel.hash)
if (sameHash && channel.status === 'Offline') return false
return result buffer.push(channel)
return true
}) })
if (playlist.channels.length !== channels.length) { if (playlist.channels.length !== channels.length) {
@ -57,21 +63,12 @@ async function removeDuplicates(playlist) {
return playlist return playlist
} }
async function removeUnsortedDuplicates(playlist) { async function removeGlobalDuplicates(playlist) {
// locally const channels = playlist.channels.filter(channel => {
let buffer = {} const url = utils.removeProtocol(channel.url)
let channels = playlist.channels.filter(i => { return !globalBuffer.includes(url)
const url = utils.removeProtocol(i.url)
const result = typeof buffer[url] === 'undefined'
if (result) buffer[url] = true
return result
}) })
// globally
const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
if (channels.length !== playlist.channels.length) { if (channels.length !== playlist.channels.length) {
log.print('updated') log.print('updated')
playlist.channels = channels playlist.channels = channels