From 0a8f8ba400426f8fcc5f5b801f09d214899ccc77 Mon Sep 17 00:00:00 2001 From: freearhey Date: Mon, 4 May 2020 16:23:03 +0300 Subject: [PATCH 1/5] Code refactoring --- scripts/helper.js | 200 ++++++++++++++++++++++++++++------------------ 1 file changed, 122 insertions(+), 78 deletions(-) diff --git a/scripts/helper.js b/scripts/helper.js index bf5cbd4980..68437196f6 100644 --- a/scripts/helper.js +++ b/scripts/helper.js @@ -15,10 +15,13 @@ let helper = {} helper.sortBy = function (arr, fields) { return arr.sort((a, b) => { for (let field of fields) { - if (a[field].toLowerCase() < b[field].toLowerCase()) { + let propA = a[field] ? a[field].toLowerCase() : '' + let propB = b[field] ? b[field].toLowerCase() : '' + + if (propA < propB) { return -1 } - if (a[field].toLowerCase() > b[field].toLowerCase()) { + if (propA > propB) { return 1 } } @@ -41,13 +44,13 @@ helper.escapeStringRegexp = function (scring) { } helper.getISO6391Name = function (code) { - const lang = iso6393.find((l) => l.iso6393 === code.toLowerCase()) + const lang = iso6393.find(l => l.iso6393 === code.toLowerCase()) return lang && lang.name ? lang.name : null } helper.getISO6391Code = function (name) { - const lang = iso6393.find((l) => l.name === name) + const lang = iso6393.find(l => l.name === name) return lang && lang.iso6393 ? lang.iso6393 : null } @@ -69,7 +72,7 @@ helper.parseEPG = async function (url) { return Promise.resolve({ url, - channels, + channels }) } @@ -80,9 +83,9 @@ helper.getEPG = function (url) { method: 'get', url: url, responseType: 'stream', - timeout: 60000, + timeout: 60000 }) - .then((res) => { + .then(res => { let stream if (/\.gz$/i.test(url)) { let gunzip = zlib.createGunzip() @@ -103,7 +106,7 @@ helper.getEPG = function (url) { reject(e) }) }) - .catch((e) => { + .catch(e => { reject(e) }) }) @@ -179,7 +182,7 @@ helper.parseMessage = function (err, u) { if (msgArr.length === 0) return - const line = msgArr.find((line) => { + const line = msgArr.find(line => { return line.indexOf(u) === 0 }) @@ -190,20 +193,65 @@ helper.parseMessage = function (err, u) { helper.filterPlaylists = function (arr, include = '', exclude = '') { if (include) { - const included = include.split(',').map((filename) => `channels/${filename}.m3u`) + const included = include.split(',').map(filename => `channels/${filename}.m3u`) - return arr.filter((i) => included.indexOf(i.url) > -1) + return arr.filter(i => included.indexOf(i.url) > -1) } if (exclude) { - const excluded = exclude.split(',').map((filename) => `channels/${filename}.m3u`) + const excluded = exclude.split(',').map(filename => `channels/${filename}.m3u`) - return arr.filter((i) => excluded.indexOf(i.url) === -1) + return arr.filter(i => excluded.indexOf(i.url) === -1) } return arr } +helper.filterGroup = function (groupTitle) { + if (!groupTitle) return '' + + const supportedCategories = [ + 'Auto', + 'Business', + 'Classic', + 'Comedy', + 'Documentary', + 'Education', + 'Entertainment', + 'Family', + 'Fashion', + 'Food', + 'General', + 'Health', + 'History', + 'Hobby', + 'Kids', + 'Legislative', + 'Lifestyle', + 'Local', + 'Movies', + 'Music', + 'News', + 'Quiz', + 'Religious', + 'Sci-Fi', + 'Shop', + 'Sport', + 'Travel', + 'Weather', + 'XXX' + ] + const groupIndex = supportedCategories.map(g => g.toLowerCase()).indexOf(groupTitle.toLowerCase()) + + if (groupIndex === -1) { + groupTitle = '' + } else { + groupTitle = supportedCategories[groupIndex] + } + + return groupTitle +} + class Playlist { constructor(data) { this.header = data.header @@ -226,97 +274,93 @@ class Playlist { class Channel { constructor(data) { - this.id = data.tvg.id - this.name = data.tvg.name - this.language = data.tvg.language - .split(';') - .filter((l) => !!helper.getISO6391Code(l)) - .join(';') - this.logo = data.tvg.logo - this.group = this._filterGroup(data.group.title) - this.url = data.url - this.title = data.name.trim() - this.userAgent = data.http['user-agent'] - this.referrer = data.http['referrer'] + this.parseData(data) } - _filterGroup(groupTitle) { - if (!groupTitle) return '' + parseData(data) { + const language = data.tvg.language + .split(';') + .map(name => { + const code = name ? helper.getISO6391Code(name) : null + if (!code) return null - const supportedCategories = [ - 'Auto', - 'Business', - 'Classic', - 'Comedy', - 'Documentary', - 'Education', - 'Entertainment', - 'Family', - 'Fashion', - 'Food', - 'General', - 'Health', - 'History', - 'Hobby', - 'Kids', - 'Legislative', - 'Lifestyle', - 'Local', - 'Movies', - 'Music', - 'News', - 'Quiz', - 'Religious', - 'Sci-Fi', - 'Shop', - 'Sport', - 'Travel', - 'Weather', - 'XXX', - ] - const groupIndex = supportedCategories - .map((g) => g.toLowerCase()) - .indexOf(groupTitle.toLowerCase()) + return { + code, + name + } + }) + .filter(l => l) - if (groupIndex === -1) { - groupTitle = '' - } else { - groupTitle = supportedCategories[groupIndex] + this.language = language + this.logo = data.tvg.logo + this.category = helper.filterGroup(data.group.title) + this.url = data.url + this.name = data.name.trim() + this.http = data.http + this.tvg = data.tvg + this.country = { + code: null, + name: null } + } - return groupTitle + get ['language.name']() { + return this.language[0] ? this.language[0].name : null + } + + get ['country.name']() { + return this.country.name || null } toString() { - const country = this.countryCode.toUpperCase() - const epg = this.id && this.epg ? this.epg : '' + const country = this.country.code ? this.country.code.toUpperCase() : '' + const tvgUrl = (this.tvg.id || this.tvg.name) && this.tvg.url ? this.tvg.url : '' + const language = this.language.map(l => l.name).join(';') - let info = `-1 tvg-id="${this.id}" tvg-name="${this.name}" tvg-language="${this.language}" tvg-logo="${this.logo}" tvg-country="${country}" tvg-url="${epg}" group-title="${this.group}",${this.title}` + let info = `-1 tvg-id="${this.tvg.id}" tvg-name="${this.tvg.name}" tvg-language="${language}" tvg-logo="${this.logo}" tvg-country="${country}" tvg-url="${tvgUrl}" group-title="${this.category}",${this.name}` - if (this.referrer) { - info += `\n#EXTVLCOPT:http-referrer=${this.referrer}` + if (this.http['referrer']) { + info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}` } - if (this.userAgent) { - info += `\n#EXTVLCOPT:http-user-agent=${this.userAgent}` + if (this.http['user-agent']) { + info += `\n#EXTVLCOPT:http-user-agent=${this.http['user-agent']}` } return '#EXTINF:' + info + '\n' + this.url + '\n' } toShortString() { - let info = `-1 tvg-id="${this.id}" tvg-name="${this.name}" tvg-language="${this.language}" tvg-logo="${this.logo}" group-title="${this.group}",${this.title}` + const language = this.language.map(l => l.name).join(';') - if (this.referrer) { - info += `\n#EXTVLCOPT:http-referrer=${this.referrer}` + let info = `-1 tvg-id="${this.tvg.id}" tvg-name="${this.tvg.name}" tvg-language="${language}" tvg-logo="${this.logo}" group-title="${this.category}",${this.namee}` + + if (this.http['referrer']) { + info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}` } - if (this.userAgent) { - info += `\n#EXTVLCOPT:http-user-agent=${this.userAgent}` + if (this.http['user-agent']) { + info += `\n#EXTVLCOPT:http-user-agent=${this.http['user-agent']}` } return '#EXTINF:' + info + '\n' + this.url + '\n' } + + toJSON() { + return { + name: this.name, + logo: this.logo || null, + url: this.url, + category: this.category || null, + language: this.language, + country: this.country, + tvg: { + id: this.tvg.id || null, + name: this.tvg.name || null, + url: this.tvg.url || null + } + } + } } module.exports = helper From ca06ca5f65ce56e1030b52c6b735f2c9996bfb58 Mon Sep 17 00:00:00 2001 From: freearhey Date: Mon, 4 May 2020 16:23:36 +0300 Subject: [PATCH 2/5] Generate channels.json Issue #1314 --- scripts/generate.js | 55 +++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/scripts/generate.js b/scripts/generate.js index 9fd5e11511..bb8655bcc4 100644 --- a/scripts/generate.js +++ b/scripts/generate.js @@ -18,6 +18,8 @@ function main() { createNoJekyllFile() console.log('Generating index.m3u...') generateIndex() + console.log('Generating channels.json...') + generateChannels() console.log('Generating index.country.m3u...') generateCountryIndex() console.log('Generating index.language.m3u...') @@ -61,9 +63,9 @@ function parseIndex() { for (let item of playlist.items) { const channel = helper.createChannel(item) - channel.countryCode = countryCode - channel.countryName = countryName - channel.epg = playlist.header.attrs['x-tvg-url'] || '' + channel.country.code = countryCode + channel.country.name = countryName + channel.tvg.url = playlist.header.attrs['x-tvg-url'] || '' // all list.all.push(channel) @@ -75,16 +77,24 @@ function parseIndex() { countries[countryCode].push(channel) // language - for (let language of channel.language.split(';')) { - const languageCode = helper.getISO6391Code(language) || 'undefined' + if (!channel.language.length) { + const languageCode = 'undefined' if (!languages[languageCode]) { languages[languageCode] = [] } languages[languageCode].push(channel) + } else { + for (let language of channel.language) { + const languageCode = language.code || 'undefined' + if (!languages[languageCode]) { + languages[languageCode] = [] + } + languages[languageCode].push(channel) + } } // category - const categoryCode = channel.group.toLowerCase() || 'other' + const categoryCode = channel.category ? channel.category.toLowerCase() : 'other' if (!categories[categoryCode]) { categories[categoryCode] = [] } @@ -101,22 +111,29 @@ function generateIndex() { const filename = `${ROOT_DIR}/index.m3u` helper.createFile(filename, '#EXTM3U\n') - const channels = helper.sortBy(list.all, ['title', 'url']) + const channels = helper.sortBy(list.all, ['name', 'url']) for (let channel of channels) { helper.appendToFile(filename, channel.toString()) } } +function generateChannels() { + const filename = `${ROOT_DIR}/channels.json` + const sorted = helper.sortBy(list.all, ['name', 'url']) + const channels = sorted.map(c => c.toJSON()) + helper.createFile(filename, JSON.stringify(channels, null, '\t')) +} + function generateCountryIndex() { const filename = `${ROOT_DIR}/index.country.m3u` helper.createFile(filename, '#EXTM3U\n') - const channels = helper.sortBy(list.all, ['countryName', 'title', 'url']) + const channels = helper.sortBy(list.all, ['country.name', 'name', 'url']) for (let channel of channels) { - const group = channel.group - channel.group = channel.countryName + const category = channel.category + channel.category = channel.country.name helper.appendToFile(filename, channel.toString()) - channel.group = group + channel.category = category } } @@ -124,12 +141,12 @@ function generateLanguageIndex() { const filename = `${ROOT_DIR}/index.language.m3u` helper.createFile(filename, '#EXTM3U\n') - const channels = helper.sortBy(list.all, ['language', 'title', 'url']) + const channels = helper.sortBy(list.all, ['language.name', 'name', 'url']) for (let channel of channels) { - const group = channel.group - channel.group = channel.language + const category = channel.category + channel.category = channel.language.map(l => l.name).join(';') helper.appendToFile(filename, channel.toString()) - channel.group = group + channel.category = category } } @@ -137,7 +154,7 @@ function generateCategoryIndex() { const filename = `${ROOT_DIR}/index.category.m3u` helper.createFile(filename, '#EXTM3U\n') - const channels = helper.sortBy(list.all, ['group', 'title', 'url']) + const channels = helper.sortBy(list.all, ['category', 'name', 'url']) for (let channel of channels) { helper.appendToFile(filename, channel.toString()) } @@ -152,7 +169,7 @@ function generateCountries() { const filename = `${outputDir}/${cid}.m3u` helper.createFile(filename, '#EXTM3U\n') - const channels = helper.sortBy(Object.values(country), ['title', 'url']) + const channels = helper.sortBy(Object.values(country), ['name', 'url']) for (let channel of channels) { helper.appendToFile(filename, channel.toString()) } @@ -168,7 +185,7 @@ function generateCategories() { const filename = `${outputDir}/${cid}.m3u` helper.createFile(filename, '#EXTM3U\n') - const channels = helper.sortBy(Object.values(category), ['title', 'url']) + const channels = helper.sortBy(Object.values(category), ['name', 'url']) for (let channel of channels) { helper.appendToFile(filename, channel.toString()) } @@ -184,7 +201,7 @@ function generateLanguages() { const filename = `${outputDir}/${lid}.m3u` helper.createFile(filename, '#EXTM3U\n') - const channels = helper.sortBy(Object.values(language), ['title', 'url']) + const channels = helper.sortBy(Object.values(language), ['name', 'url']) for (let channel of channels) { helper.appendToFile(filename, channel.toString()) } From 595b4e044c5c6834ba33fced8f7c425fcd12194d Mon Sep 17 00:00:00 2001 From: freearhey Date: Mon, 4 May 2020 16:31:58 +0300 Subject: [PATCH 3/5] Created channel.setLanguage() method --- scripts/helper.js | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/scripts/helper.js b/scripts/helper.js index 68437196f6..30707ebe60 100644 --- a/scripts/helper.js +++ b/scripts/helper.js @@ -278,7 +278,30 @@ class Channel { } parseData(data) { - const language = data.tvg.language + this.logo = data.tvg.logo + this.category = helper.filterGroup(data.group.title) + this.url = data.url + this.name = data.name.trim() + this.http = data.http + this.tvg = data.tvg + this.country = { + code: null, + name: null + } + + this.setLanguage(data.tvg.language) + } + + get ['language.name']() { + return this.language[0] ? this.language[0].name : null + } + + get ['country.name']() { + return this.country.name || null + } + + setLanguage(lang) { + this.language = lang .split(';') .map(name => { const code = name ? helper.getISO6391Code(name) : null @@ -290,26 +313,6 @@ class Channel { } }) .filter(l => l) - - this.language = language - this.logo = data.tvg.logo - this.category = helper.filterGroup(data.group.title) - this.url = data.url - this.name = data.name.trim() - this.http = data.http - this.tvg = data.tvg - this.country = { - code: null, - name: null - } - } - - get ['language.name']() { - return this.language[0] ? this.language[0].name : null - } - - get ['country.name']() { - return this.country.name || null } toString() { From 78cecd8594df3e81d370a76bb54de7279eb4184c Mon Sep 17 00:00:00 2001 From: freearhey Date: Mon, 4 May 2020 16:32:58 +0300 Subject: [PATCH 4/5] Fixes channel name issue --- scripts/helper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helper.js b/scripts/helper.js index 30707ebe60..7d7791bb07 100644 --- a/scripts/helper.js +++ b/scripts/helper.js @@ -336,7 +336,7 @@ class Channel { toShortString() { const language = this.language.map(l => l.name).join(';') - let info = `-1 tvg-id="${this.tvg.id}" tvg-name="${this.tvg.name}" tvg-language="${language}" tvg-logo="${this.logo}" group-title="${this.category}",${this.namee}` + let info = `-1 tvg-id="${this.tvg.id}" tvg-name="${this.tvg.name}" tvg-language="${language}" tvg-logo="${this.logo}" group-title="${this.category}",${this.name}` if (this.http['referrer']) { info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}` From 0643d5eb1ceb04d61df267d9722a820c6b6fc884 Mon Sep 17 00:00:00 2001 From: freearhey Date: Mon, 4 May 2020 16:33:02 +0300 Subject: [PATCH 5/5] Update format.js --- scripts/format.js | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index ea804fb08e..70d963cd8d 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -4,7 +4,7 @@ const config = { debug: process.env.npm_config_debug || false, country: process.env.npm_config_country, exclude: process.env.npm_config_exclude, - epg: process.env.npm_config_epg || false, + epg: process.env.npm_config_epg || false } let updated = 0 @@ -75,7 +75,7 @@ function parseIndex() { function parsePlaylist(url) { const playlist = helper.parsePlaylist(url) - playlist.items = playlist.items.map((item) => { + playlist.items = playlist.items.map(item => { return helper.createChannel(item) }) @@ -84,7 +84,7 @@ function parsePlaylist(url) { function sortChannels(playlist) { const channels = JSON.stringify(playlist.items) - playlist.items = helper.sortBy(playlist.items, ['title', 'url']) + playlist.items = helper.sortBy(playlist.items, ['name', 'url']) if (channels !== JSON.stringify(playlist.items)) { playlist.changed = true } @@ -95,14 +95,14 @@ function sortChannels(playlist) { function removeDuplicates(playlist) { let buffer = {} const channels = JSON.stringify(playlist.items) - playlist.items = playlist.items.filter((i) => { + playlist.items = playlist.items.filter(i => { let result = typeof buffer[i.url] === 'undefined' if (result) { buffer[i.url] = true } else { if (config.debug) { - console.log(`Duplicate of '${i.title}' has been removed`) + console.log(`Duplicate of '${i.name}' has been removed`) } } @@ -128,34 +128,34 @@ async function loadEPG(url) { function addDataFromEPG(playlist, epg) { if (!epg) return playlist - for (let item of playlist.items) { - if (!item.id) continue + for (let channel of playlist.items) { + if (!channel.tvg.id) continue - const channel = epg.channels[item.id] + const epgItem = epg.channels[channel.tvg.id] - if (!channel) continue + if (!epgItem) continue - if (!item.name && channel.name.length) { - item.name = channel.name[0].value + if (!channel.tvg.name && epgItem.name.length) { + channel.tvg.name = epgItem.name[0].value playlist.changed = true if (config.debug) { - console.log(`Added tvg-name '${item.name}' to '${item.title}'`) + console.log(`Added tvg-name '${channel.tvg.name}' to '${channel.name}'`) } } - if (!item.language && channel.name.length && channel.name[0].lang) { - item.language = channel.name[0].lang + if (!channel.language.length && epgItem.name.length && epgItem.name[0].lang) { + channel.setLanguage(epgItem.name[0].lang) playlist.changed = true if (config.debug) { - console.log(`Added tvg-language '${item.language}' to '${item.title}'`) + console.log(`Added tvg-language '${epgItem.name[0].lang}' to '${channel.name}'`) } } - if (!item.logo && channel.icon.length) { - item.logo = channel.icon[0] + if (!channel.logo && epgItem.icon.length) { + channel.logo = epgItem.icon[0] playlist.changed = true if (config.debug) { - console.log(`Added tvg-logo '${item.logo}' to '${item.title}'`) + console.log(`Added tvg-logo '${channel.logo}' to '${channel.name}'`) } } } @@ -173,10 +173,10 @@ function updatePlaylist(filepath, playlist) { } function filterUnsorted() { - const urls = items.map((i) => i.url) + const urls = items.map(i => i.url) const unsortedPlaylist = parsePlaylist('channels/unsorted.m3u') const before = unsortedPlaylist.items.length - unsortedPlaylist.items = unsortedPlaylist.items.filter((i) => !urls.includes(i.url)) + unsortedPlaylist.items = unsortedPlaylist.items.filter(i => !urls.includes(i.url)) if (before !== unsortedPlaylist.items.length) { updatePlaylist('channels/unsorted.m3u', unsortedPlaylist)