diff --git a/package.json b/package.json index ec8ffda009..cb70ef147a 100644 --- a/package.json +++ b/package.json @@ -2,9 +2,9 @@ "name": "iptv", "scripts": { "test": "node test/index.js", - "generate": "node helpers/generate.js", - "format": "node helpers/format.js", - "update-readme": "node helpers/update-readme.js" + "generate": "node scripts/generate.js", + "format": "node scripts/format.js", + "update-readme": "node scripts/update-readme.js" }, "author": "Arhey ", "private": true, diff --git a/helpers/format.js b/scripts/format.js similarity index 77% rename from helpers/format.js rename to scripts/format.js index b18588a72f..cd521d96fc 100644 --- a/helpers/format.js +++ b/scripts/format.js @@ -1,6 +1,4 @@ -const util = require('./util') -const escapeStringRegexp = require('escape-string-regexp') -const ISO6391 = require('iso-639-1') +const helper = require('./helper') const debug = false const verbose = false @@ -19,46 +17,46 @@ let unsorted = {} async function main() { console.log(`Parsing 'index.m3u'...`) - const playlist = util.parsePlaylist('index.m3u') + const playlist = helper.parsePlaylist('index.m3u') const countries = playlist.items if(debug) { console.log('Debug mode is turn on') } - const unsortedPlaylist = util.parsePlaylist('channels/unsorted.m3u') + const unsortedPlaylist = helper.parsePlaylist('channels/unsorted.m3u') for(const item of unsortedPlaylist.items) { - unsorted[item.url] = util.createChannel(item) + unsorted[item.url] = helper.createChannel(item) } for(let country of countries) { - if (util.skipPlaylist(country.url)) { + if (helper.skipPlaylist(country.url)) { continue } if(verbose) { console.log(`Clear cache...`) } - util.clearCache() + helper.clearCache() console.log(`Parsing '${country.url}'...`) - const playlist = util.parsePlaylist(country.url) + const playlist = helper.parsePlaylist(country.url) if(verbose) { console.log(`Creating channels list...`) } let channels = [] for(let item of playlist.items) { - let channel = util.createChannel(item) + let channel = helper.createChannel(item) - if(util.checkCache(channel.url)) { + if(helper.checkCache(channel.url)) { stats.duplicates++ - } else if(!util.validateUrl(channel.url)) { + } else if(!helper.validateUrl(channel.url)) { stats.unvalid++ } else { channels.push(channel) - util.addToCache(channel.url) + helper.addToCache(channel.url) } if(unsorted[channel.url]) { @@ -75,7 +73,7 @@ async function main() { if(epgUrl && !buffer[epgUrl] && parseEpg) { try { console.log(`Loading '${epgUrl}'...`) - const epg = await util.loadEPG(epgUrl) + const epg = await helper.loadEPG(epgUrl) console.log(`Adding '${epgUrl}' to buffer...`) buffer[epgUrl] = epg } catch(e) { @@ -90,7 +88,7 @@ async function main() { let c = buffer[epgUrl].channels[channelId] for(let epgName of c.name) { if(epgName.value) { - let escaped = escapeStringRegexp(epgName.value) + let escaped = helper.escapeStringRegexp(epgName.value) channelTitle = channel.title.replace(/(fhd|hd|sd|高清)$/i, '').trim() let regexp = new RegExp(`^${escaped}$`, 'i') if(regexp.test(channelTitle)) { @@ -123,7 +121,7 @@ async function main() { } if(!channel.language && c.name.length && c.name[0].lang) { - let language = ISO6391.getName(c.name[0].lang) + let language = helper.getISO6391Name(c.name[0].lang) channel.language = language updated = true if(verbose) { @@ -149,13 +147,13 @@ async function main() { if(verbose) { console.log(`Sorting channels...`) } - channels = util.sortByTitleAndUrl(channels) + channels = helper.sortByTitleAndUrl(channels) if(!debug) { console.log(`Updating '${country.url}'...`) - util.createFile(country.url, playlist.getHeader()) + helper.createFile(country.url, playlist.getHeader()) channels.forEach(channel => { - util.appendToFile(country.url, channel.toString()) + helper.appendToFile(country.url, channel.toString()) }) } @@ -164,9 +162,9 @@ async function main() { if(!debug & stats.removed > 0) { console.log(`Updating 'channels/unsorted.m3u'...`) - util.createFile('channels/unsorted.m3u', playlist.getHeader()) + helper.createFile('channels/unsorted.m3u', playlist.getHeader()) Object.values(unsorted).forEach(channel => { - util.appendToFile('channels/unsorted.m3u', channel.toString()) + helper.appendToFile('channels/unsorted.m3u', channel.toString()) }) } diff --git a/helpers/generate.js b/scripts/generate.js similarity index 78% rename from helpers/generate.js rename to scripts/generate.js index eb50b239a8..671757decd 100644 --- a/helpers/generate.js +++ b/scripts/generate.js @@ -1,5 +1,4 @@ -const util = require('./util') -const ISO6391 = require('iso-639-1') +const helper = require('./helper') let list = { all: [], @@ -29,19 +28,19 @@ function main() { } function parseIndex() { - const root = util.parsePlaylist('index.m3u') + const root = helper.parsePlaylist('index.m3u') let countries = {} let languages = {} let categories = {} for(let rootItem of root.items) { - const playlist = util.parsePlaylist(rootItem.url) - const countryCode = util.getBasename(rootItem.url).toLowerCase() + const playlist = helper.parsePlaylist(rootItem.url) + const countryCode = helper.getBasename(rootItem.url).toLowerCase() const countryName = rootItem.name for(let item of playlist.items) { - const channel = util.createChannel(item) + const channel = helper.createChannel(item) channel.countryCode = countryCode channel.countryName = countryName @@ -55,7 +54,7 @@ function parseIndex() { countries[countryCode].push(channel) // language - const languageCode = ISO6391.getCode(channel.language) || 'undefined' + const languageCode = helper.getISO6391Code(channel.language) || 'undefined' if(!languages[languageCode]) { languages[languageCode] = [] } @@ -77,19 +76,19 @@ function parseIndex() { function generateCountryIndex() { const filename = `index.country.m3u` - util.createFile(filename, '#EXTM3U\n') + helper.createFile(filename, '#EXTM3U\n') for(let channel of list.all) { const group = channel.group channel.group = channel.countryName - util.appendToFile(filename, channel.toString()) + helper.appendToFile(filename, channel.toString()) channel.group = group } } function generateLanguageIndex() { const filename = `index.language.m3u` - util.createFile(filename, '#EXTM3U\n') + helper.createFile(filename, '#EXTM3U\n') const channels = list.all.sort((a, b) => { if(a.language < b.language) { return -1 } @@ -100,14 +99,14 @@ function generateLanguageIndex() { for(let channel of channels) { const group = channel.group channel.group = channel.language - util.appendToFile(filename, channel.toString()) + helper.appendToFile(filename, channel.toString()) channel.group = group } } function generateContentIndex() { const filename = `index.content.m3u` - util.createFile(filename, '#EXTM3U\n') + helper.createFile(filename, '#EXTM3U\n') const channels = list.all.sort((a, b) => { if(a.group < b.group) { return -1 } @@ -116,13 +115,13 @@ function generateContentIndex() { }) for(let channel of channels) { - util.appendToFile(filename, channel.toString()) + helper.appendToFile(filename, channel.toString()) } } function generateFullIndex() { const filename = `index.full.m3u` - util.createFile(filename, '#EXTM3U\n') + helper.createFile(filename, '#EXTM3U\n') const channels = list.all.sort((a, b) => { if(a.countryName < b.countryName) { return -1 } @@ -135,7 +134,7 @@ function generateFullIndex() { for(let channel of channels) { const group = channel.group channel.group = [ channel.countryName, channel.group ].filter(i => i).join(';') - util.appendToFile(filename, channel.toString()) + helper.appendToFile(filename, channel.toString()) channel.group = group } } @@ -144,9 +143,9 @@ function generateCategories() { for(let cid in list.categories) { let category = list.categories[cid] const filename = `categories/${cid}.m3u` - util.createFile(filename, '#EXTM3U\n') + helper.createFile(filename, '#EXTM3U\n') for(let channel of category) { - util.appendToFile(filename, channel.toString()) + helper.appendToFile(filename, channel.toString()) } } } @@ -155,9 +154,9 @@ function generateLanguages() { for(let lid in list.languages) { let language = list.languages[lid] const filename = `languages/${lid}.m3u` - util.createFile(filename, '#EXTM3U\n') + helper.createFile(filename, '#EXTM3U\n') for(let channel of language) { - util.appendToFile(filename, channel.toString()) + helper.appendToFile(filename, channel.toString()) } } } diff --git a/helpers/util.js b/scripts/helper.js similarity index 63% rename from helpers/util.js rename to scripts/helper.js index 63bbe3a9c9..ecfa0322be 100644 --- a/helpers/util.js +++ b/scripts/helper.js @@ -6,88 +6,48 @@ const zlib = require("zlib") const epgParser = require('epg-parser') const urlParser = require('url') const ISO6391 = require('iso-639-1') - -const supportedCategories = [ 'Auto','Business', 'Classic','Comedy','Documentary','Education','Entertainment', 'Family','Fashion','Food', 'General', 'Health', 'History', 'Hobby', 'Kids', 'Legislative','Lifestyle','Local', 'Movies', 'Music', 'News', 'Quiz', 'Religious','Sci-Fi', 'Shop', 'Sport', 'Travel', 'Weather', 'XXX' ] - -const blacklist = [ - '80.80.160.168', // repeats on a loop - '63.237.48.3', // not a live stream - '189.216.247.113', // not working streams -] +const escapeStringRegexp = require('escape-string-regexp') +const markdownInclude = require('markdown-include') let cache = {} +let helper = {} -class Playlist { - constructor(data) { - this.header = data.header - this.items = data.items - } - - getHeader() { - let parts = ['#EXTM3U'] - for(let key in this.header.attrs) { - let value = this.header.attrs[key] - if(value) { - parts.push(`${key}="${value}"`) - } - } - - return `${parts.join(' ')}\n` - } +helper.compileMarkdown = function(filepath) { + return markdownInclude.compileFiles(path.resolve(__dirname, filepath)) } -class Channel { - constructor(data) { - this.id = data.tvg.id - this.name = data.tvg.name - this.language = this._filterLanguage(data.tvg.language) - this.logo = data.tvg.logo - this.group = this._filterGroup(data.group.title) - this.url = data.url - this.title = data.name - } - - _filterGroup(groupTitle) { - if(!groupTitle) return '' - - const groupIndex = supportedCategories.map(g => g.toLowerCase()).indexOf(groupTitle.toLowerCase()) - - if(groupIndex === -1) { - groupTitle = '' - } else { - groupTitle = supportedCategories[groupIndex] - } - - return groupTitle - } - - _filterLanguage(languageName) { - if(ISO6391.getCode(languageName) !== '') { - return languageName - } - - return '' - } - - toString() { - const info = `-1 tvg-id="${this.id}" tvg-name="${this.name}" tvg-language="${this.language}" tvg-logo="${this.logo}" group-title="${this.group}",${this.title}` - - return '#EXTINF:' + info + '\n' + this.url + '\n' - } +helper.escapeStringRegexp = function(scring) { + return escapeStringRegexp(string) } -function parsePlaylist(filename) { +helper.getISO6391Name = function(code) { + return ISO6391.getName(code) +} + +helper.getISO6391Code = function(name) { + return ISO6391.getCode(name) +} + +helper.parsePlaylist = function(filename) { const content = readFile(filename) const result = parser.parse(content) return new Playlist(result) } -function createChannel(data) { - return new Channel(data) +helper.createChannel = function(data) { + return new Channel({ + id: data.tvg.id, + name: data.tvg.name, + language: data.tvg.language, + logo: data.tvg.logo, + group: data.group.title, + url: data.url, + title: data.name + }) } -async function loadEPG(url) { +helper.loadEPG = async function(url) { const content = await getEPGFile(url) const result = epgParser.parse(content) const channels = {} @@ -101,7 +61,7 @@ async function loadEPG(url) { }) } -function getEPGFile(url) { +helper.getEPGFile = function(url) { return new Promise((resolve, reject) => { var buffer = [] axios({ @@ -131,58 +91,56 @@ function getEPGFile(url) { }) } -function byTitleAndUrl(a, b) { - var titleA = a.title.toLowerCase() - var titleB = b.title.toLowerCase() - var urlA = a.url.toLowerCase() - var urlB = b.url.toLowerCase() - - if(titleA < titleB) return -1 - if(titleA > titleB) return 1 +helper.sortByTitleAndUrl = function(arr) { + return arr.sort(function(a, b) { + var titleA = a.title.toLowerCase() + var titleB = b.title.toLowerCase() + var urlA = a.url.toLowerCase() + var urlB = b.url.toLowerCase() + + if(titleA < titleB) return -1 + if(titleA > titleB) return 1 - if(urlA < urlB) return -1 - if(urlA > urlB) return 1 + if(urlA < urlB) return -1 + if(urlA > urlB) return 1 - return 0 + return 0 + }) } -function sortByTitleAndUrl(arr) { - return arr.sort(byTitleAndUrl) -} - -function readFile(filename) { +helper.readFile = function(filename) { return fs.readFileSync(path.resolve(__dirname) + `/../${filename}`, { encoding: "utf8" }) } -function appendToFile(filename, data) { +helper.appendToFile = function(filename, data) { fs.appendFileSync(path.resolve(__dirname) + '/../' + filename, data) } -function createFile(filename, data) { +helper.createFile = function(filename, data) { fs.writeFileSync(path.resolve(__dirname) + '/../' + filename, data) } -function getBasename(filename) { +helper.getBasename = function(filename) { return path.basename(filename, path.extname(filename)) } -function addToCache(url) { +helper.addToCache = function(url) { let id = getUrlPath(url) cache[id] = true } -function checkCache(url) { +helper.checkCache = function(url) { let id = getUrlPath(url) return cache.hasOwnProperty(id) } -function clearCache() { +helper.clearCache = function() { cache = {} } -function getUrlPath(u) { +helper.getUrlPath = function(u) { let parsed = urlParser.parse(u) let searchQuery = parsed.search || '' let path = parsed.host + parsed.pathname + searchQuery @@ -190,14 +148,19 @@ function getUrlPath(u) { return path.toLowerCase() } -function validateUrl(channelUrl) { +helper.validateUrl = function(channelUrl) { const url = new URL(channelUrl) const host = url.hostname + const blacklist = [ + '80.80.160.168', // repeats on a loop + '63.237.48.3', // not a live stream + '189.216.247.113', // not working streams + ] return blacklist.indexOf(host) === -1 } -function skipPlaylist(filename) { +helper.skipPlaylist = function(filename) { let testCountry = process.env.npm_config_country let excludeList = process.env.npm_config_exclude let excludeCountries = excludeList ? excludeList.split(',') : [] @@ -211,7 +174,7 @@ function skipPlaylist(filename) { return false } -function generateTable(data, options) { +helper.generateTable = function(data, options) { let output = '' output += '' @@ -240,20 +203,64 @@ function generateTable(data, options) { return output } -module.exports = { - parsePlaylist, - sortByTitleAndUrl, - appendToFile, - createFile, - readFile, - loadEPG, - createChannel, - getBasename, - addToCache, - checkCache, - clearCache, - validateUrl, - skipPlaylist, - supportedCategories, - generateTable -} \ No newline at end of file +class Playlist { + constructor(data) { + this.header = data.header + this.items = data.items + } + + getHeader() { + let parts = ['#EXTM3U'] + for(let key in this.header.attrs) { + let value = this.header.attrs[key] + if(value) { + parts.push(`${key}="${value}"`) + } + } + + return `${parts.join(' ')}\n` + } +} + +class Channel { + constructor(data) { + this.id = data.id + this.name = data.name + this.language = this._filterLanguage(data.language) + this.logo = data.logo + this.group = this._filterGroup(data.group) + this.url = data.url + this.title = data.title + } + + _filterGroup(groupTitle) { + if(!groupTitle) return '' + + const supportedCategories = [ 'Auto','Business', 'Classic','Comedy','Documentary','Education','Entertainment', 'Family','Fashion','Food', 'General', 'Health', 'History', 'Hobby', 'Kids', 'Legislative','Lifestyle','Local', 'Movies', 'Music', 'News', 'Quiz', 'Religious','Sci-Fi', 'Shop', 'Sport', 'Travel', 'Weather', 'XXX' ] + const groupIndex = supportedCategories.map(g => g.toLowerCase()).indexOf(groupTitle.toLowerCase()) + + if(groupIndex === -1) { + groupTitle = '' + } else { + groupTitle = supportedCategories[groupIndex] + } + + return groupTitle + } + + _filterLanguage(languageName) { + if(ISO6391.getCode(languageName) !== '') { + return languageName + } + + return '' + } + + toString() { + const info = `-1 tvg-id="${this.id}" tvg-name="${this.name}" tvg-language="${this.language}" tvg-logo="${this.logo}" group-title="${this.group}",${this.title}` + + return '#EXTINF:' + info + '\n' + this.url + '\n' + } +} + +module.exports = helper \ No newline at end of file diff --git a/helpers/update-readme.js b/scripts/update-readme.js similarity index 76% rename from helpers/update-readme.js rename to scripts/update-readme.js index 83648e29f6..ac4f9a8b38 100644 --- a/helpers/update-readme.js +++ b/scripts/update-readme.js @@ -1,7 +1,4 @@ -const util = require('./util') -const ISO6391 = require('iso-639-1') -const markdownInclude = require('markdown-include') -const path = require('path') +const helper = require('./helper') let output = { countries: [], @@ -24,13 +21,13 @@ function main() { } function parseIndex() { - const root = util.parsePlaylist('index.m3u') + const root = helper.parsePlaylist('index.m3u') let languages = {} let categories = {} for(let rootItem of root.items) { - const playlist = util.parsePlaylist(rootItem.url) - const countryCode = util.getBasename(rootItem.url).toUpperCase() + const playlist = helper.parsePlaylist(rootItem.url) + const countryCode = helper.getBasename(rootItem.url).toUpperCase() const epg = playlist.header.attrs['x-tvg-url'] ? `${playlist.header.attrs['x-tvg-url']}` : '' // country @@ -45,7 +42,7 @@ function parseIndex() { // language const languageName = item.tvg.language || 'Undefined' - const languageCode = ISO6391.getCode(languageName) || 'undefined' + const languageCode = helper.getISO6391Code(languageName) || 'undefined' if(languages[languageCode]) { languages[languageCode].channels++ } else { @@ -57,7 +54,7 @@ function parseIndex() { } // category - const categoryName = util.supportedCategories.find(c => c === item.group.title) || 'Other' + const categoryName = item.group.title || 'Other' const categoryCode = categoryName.toLowerCase() if(categories[categoryCode]) { categories[categoryCode].channels++ @@ -76,7 +73,7 @@ function parseIndex() { } function generateCountriesTable() { - const table = util.generateTable(output.countries, { + const table = helper.generateTable(output.countries, { columns: [ { name: 'Country', align: 'left' }, { name: 'Channels', align: 'right' }, @@ -85,7 +82,7 @@ function generateCountriesTable() { ] }) - util.createFile('./.readme/_countries.md', table) + helper.createFile('./.readme/_countries.md', table) } function generateLanguagesTable() { @@ -97,7 +94,7 @@ function generateLanguagesTable() { return 0 }) - const table = util.generateTable(output.languages, { + const table = helper.generateTable(output.languages, { columns: [ { name: 'Language', align: 'left' }, { name: 'Channels', align: 'right' }, @@ -105,7 +102,7 @@ function generateLanguagesTable() { ] }) - util.createFile('./.readme/_languages.md', table) + helper.createFile('./.readme/_languages.md', table) } function generateCategoriesTable() { @@ -117,7 +114,7 @@ function generateCategoriesTable() { return 0 }) - const table = util.generateTable(output.categories, { + const table = helper.generateTable(output.categories, { columns: [ { name: 'Category', align: 'left' }, { name: 'Channels', align: 'right' }, @@ -125,11 +122,11 @@ function generateCategoriesTable() { ] }) - util.createFile('./.readme/_categories.md', table) + helper.createFile('./.readme/_categories.md', table) } function generateReadme() { - markdownInclude.compileFiles(path.resolve(__dirname, '../.readme/config.json')) + helper.compileMarkdown('../.readme/config.json') } main() diff --git a/test/index.js b/test/index.js index 5cd462f451..4bc9cf606f 100644 --- a/test/index.js +++ b/test/index.js @@ -1,6 +1,6 @@ process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0 -const util = require('../helpers/util') +const helper = require('../scripts/helper') const ffmpeg = require('fluent-ffmpeg') const verbose = process.env.npm_config_debug || false @@ -19,19 +19,19 @@ async function test() { stats.tests++ - const playlist = util.parsePlaylist('index.m3u') + const playlist = helper.parsePlaylist('index.m3u') const countries = playlist.items for(let country of countries) { - if (util.skipPlaylist(country.url)) { + if (helper.skipPlaylist(country.url)) { continue } console.log(`Checking '${country.url}'...`) - const playlist = util.parsePlaylist(country.url) + const playlist = helper.parsePlaylist(country.url) for(let item of playlist.items) { @@ -90,7 +90,7 @@ test() function writeToLog(country, msg, url) { var now = new Date() var line = `${country}: ${msg} '${url}'` - util.appendToFile(errorLog, now.toISOString() + ' ' + line + '\n') + helper.appendToFile(errorLog, now.toISOString() + ' ' + line + '\n') console.log(`${msg} '${url}'`) }