ckyri015
/
iptv
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move all functions to scripts folder

This commit is contained in:
freearhey 2019-11-02 12:45:09 +03:00
parent 36f94d807a
commit 97f6a68df8
6 changed files with 176 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
package.json
View File

@ -2,9 +2,9 @@
"name": "iptv",
"scripts": {
"test": "node test/index.js",
"generate": "node helpers/generate.js",
"format": "node helpers/format.js",
"update-readme": "node helpers/update-readme.js"
"generate": "node scripts/generate.js",
"format": "node scripts/format.js",
"update-readme": "node scripts/update-readme.js"
},
"author": "Arhey <free.arhey@gmail.com>",
"private": true,

40
helpers/format.js → scripts/format.js
View File

@ -1,6 +1,4 @@
const util = require('./util')
const escapeStringRegexp = require('escape-string-regexp')
const ISO6391 = require('iso-639-1')
const helper = require('./helper')
const debug = false
const verbose = false
@ -19,46 +17,46 @@ let unsorted = {}
async function main() {
console.log(`Parsing 'index.m3u'...`)
const playlist = util.parsePlaylist('index.m3u')
const playlist = helper.parsePlaylist('index.m3u')
const countries = playlist.items
if(debug) {
console.log('Debug mode is turn on')
}
const unsortedPlaylist = util.parsePlaylist('channels/unsorted.m3u')
const unsortedPlaylist = helper.parsePlaylist('channels/unsorted.m3u')
for(const item of unsortedPlaylist.items) {
unsorted[item.url] = util.createChannel(item)
unsorted[item.url] = helper.createChannel(item)
}
for(let country of countries) {
if (util.skipPlaylist(country.url)) {
if (helper.skipPlaylist(country.url)) {
continue
}
if(verbose) {
console.log(`Clear cache...`)
}
util.clearCache()
helper.clearCache()
console.log(`Parsing '${country.url}'...`)
const playlist = util.parsePlaylist(country.url)
const playlist = helper.parsePlaylist(country.url)
if(verbose) {
console.log(`Creating channels list...`)
}
let channels = []
for(let item of playlist.items) {
let channel = util.createChannel(item)
let channel = helper.createChannel(item)
if(util.checkCache(channel.url)) {
if(helper.checkCache(channel.url)) {
stats.duplicates++
} else if(!util.validateUrl(channel.url)) {
} else if(!helper.validateUrl(channel.url)) {
stats.unvalid++
} else {
channels.push(channel)
util.addToCache(channel.url)
helper.addToCache(channel.url)
}
if(unsorted[channel.url]) {
@ -75,7 +73,7 @@ async function main() {
if(epgUrl && !buffer[epgUrl] && parseEpg) {
try {
console.log(`Loading '${epgUrl}'...`)
const epg = await util.loadEPG(epgUrl)
const epg = await helper.loadEPG(epgUrl)
console.log(`Adding '${epgUrl}' to buffer...`)
buffer[epgUrl] = epg
} catch(e) {
@ -90,7 +88,7 @@ async function main() {
let c = buffer[epgUrl].channels[channelId]
for(let epgName of c.name) {
if(epgName.value) {
let escaped = escapeStringRegexp(epgName.value)
let escaped = helper.escapeStringRegexp(epgName.value)
channelTitle = channel.title.replace(/(fhd|hd|sd|高清)$/i, '').trim()
let regexp = new RegExp(`^${escaped}$`, 'i')
if(regexp.test(channelTitle)) {
@ -123,7 +121,7 @@ async function main() {
}
if(!channel.language && c.name.length && c.name[0].lang) {
let language = ISO6391.getName(c.name[0].lang)
let language = helper.getISO6391Name(c.name[0].lang)
channel.language = language
updated = true
if(verbose) {
@ -149,13 +147,13 @@ async function main() {
if(verbose) {
console.log(`Sorting channels...`)
}
channels = util.sortByTitleAndUrl(channels)
channels = helper.sortByTitleAndUrl(channels)
if(!debug) {
console.log(`Updating '${country.url}'...`)
util.createFile(country.url, playlist.getHeader())
helper.createFile(country.url, playlist.getHeader())
channels.forEach(channel => {
util.appendToFile(country.url, channel.toString())
helper.appendToFile(country.url, channel.toString())
})
}
@ -164,9 +162,9 @@ async function main() {
if(!debug & stats.removed > 0) {
console.log(`Updating 'channels/unsorted.m3u'...`)
util.createFile('channels/unsorted.m3u', playlist.getHeader())
helper.createFile('channels/unsorted.m3u', playlist.getHeader())
Object.values(unsorted).forEach(channel => {
util.appendToFile('channels/unsorted.m3u', channel.toString())
helper.appendToFile('channels/unsorted.m3u', channel.toString())
})
}

37
helpers/generate.js → scripts/generate.js
View File

@ -1,5 +1,4 @@
const util = require('./util')
const ISO6391 = require('iso-639-1')
const helper = require('./helper')
let list = {
all: [],
@ -29,19 +28,19 @@ function main() {
}
function parseIndex() {
const root = util.parsePlaylist('index.m3u')
const root = helper.parsePlaylist('index.m3u')
let countries = {}
let languages = {}
let categories = {}
for(let rootItem of root.items) {
const playlist = util.parsePlaylist(rootItem.url)
const countryCode = util.getBasename(rootItem.url).toLowerCase()
const playlist = helper.parsePlaylist(rootItem.url)
const countryCode = helper.getBasename(rootItem.url).toLowerCase()
const countryName = rootItem.name
for(let item of playlist.items) {
const channel = util.createChannel(item)
const channel = helper.createChannel(item)
channel.countryCode = countryCode
channel.countryName = countryName
@ -55,7 +54,7 @@ function parseIndex() {
countries[countryCode].push(channel)
// language
const languageCode = ISO6391.getCode(channel.language) || 'undefined'
const languageCode = helper.getISO6391Code(channel.language) || 'undefined'
if(!languages[languageCode]) {
languages[languageCode] = []
}
@ -77,19 +76,19 @@ function parseIndex() {
function generateCountryIndex() {
const filename = `index.country.m3u`
util.createFile(filename, '#EXTM3U\n')
helper.createFile(filename, '#EXTM3U\n')
for(let channel of list.all) {
const group = channel.group
channel.group = channel.countryName
util.appendToFile(filename, channel.toString())
helper.appendToFile(filename, channel.toString())
channel.group = group
}
}
function generateLanguageIndex() {
const filename = `index.language.m3u`
util.createFile(filename, '#EXTM3U\n')
helper.createFile(filename, '#EXTM3U\n')
const channels = list.all.sort((a, b) => {
if(a.language < b.language) { return -1 }
@ -100,14 +99,14 @@ function generateLanguageIndex() {
for(let channel of channels) {
const group = channel.group
channel.group = channel.language
util.appendToFile(filename, channel.toString())
helper.appendToFile(filename, channel.toString())
channel.group = group
}
}
function generateContentIndex() {
const filename = `index.content.m3u`
util.createFile(filename, '#EXTM3U\n')
helper.createFile(filename, '#EXTM3U\n')
const channels = list.all.sort((a, b) => {
if(a.group < b.group) { return -1 }
@ -116,13 +115,13 @@ function generateContentIndex() {
})
for(let channel of channels) {
util.appendToFile(filename, channel.toString())
helper.appendToFile(filename, channel.toString())
}
}
function generateFullIndex() {
const filename = `index.full.m3u`
util.createFile(filename, '#EXTM3U\n')
helper.createFile(filename, '#EXTM3U\n')
const channels = list.all.sort((a, b) => {
if(a.countryName < b.countryName) { return -1 }
@ -135,7 +134,7 @@ function generateFullIndex() {
for(let channel of channels) {
const group = channel.group
channel.group = [ channel.countryName, channel.group ].filter(i => i).join(';')
util.appendToFile(filename, channel.toString())
helper.appendToFile(filename, channel.toString())
channel.group = group
}
}
@ -144,9 +143,9 @@ function generateCategories() {
for(let cid in list.categories) {
let category = list.categories[cid]
const filename = `categories/${cid}.m3u`
util.createFile(filename, '#EXTM3U\n')
helper.createFile(filename, '#EXTM3U\n')
for(let channel of category) {
util.appendToFile(filename, channel.toString())
helper.appendToFile(filename, channel.toString())
}
}
}
@ -155,9 +154,9 @@ function generateLanguages() {
for(let lid in list.languages) {
let language = list.languages[lid]
const filename = `languages/${lid}.m3u`
util.createFile(filename, '#EXTM3U\n')
helper.createFile(filename, '#EXTM3U\n')
for(let channel of language) {
util.appendToFile(filename, channel.toString())
helper.appendToFile(filename, channel.toString())
}
}
}

229
helpers/util.js → scripts/helper.js
View File

@ -6,88 +6,48 @@ const zlib = require("zlib")
const epgParser = require('epg-parser')
const urlParser = require('url')
const ISO6391 = require('iso-639-1')
const supportedCategories = [ 'Auto','Business', 'Classic','Comedy','Documentary','Education','Entertainment', 'Family','Fashion','Food', 'General', 'Health', 'History', 'Hobby', 'Kids', 'Legislative','Lifestyle','Local', 'Movies', 'Music', 'News', 'Quiz', 'Religious','Sci-Fi', 'Shop', 'Sport', 'Travel', 'Weather', 'XXX' ]
const blacklist = [
'80.80.160.168', // repeats on a loop
'63.237.48.3', // not a live stream
'189.216.247.113', // not working streams
]
const escapeStringRegexp = require('escape-string-regexp')
const markdownInclude = require('markdown-include')
let cache = {}
let helper = {}
class Playlist {
constructor(data) {
this.header = data.header
this.items = data.items
}
getHeader() {
let parts = ['#EXTM3U']
for(let key in this.header.attrs) {
let value = this.header.attrs[key]
if(value) {
parts.push(`${key}="${value}"`)
}
}
return `${parts.join(' ')}\n`
}
helper.compileMarkdown = function(filepath) {
return markdownInclude.compileFiles(path.resolve(__dirname, filepath))
}
class Channel {
constructor(data) {
this.id = data.tvg.id
this.name = data.tvg.name
this.language = this._filterLanguage(data.tvg.language)
this.logo = data.tvg.logo
this.group = this._filterGroup(data.group.title)
this.url = data.url
this.title = data.name
}
_filterGroup(groupTitle) {
if(!groupTitle) return ''
const groupIndex = supportedCategories.map(g => g.toLowerCase()).indexOf(groupTitle.toLowerCase())
if(groupIndex === -1) {
groupTitle = ''
} else {
groupTitle = supportedCategories[groupIndex]
}
return groupTitle
}
_filterLanguage(languageName) {
if(ISO6391.getCode(languageName) !== '') {
return languageName
}
return ''
}
toString() {
const info = `-1 tvg-id="${this.id}" tvg-name="${this.name}" tvg-language="${this.language}" tvg-logo="${this.logo}" group-title="${this.group}",${this.title}`
return '#EXTINF:' + info + '\n' + this.url + '\n'
}
helper.escapeStringRegexp = function(scring) {
return escapeStringRegexp(string)
}
function parsePlaylist(filename) {
helper.getISO6391Name = function(code) {
return ISO6391.getName(code)
}
helper.getISO6391Code = function(name) {
return ISO6391.getCode(name)
}
helper.parsePlaylist = function(filename) {
const content = readFile(filename)
const result = parser.parse(content)
return new Playlist(result)
}
function createChannel(data) {
return new Channel(data)
helper.createChannel = function(data) {
return new Channel({
id: data.tvg.id,
name: data.tvg.name,
language: data.tvg.language,
logo: data.tvg.logo,
group: data.group.title,
url: data.url,
title: data.name
})
}
async function loadEPG(url) {
helper.loadEPG = async function(url) {
const content = await getEPGFile(url)
const result = epgParser.parse(content)
const channels = {}
@ -101,7 +61,7 @@ async function loadEPG(url) {
})
}
function getEPGFile(url) {
helper.getEPGFile = function(url) {
return new Promise((resolve, reject) => {
var buffer = []
axios({
@ -131,58 +91,56 @@ function getEPGFile(url) {
})
}
function byTitleAndUrl(a, b) {
var titleA = a.title.toLowerCase()
var titleB = b.title.toLowerCase()
var urlA = a.url.toLowerCase()
var urlB = b.url.toLowerCase()
if(titleA < titleB) return -1
if(titleA > titleB) return 1
helper.sortByTitleAndUrl = function(arr) {
return arr.sort(function(a, b) {
var titleA = a.title.toLowerCase()
var titleB = b.title.toLowerCase()
var urlA = a.url.toLowerCase()
var urlB = b.url.toLowerCase()
if(titleA < titleB) return -1
if(titleA > titleB) return 1
if(urlA < urlB) return -1
if(urlA > urlB) return 1
if(urlA < urlB) return -1
if(urlA > urlB) return 1
return 0
return 0
})
}
function sortByTitleAndUrl(arr) {
return arr.sort(byTitleAndUrl)
}
function readFile(filename) {
helper.readFile = function(filename) {
return fs.readFileSync(path.resolve(__dirname) + `/../${filename}`, { encoding: "utf8" })
}
function appendToFile(filename, data) {
helper.appendToFile = function(filename, data) {
fs.appendFileSync(path.resolve(__dirname) + '/../' + filename, data)
}
function createFile(filename, data) {
helper.createFile = function(filename, data) {
fs.writeFileSync(path.resolve(__dirname) + '/../' + filename, data)
}
function getBasename(filename) {
helper.getBasename = function(filename) {
return path.basename(filename, path.extname(filename))
}
function addToCache(url) {
helper.addToCache = function(url) {
let id = getUrlPath(url)
cache[id] = true
}
function checkCache(url) {
helper.checkCache = function(url) {
let id = getUrlPath(url)
return cache.hasOwnProperty(id)
}
function clearCache() {
helper.clearCache = function() {
cache = {}
}
function getUrlPath(u) {
helper.getUrlPath = function(u) {
let parsed = urlParser.parse(u)
let searchQuery = parsed.search || ''
let path = parsed.host + parsed.pathname + searchQuery
@ -190,14 +148,19 @@ function getUrlPath(u) {
return path.toLowerCase()
}
function validateUrl(channelUrl) {
helper.validateUrl = function(channelUrl) {
const url = new URL(channelUrl)
const host = url.hostname
const blacklist = [
'80.80.160.168', // repeats on a loop
'63.237.48.3', // not a live stream
'189.216.247.113', // not working streams
]
return blacklist.indexOf(host) === -1
}
function skipPlaylist(filename) {
helper.skipPlaylist = function(filename) {
let testCountry = process.env.npm_config_country
let excludeList = process.env.npm_config_exclude
let excludeCountries = excludeList ? excludeList.split(',') : []
@ -211,7 +174,7 @@ function skipPlaylist(filename) {
return false
}
function generateTable(data, options) {
helper.generateTable = function(data, options) {
let output = '<table>'
output += '<thead><tr>'
@ -240,20 +203,64 @@ function generateTable(data, options) {
return output
}
module.exports = {
parsePlaylist,
sortByTitleAndUrl,
appendToFile,
createFile,
readFile,
loadEPG,
createChannel,
getBasename,
addToCache,
checkCache,
clearCache,
validateUrl,
skipPlaylist,
supportedCategories,
generateTable
}
class Playlist {
constructor(data) {
this.header = data.header
this.items = data.items
}
getHeader() {
let parts = ['#EXTM3U']
for(let key in this.header.attrs) {
let value = this.header.attrs[key]
if(value) {
parts.push(`${key}="${value}"`)
}
}
return `${parts.join(' ')}\n`
}
}
class Channel {
constructor(data) {
this.id = data.id
this.name = data.name
this.language = this._filterLanguage(data.language)
this.logo = data.logo
this.group = this._filterGroup(data.group)
this.url = data.url
this.title = data.title
}
_filterGroup(groupTitle) {
if(!groupTitle) return ''
const supportedCategories = [ 'Auto','Business', 'Classic','Comedy','Documentary','Education','Entertainment', 'Family','Fashion','Food', 'General', 'Health', 'History', 'Hobby', 'Kids', 'Legislative','Lifestyle','Local', 'Movies', 'Music', 'News', 'Quiz', 'Religious','Sci-Fi', 'Shop', 'Sport', 'Travel', 'Weather', 'XXX' ]
const groupIndex = supportedCategories.map(g => g.toLowerCase()).indexOf(groupTitle.toLowerCase())
if(groupIndex === -1) {
groupTitle = ''
} else {
groupTitle = supportedCategories[groupIndex]
}
return groupTitle
}
_filterLanguage(languageName) {
if(ISO6391.getCode(languageName) !== '') {
return languageName
}
return ''
}
toString() {
const info = `-1 tvg-id="${this.id}" tvg-name="${this.name}" tvg-language="${this.language}" tvg-logo="${this.logo}" group-title="${this.group}",${this.title}`
return '#EXTINF:' + info + '\n' + this.url + '\n'
}
}
module.exports = helper

29
helpers/update-readme.js → scripts/update-readme.js
View File

@ -1,7 +1,4 @@
const util = require('./util')
const ISO6391 = require('iso-639-1')
const markdownInclude = require('markdown-include')
const path = require('path')
const helper = require('./helper')
let output = {
countries: [],
@ -24,13 +21,13 @@ function main() {
}
function parseIndex() {
const root = util.parsePlaylist('index.m3u')
const root = helper.parsePlaylist('index.m3u')
let languages = {}
let categories = {}
for(let rootItem of root.items) {
const playlist = util.parsePlaylist(rootItem.url)
const countryCode = util.getBasename(rootItem.url).toUpperCase()
const playlist = helper.parsePlaylist(rootItem.url)
const countryCode = helper.getBasename(rootItem.url).toUpperCase()
const epg = playlist.header.attrs['x-tvg-url'] ? `<code>${playlist.header.attrs['x-tvg-url']}</code>` : ''
// country
@ -45,7 +42,7 @@ function parseIndex() {
// language
const languageName = item.tvg.language || 'Undefined'
const languageCode = ISO6391.getCode(languageName) || 'undefined'
const languageCode = helper.getISO6391Code(languageName) || 'undefined'
if(languages[languageCode]) {
languages[languageCode].channels++
} else {
@ -57,7 +54,7 @@ function parseIndex() {
}
// category
const categoryName = util.supportedCategories.find(c => c === item.group.title) || 'Other'
const categoryName = item.group.title || 'Other'
const categoryCode = categoryName.toLowerCase()
if(categories[categoryCode]) {
categories[categoryCode].channels++
@ -76,7 +73,7 @@ function parseIndex() {
}
function generateCountriesTable() {
const table = util.generateTable(output.countries, {
const table = helper.generateTable(output.countries, {
columns: [
{ name: 'Country', align: 'left' },
{ name: 'Channels', align: 'right' },
@ -85,7 +82,7 @@ function generateCountriesTable() {
]
})
util.createFile('./.readme/_countries.md', table)
helper.createFile('./.readme/_countries.md', table)
}
function generateLanguagesTable() {
@ -97,7 +94,7 @@ function generateLanguagesTable() {
return 0
})
const table = util.generateTable(output.languages, {
const table = helper.generateTable(output.languages, {
columns: [
{ name: 'Language', align: 'left' },
{ name: 'Channels', align: 'right' },
@ -105,7 +102,7 @@ function generateLanguagesTable() {
]
})
util.createFile('./.readme/_languages.md', table)
helper.createFile('./.readme/_languages.md', table)
}
function generateCategoriesTable() {
@ -117,7 +114,7 @@ function generateCategoriesTable() {
return 0
})
const table = util.generateTable(output.categories, {
const table = helper.generateTable(output.categories, {
columns: [
{ name: 'Category', align: 'left' },
{ name: 'Channels', align: 'right' },
@ -125,11 +122,11 @@ function generateCategoriesTable() {
]
})
util.createFile('./.readme/_categories.md', table)
helper.createFile('./.readme/_categories.md', table)
}
function generateReadme() {
markdownInclude.compileFiles(path.resolve(__dirname, '../.readme/config.json'))
helper.compileMarkdown('../.readme/config.json')
}
main()

10
test/index.js
View File

@ -1,6 +1,6 @@
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0
const util = require('../helpers/util')
const helper = require('../scripts/helper')
const ffmpeg = require('fluent-ffmpeg')
const verbose = process.env.npm_config_debug || false
@ -19,19 +19,19 @@ async function test() {
stats.tests++
const playlist = util.parsePlaylist('index.m3u')
const playlist = helper.parsePlaylist('index.m3u')
const countries = playlist.items
for(let country of countries) {
if (util.skipPlaylist(country.url)) {
if (helper.skipPlaylist(country.url)) {
continue
}
console.log(`Checking '${country.url}'...`)
const playlist = util.parsePlaylist(country.url)
const playlist = helper.parsePlaylist(country.url)
for(let item of playlist.items) {
@ -90,7 +90,7 @@ test()
function writeToLog(country, msg, url) {
var now = new Date()
var line = `${country}: ${msg} '${url}'`
util.appendToFile(errorLog, now.toISOString() + ' ' + line + '\n')
helper.appendToFile(errorLog, now.toISOString() + ' ' + line + '\n')
console.log(`${msg} '${url}'`)
}