Remove old scripts

This commit is contained in:
Aleksandr Statciuk 2021-12-12 07:10:52 +03:00
parent c6c3154522
commit 958703d6d6
16 changed files with 0 additions and 1525 deletions

View File

@ -1,9 +0,0 @@
const file = require('./helpers/file')
file.list().then(files => {
files = files.filter(file => file !== 'channels/unsorted.m3u')
const country = files.map(file => file.replace(/channels\/|\.m3u/gi, ''))
const matrix = { country }
const output = `::set-output name=matrix::${JSON.stringify(matrix)}`
console.log(output)
})

View File

@ -1,43 +0,0 @@
const blacklist = require('./data/blacklist.json')
const parser = require('./helpers/parser')
const file = require('./helpers/file')
const log = require('./helpers/log')
async function main() {
log.start()
const files = await file.list()
if (!files.length) log.print(`No files is selected\n`)
for (const file of files) {
log.print(`\nProcessing '${file}'...`)
await parser
.parsePlaylist(file)
.then(removeBlacklisted)
.then(p => p.save())
}
log.print('\n')
log.finish()
}
function removeBlacklisted(playlist) {
const channels = playlist.channels.filter(channel => {
return !blacklist.find(item => {
const regexp = new RegExp(item.regex, 'i')
const hasSameName = regexp.test(channel.name)
const fromSameCountry = playlist.country.code === item.country
return hasSameName && fromSameCountry
})
})
if (playlist.channels.length !== channels.length) {
log.print(`updated`)
playlist.channels = channels
playlist.updated = true
}
return playlist
}
main()

View File

@ -1,307 +0,0 @@
const axios = require('axios')
const { program } = require('commander')
const normalize = require('normalize-url')
const IPTVChecker = require('iptv-checker')
const parser = require('./helpers/parser')
const utils = require('./helpers/utils')
const file = require('./helpers/file')
const log = require('./helpers/log')
const epg = require('./helpers/epg')
const ignoreStatus = ['Geo-blocked']
program
.usage('[OPTIONS]...')
.option('--debug', 'Enable debug mode')
.option('--offline', 'Enable offline mode')
.option('-d, --delay <delay>', 'Set delay for each request', parseNumber, 0)
.option('-t, --timeout <timeout>', 'Set timeout for each request', parseNumber, 5000)
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
.parse(process.argv)
const config = program.opts()
const checker = new IPTVChecker({
timeout: config.timeout
})
let buffer, origins
async function main() {
log.start()
const include = config.country.split(',').filter(i => i)
const exclude = config.exclude.split(',').filter(i => i)
let files = await file.list(include, exclude)
if (!files.length) log.print(`No files is selected\n`)
for (const file of files) {
await parser.parsePlaylist(file).then(updatePlaylist).then(savePlaylist)
}
log.finish()
}
function savePlaylist(playlist) {
if (file.read(playlist.url) !== playlist.toString()) {
log.print(`File '${playlist.url}' has been updated\n`)
playlist.updated = true
}
playlist.save()
}
async function updatePlaylist(playlist) {
const total = playlist.channels.length
log.print(`Processing '${playlist.url}'...\n`)
let channels = {}
let codes = {}
if (!config.offline) {
channels = await loadChannelsJson()
codes = await loadCodes()
}
buffer = {}
origins = {}
for (const [i, channel] of playlist.channels.entries()) {
const curr = i + 1
updateTvgName(channel)
updateTvgId(channel, playlist)
updateTvgCountry(channel)
normalizeUrl(channel)
const data = channels[channel.tvg.id]
const epgData = codes[channel.tvg.id]
updateLogo(channel, data, epgData)
updateGroupTitle(channel, data)
updateTvgLanguage(channel, data)
if (config.offline || ignoreStatus.includes(channel.status)) {
continue
}
await checker
.checkStream(channel.data)
.then(parseResult)
.then(result => {
updateStatus(channel, result.status)
if (result.status === 'online') {
buffer[i] = result
updateOrigins(channel, result.requests)
updateResolution(channel, result.resolution)
} else {
buffer[i] = null
if (config.debug) {
log.print(` INFO: ${channel.url} (${result.error})\n`)
}
}
})
.catch(err => {
buffer[i] = null
if (config.debug) {
log.print(` ERR: ${channel.data.url} (${err.message})\n`)
}
})
}
for (const [i, channel] of playlist.channels.entries()) {
if (!buffer[i]) continue
const { requests } = buffer[i]
updateUrl(channel, requests)
}
return playlist
}
function updateOrigins(channel, requests) {
if (!requests) return
const origin = new URL(channel.url)
const target = new URL(requests[0])
const type = origin.host === target.host ? 'origin' : 'redirect'
requests.forEach(url => {
const key = utils.removeProtocol(url)
if (!origins[key] && type === 'origin') {
origins[key] = channel.url
}
})
}
function updateStatus(channel, status) {
switch (status) {
case 'online':
if (channel.status !== 'Not 24/7')
channel.status = channel.status === 'Offline' ? 'Not 24/7' : null
break
case 'error_403':
if (!channel.status) channel.status = 'Geo-blocked'
break
case 'offline':
if (channel.status !== 'Not 24/7') channel.status = 'Offline'
break
}
}
function updateResolution(channel, resolution) {
if (!channel.resolution.height && resolution) {
channel.resolution = resolution
}
}
function updateUrl(channel, requests) {
for (const request of requests) {
let key = utils.removeProtocol(channel.url)
if (origins[key]) {
channel.updateUrl(origins[key])
break
}
key = utils.removeProtocol(request)
if (origins[key]) {
channel.updateUrl(origins[key])
break
}
}
}
function parseResult(result) {
return {
status: parseStatus(result.status),
resolution: result.status.ok ? parseResolution(result.status.metadata.streams) : null,
requests: result.status.ok ? parseRequests(result.status.metadata.requests) : [],
error: !result.status.ok ? result.status.reason : null
}
}
function parseStatus(status) {
if (status.ok) {
return 'online'
} else if (status.reason.includes('timed out')) {
return 'timeout'
} else if (status.reason.includes('403')) {
return 'error_403'
} else if (status.reason.includes('not one of 40{0,1,3,4}')) {
return 'error_40x' // 402, 451
} else {
return 'offline'
}
}
function parseResolution(streams) {
const resolution = streams
.filter(stream => stream.codec_type === 'video')
.reduce(
(acc, curr) => {
if (curr.height > acc.height) return { width: curr.width, height: curr.height }
return acc
},
{ width: 0, height: 0 }
)
return resolution.width > 0 && resolution.height > 0 ? resolution : null
}
function parseRequests(requests) {
requests = requests.map(r => r.url)
requests.shift()
return requests
}
function updateTvgName(channel) {
if (!channel.tvg.name) {
channel.tvg.name = channel.name.replace(/\"/gi, '')
}
}
function updateTvgId(channel, playlist) {
const code = playlist.country.code
if (!channel.tvg.id && channel.tvg.name) {
const id = utils.name2id(channel.tvg.name)
channel.tvg.id = id ? `${id}.${code}` : ''
}
}
function updateTvgCountry(channel) {
if (!channel.countries.length && channel.tvg.id) {
const code = channel.tvg.id.split('.')[1] || null
const name = utils.code2name(code)
channel.countries = name ? [{ code, name }] : []
channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';')
}
}
function updateLogo(channel, data, epgData) {
if (!channel.logo) {
if (data && data.logo) {
channel.logo = data.logo
} else if (epgData && epgData.logo) {
channel.logo = epgData.logo
}
}
}
function updateTvgLanguage(channel, data) {
if (!channel.tvg.language) {
if (data && data.languages.length) {
channel.tvg.language = data.languages.map(l => l.name).join(';')
} else if (channel.countries.length) {
const countryCode = channel.countries[0].code
channel.tvg.language = utils.country2language(countryCode)
}
}
}
function updateGroupTitle(channel, data) {
if (!channel.group.title) {
if (channel.category) {
channel.group.title = channel.category
} else if (data && data.category) {
channel.group.title = data.category
}
}
}
function normalizeUrl(channel) {
const normalized = normalize(channel.url, { stripWWW: false })
const decoded = decodeURIComponent(normalized).replace(/\s/g, '+')
channel.updateUrl(decoded)
}
function parseNumber(str) {
return parseInt(str)
}
function loadCodes() {
return epg.codes
.load()
.then(codes => {
let output = {}
codes.forEach(item => {
output[item['tvg_id']] = item
})
return output
})
.catch(console.log)
}
function loadChannelsJson() {
return axios
.get('https://iptv-org.github.io/iptv/channels.json')
.then(r => r.data)
.then(channels => {
let output = {}
channels.forEach(channel => {
const item = output[channel.tvg.id]
if (!item) {
output[channel.tvg.id] = channel
} else {
item.logo = item.logo || channel.logo
item.languages = item.languages.length ? item.languages : channel.languages
item.category = item.category || channel.category
}
})
return output
})
.catch(console.log)
}
main()

View File

@ -1,232 +0,0 @@
const file = require('./helpers/file')
const log = require('./helpers/log')
const db = require('./helpers/db')
const ROOT_DIR = './.gh-pages'
async function main() {
await loadDatabase()
createRootDirectory()
createNoJekyllFile()
generateIndex()
generateCategoryIndex()
generateCountryIndex()
generateLanguageIndex()
generateCategories()
generateCountries()
generateLanguages()
generateChannelsJson()
showResults()
}
async function loadDatabase() {
log.print('Loading database...\n')
await db.load()
}
function createRootDirectory() {
log.print('Creating .gh-pages folder...\n')
file.createDir(ROOT_DIR)
}
function createNoJekyllFile() {
log.print('Creating .nojekyll...\n')
file.create(`${ROOT_DIR}/.nojekyll`)
}
function generateIndex() {
log.print('Generating index.m3u...\n')
const channels = db.channels
.sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc'])
.removeDuplicates()
.removeOffline()
.get()
const guides = channels.map(channel => channel.tvg.url)
const filename = `${ROOT_DIR}/index.m3u`
const urlTvg = generateUrlTvg(guides)
file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n`)
const nsfwFilename = `${ROOT_DIR}/index.nsfw.m3u`
file.create(nsfwFilename, `#EXTM3U url-tvg="${urlTvg}"\n`)
for (const channel of channels) {
if (!channel.isNSFW()) {
file.append(filename, channel.toString())
}
file.append(nsfwFilename, channel.toString())
}
}
function generateCategoryIndex() {
log.print('Generating index.category.m3u...\n')
const channels = db.channels
.sortBy(
['category', 'name', 'status', 'resolution.height', 'url'],
['asc', 'asc', 'asc', 'desc', 'asc']
)
.removeDuplicates()
.removeOffline()
.get()
const guides = channels.map(channel => channel.tvg.url)
const filename = `${ROOT_DIR}/index.category.m3u`
const urlTvg = generateUrlTvg(guides)
file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n`)
for (const channel of channels) {
file.append(filename, channel.toString())
}
}
function generateCountryIndex() {
log.print('Generating index.country.m3u...\n')
const guides = []
const lines = []
for (const country of [{ code: 'undefined' }, ...db.countries.sortBy(['name']).all()]) {
const channels = db.channels
.sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc'])
.forCountry(country)
.removeDuplicates()
.removeNSFW()
.removeOffline()
.get()
for (const channel of channels) {
const groupTitle = channel.group.title
channel.group.title = country.name || ''
lines.push(channel.toString())
channel.group.title = groupTitle
guides.push(channel.tvg.url)
}
}
const filename = `${ROOT_DIR}/index.country.m3u`
const urlTvg = generateUrlTvg(guides)
file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n${lines.join('')}`)
}
function generateLanguageIndex() {
log.print('Generating index.language.m3u...\n')
const guides = []
const lines = []
for (const language of [{ code: 'undefined' }, ...db.languages.sortBy(['name']).all()]) {
const channels = db.channels
.sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc'])
.forLanguage(language)
.removeDuplicates()
.removeNSFW()
.removeOffline()
.get()
for (const channel of channels) {
const groupTitle = channel.group.title
channel.group.title = language.name || ''
lines.push(channel.toString())
channel.group.title = groupTitle
guides.push(channel.tvg.url)
}
}
const filename = `${ROOT_DIR}/index.language.m3u`
const urlTvg = generateUrlTvg(guides)
file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n${lines.join('')}`)
}
function generateCategories() {
log.print(`Generating /categories...\n`)
const outputDir = `${ROOT_DIR}/categories`
file.createDir(outputDir)
for (const category of [...db.categories.all(), { id: 'other' }]) {
const channels = db.channels
.sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc'])
.forCategory(category)
.removeDuplicates()
.removeOffline()
.get()
const guides = channels.map(channel => channel.tvg.url)
const filename = `${outputDir}/${category.id}.m3u`
const urlTvg = generateUrlTvg(guides)
file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n`)
for (const channel of channels) {
file.append(filename, channel.toString())
}
}
}
function generateCountries() {
log.print(`Generating /countries...\n`)
const outputDir = `${ROOT_DIR}/countries`
file.createDir(outputDir)
for (const country of [...db.countries.all(), { code: 'undefined' }]) {
const channels = db.channels
.sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc'])
.forCountry(country)
.removeDuplicates()
.removeOffline()
.removeNSFW()
.get()
const guides = channels.map(channel => channel.tvg.url)
const filename = `${outputDir}/${country.code}.m3u`
const urlTvg = generateUrlTvg(guides)
file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n`)
for (const channel of channels) {
file.append(filename, channel.toString())
}
}
}
function generateLanguages() {
log.print(`Generating /languages...\n`)
const outputDir = `${ROOT_DIR}/languages`
file.createDir(outputDir)
for (const language of [...db.languages.all(), { code: 'undefined' }]) {
const channels = db.channels
.sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc'])
.forLanguage(language)
.removeDuplicates()
.removeOffline()
.removeNSFW()
.get()
const guides = channels.map(channel => channel.tvg.url)
const filename = `${outputDir}/${language.code}.m3u`
const urlTvg = generateUrlTvg(guides)
file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n`)
for (const channel of channels) {
file.append(filename, channel.toString())
}
}
}
function generateChannelsJson() {
log.print('Generating channels.json...\n')
const filename = `${ROOT_DIR}/channels.json`
const channels = db.channels
.sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc'])
.get()
.map(c => c.toObject())
file.create(filename, JSON.stringify(channels))
}
function showResults() {
log.print(
`Total: ${db.channels.count()} channels, ${db.countries.count()} countries, ${db.languages.count()} languages, ${db.categories.count()} categories.\n`
)
}
function generateUrlTvg(guides) {
const output = guides.reduce((acc, curr) => {
if (curr && !acc.includes(curr)) acc.push(curr)
return acc
}, [])
return output.sort().join(',')
}
main()

View File

@ -1,162 +0,0 @@
const categories = require('../data/categories')
const utils = require('./utils')
const file = require('./file')
const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name)
const nsfwCategories = categories.filter(c => c.nsfw).map(c => c.name)
module.exports = class Channel {
constructor(data) {
this.data = data
this.raw = data.raw
this.tvg = data.tvg
this.http = data.http
this.url = data.url
this.logo = data.tvg.logo
this.group = data.group
this.name = this.parseName(data.name)
this.status = this.parseStatus(data.name)
this.resolution = this.parseResolution(data.name)
this.category = this.parseCategory(data.group.title)
this.countries = this.parseCountries(data.tvg.country)
this.languages = this.parseLanguages(data.tvg.language)
this.hash = this.generateHash()
}
generateHash() {
return `${this.tvg.id}:${this.tvg.country}:${this.tvg.language}:${this.logo}:${this.group.title}:${this.name}`.toLowerCase()
}
updateUrl(url) {
this.url = url
this.data.url = url
}
parseName(title) {
return title
.trim()
.split(' ')
.map(s => s.trim())
.filter(s => {
return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s)
})
.join(' ')
}
parseStatus(title) {
const match = title.match(/\[(.*)\]/i)
return match ? match[1] : null
}
parseResolution(title) {
const match = title.match(/\((\d+)P\)/i)
const height = match ? parseInt(match[1]) : null
return { width: null, height }
}
parseCategory(string) {
const category = categories.find(c => c.id === string.toLowerCase())
if (!category) return ''
return category.name
}
parseCountries(string) {
const list = string.split(';')
return list
.reduce((acc, curr) => {
const codes = utils.region2codes(curr)
if (codes.length) {
for (let code of codes) {
if (!acc.includes(code)) {
acc.push(code)
}
}
} else {
acc.push(curr)
}
return acc
}, [])
.map(code => {
const name = code ? utils.code2name(code) : null
if (!name) return null
return { code: code.toLowerCase(), name }
})
.filter(c => c)
}
parseLanguages(string) {
const list = string.split(';')
return list
.map(name => {
const code = name ? utils.language2code(name) : null
if (!code) return null
return { code, name }
})
.filter(l => l)
}
isSFW() {
return sfwCategories.includes(this.category)
}
isNSFW() {
return nsfwCategories.includes(this.category)
}
getInfo() {
let info = `-1 tvg-id="${this.tvg.id}" tvg-country="${this.tvg.country || ''}" tvg-language="${
this.tvg.language || ''
}" tvg-logo="${this.logo || ''}"`
if (this.http['user-agent']) {
info += ` user-agent="${this.http['user-agent']}"`
}
info += ` group-title="${this.group.title || ''}",${this.name}`
if (this.resolution.height) {
info += ` (${this.resolution.height}p)`
}
if (this.status) {
info += ` [${this.status}]`
}
if (this.http['referrer']) {
info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}`
}
if (this.http['user-agent']) {
info += `\n#EXTVLCOPT:http-user-agent=${this.http['user-agent']}`
}
return info
}
toString(raw = false) {
if (raw) return this.raw + '\n'
return '#EXTINF:' + this.getInfo() + '\n' + this.url + '\n'
}
toObject() {
return {
name: this.name,
logo: this.logo || null,
url: this.url,
category: this.category || null,
languages: this.languages,
countries: this.countries,
tvg: {
id: this.tvg.id || null,
name: this.tvg.name || this.name.replace(/\"/gi, ''),
url: this.tvg.url || null
}
}
}
}

View File

@ -1,41 +0,0 @@
const Channel = require('./Channel')
const file = require('./file')
module.exports = class Playlist {
constructor({ header, items, url, name, country }) {
this.url = url
this.name = name
this.country = country
this.header = header
this.channels = items.map(item => new Channel(item)).filter(channel => channel.url)
this.updated = false
}
getHeader() {
let header = ['#EXTM3U']
for (let key in this.header.attrs) {
let value = this.header.attrs[key]
if (value) {
header.push(`${key}="${value}"`)
}
}
return header.join(' ')
}
toString(options = {}) {
const config = { raw: false, ...options }
let output = `${this.getHeader()}\n`
for (let channel of this.channels) {
output += channel.toString(config.raw)
}
return output
}
save() {
if (this.updated) {
file.create(this.url, this.toString())
}
}
}

View File

@ -1,237 +0,0 @@
const categories = require('../data/categories')
const parser = require('./parser')
const utils = require('./utils')
const file = require('./file')
const epg = require('./epg')
const db = {}
db.load = async function () {
const files = await file.list()
const codes = await epg.codes.load()
for (const file of files) {
const playlist = await parser.parsePlaylist(file)
for (const channel of playlist.channels) {
const code = codes.find(ch => ch['tvg_id'] === channel.tvg.id)
if (code && Array.isArray(code.guides) && code.guides.length) {
channel.tvg.url = code.guides[0]
}
db.channels.add(channel)
for (const country of channel.countries) {
if (!db.countries.has(country)) {
db.countries.add(country)
}
}
for (const language of channel.languages) {
if (!db.languages.has(language)) {
db.languages.add(language)
}
}
}
db.playlists.add(playlist)
}
}
db.channels = {
list: [],
filter: null,
duplicates: true,
offline: true,
nsfw: true,
add(channel) {
this.list.push(channel)
},
get() {
let output
if (this.filter) {
switch (this.filter.field) {
case 'countries':
if (this.filter.value === 'undefined') {
output = this.list.filter(channel => !channel.countries.length)
} else {
output = this.list.filter(channel =>
channel.countries.map(c => c.code).includes(this.filter.value)
)
}
break
case 'languages':
if (this.filter.value === 'undefined') {
output = this.list.filter(channel => !channel.languages.length)
} else {
output = this.list.filter(channel =>
channel.languages.map(c => c.code).includes(this.filter.value)
)
}
break
case 'category':
if (this.filter.value === 'other') {
output = this.list.filter(channel => !channel.category)
} else {
output = this.list.filter(
channel => channel.category.toLowerCase() === this.filter.value
)
}
break
}
} else {
output = this.list
}
if (!this.duplicates) {
const buffer = []
output = output.filter(channel => {
if (buffer.includes(channel.hash)) return false
buffer.push(channel.hash)
return true
})
}
if (!this.nsfw) {
output = output.filter(channel => !channel.isNSFW())
}
if (!this.offline) {
output = output.filter(channel => channel.status !== 'Offline')
}
this.nsfw = true
this.duplicates = true
this.offline = true
this.filter = null
return output
},
removeDuplicates() {
this.duplicates = false
return this
},
removeNSFW() {
this.nsfw = false
return this
},
removeOffline() {
this.offline = false
return this
},
all() {
return this.list
},
forCountry(country) {
this.filter = {
field: 'countries',
value: country.code
}
return this
},
forLanguage(language) {
this.filter = {
field: 'languages',
value: language.code
}
return this
},
forCategory(category) {
this.filter = {
field: 'category',
value: category.id
}
return this
},
count() {
return this.get().length
},
sortBy(fields, order) {
this.list = utils.sortBy(this.list, fields, order)
return this
}
}
db.countries = {
list: [],
has(country) {
return this.list.map(c => c.code).includes(country.code)
},
add(country) {
this.list.push(country)
},
all() {
return this.list
},
count() {
return this.list.length
},
sortBy(fields, order) {
this.list = utils.sortBy(this.list, fields, order)
return this
}
}
db.languages = {
list: [],
has(language) {
return this.list.map(c => c.code).includes(language.code)
},
add(language) {
this.list.push(language)
},
all() {
return this.list
},
count() {
return this.list.length
},
sortBy(fields, order) {
this.list = utils.sortBy(this.list, fields, order)
return this
}
}
db.categories = {
list: categories,
all() {
return this.list
},
count() {
return this.list.length
}
}
db.playlists = {
list: [],
add(playlist) {
this.list.push(playlist)
},
all() {
return this.list
},
only(list = []) {
return this.list.filter(playlist => list.includes(playlist.filename))
},
except(list = []) {
return this.list.filter(playlist => !list.includes(playlist.filename))
},
sortBy(fields, order) {
this.list = utils.sortBy(this.list, fields, order)
return this
},
count() {
return this.list.length
}
}
module.exports = db

View File

@ -1,12 +0,0 @@
const axios = require('axios')
module.exports = {
codes: {
async load() {
return await axios
.get('https://iptv-org.github.io/epg/codes.json')
.then(r => r.data)
.catch(console.log)
}
}
}

View File

@ -1,53 +0,0 @@
const markdownInclude = require('markdown-include')
const path = require('path')
const glob = require('glob')
const fs = require('fs')
const rootPath = path.resolve(__dirname) + '/../../'
const file = {}
file.list = function (include = [], exclude = []) {
return new Promise(resolve => {
glob('channels/**/*.m3u', function (err, files) {
if (include.length) {
include = include.map(filename => `channels/${filename}.m3u`)
files = files.filter(filename => include.includes(filename))
}
if (exclude.length) {
exclude = exclude.map(filename => `channels/${filename}.m3u`)
files = files.filter(filename => !exclude.includes(filename))
}
resolve(files)
})
})
}
file.getFilename = function (filename) {
return path.parse(filename).name
}
file.createDir = function (dir) {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir)
}
}
file.read = function (filename) {
return fs.readFileSync(rootPath + filename, { encoding: 'utf8' })
}
file.append = function (filename, data) {
fs.appendFileSync(rootPath + filename, data)
}
file.create = function (filename, data = '') {
fs.writeFileSync(rootPath + filename, data)
}
file.compileMarkdown = function (filename) {
markdownInclude.compileFiles(rootPath + filename)
}
module.exports = file

View File

@ -1,17 +0,0 @@
const log = {}
log.print = function (message) {
if (typeof message === 'object') message = JSON.stringify(message, null, 2)
process.stdout.write(message)
}
log.start = function () {
this.print('Starting...\n')
console.time('Done in')
}
log.finish = function () {
console.timeEnd('Done in')
}
module.exports = log

View File

@ -1,20 +0,0 @@
const playlistParser = require('iptv-playlist-parser')
const Playlist = require('./Playlist')
const utils = require('./utils')
const file = require('./file')
const parser = {}
parser.parsePlaylist = async function (url) {
const content = file.read(url)
const result = playlistParser.parse(content)
const filename = file.getFilename(url)
const country = {
code: filename,
name: utils.code2name(filename)
}
return new Playlist({ header: result.header, items: result.items, url, filename, country })
}
module.exports = parser

View File

@ -1,86 +0,0 @@
const { orderBy } = require('natural-orderby')
const transliteration = require('transliteration')
const countries = require('../data/countries')
const categories = require('../data/categories')
const languages = require('../data/languages')
const regions = require('../data/regions')
const utils = {}
const intlDisplayNames = new Intl.DisplayNames(['en'], {
style: 'narrow',
type: 'region'
})
utils.name2id = function (name) {
return transliteration
.transliterate(name)
.replace(/\+/gi, 'Plus')
.replace(/[^a-z\d]+/gi, '')
}
utils.code2flag = function (code) {
code = code.toUpperCase()
switch (code) {
case 'UK':
return '🇬🇧'
case 'INT':
return '🌍'
case 'UNDEFINED':
return ''
default:
return code.replace(/./g, char => String.fromCodePoint(char.charCodeAt(0) + 127397))
}
}
utils.region2codes = function (region) {
region = region.toUpperCase()
return regions[region] ? regions[region].codes : []
}
utils.code2name = function (code) {
try {
code = code.toUpperCase()
if (regions[code]) return regions[code].name
if (code === 'US') return 'United States'
if (code === 'INT') return 'International'
return intlDisplayNames.of(code)
} catch (e) {
return null
}
}
utils.language2code = function (name) {
const lang = languages.find(l => l.name === name)
return lang && lang.code ? lang.code : null
}
utils.country2language = function (code) {
const country = countries[code.toUpperCase()]
if (!country.languages.length) return ''
const language = languages.find(l => l.code === country.languages[0])
return language ? language.name : ''
}
utils.sortBy = function (arr, fields, order = null) {
fields = fields.map(field => {
if (field === 'resolution.height') return channel => channel.resolution.height || 0
if (field === 'status') return channel => channel.status || ''
return channel => channel[field]
})
return orderBy(arr, fields, order)
}
utils.removeProtocol = function (string) {
return string.replace(/(^\w+:|^)\/\//, '')
}
utils.sleep = function (ms) {
return function (x) {
return new Promise(resolve => setTimeout(() => resolve(x), ms))
}
}
module.exports = utils

View File

@ -1,42 +0,0 @@
const parser = require('./helpers/parser')
const file = require('./helpers/file')
const log = require('./helpers/log')
async function main() {
log.start()
let files = await file.list()
if (!files.length) log.print(`No files is selected\n`)
files = files.filter(file => file !== 'channels/unsorted.m3u')
for (const file of files) {
log.print(`\nProcessing '${file}'...`)
await parser
.parsePlaylist(file)
.then(removeBrokenLinks)
.then(p => p.save())
}
log.print('\n')
log.finish()
}
async function removeBrokenLinks(playlist) {
const buffer = []
const channels = playlist.channels.filter(channel => {
const sameHash = buffer.find(item => item.hash === channel.hash)
if (sameHash && channel.status === 'Offline') return false
buffer.push(channel)
return true
})
if (playlist.channels.length !== channels.length) {
log.print('updated')
playlist.channels = channels
playlist.updated = true
}
return playlist
}
main()

View File

@ -1,81 +0,0 @@
const parser = require('./helpers/parser')
const utils = require('./helpers/utils')
const file = require('./helpers/file')
const log = require('./helpers/log')
let globalBuffer = []
async function main() {
log.start()
let files = await file.list()
if (!files.length) log.print(`No files is selected\n`)
files = files.filter(file => file !== 'channels/unsorted.m3u')
for (const file of files) {
log.print(`\nProcessing '${file}'...`)
await parser
.parsePlaylist(file)
.then(addToGlobalBuffer)
.then(removeDuplicates)
.then(p => p.save())
}
if (files.length) {
log.print(`\nProcessing 'channels/unsorted.m3u'...`)
await parser
.parsePlaylist('channels/unsorted.m3u')
.then(removeDuplicates)
.then(removeGlobalDuplicates)
.then(p => p.save())
}
log.print('\n')
log.finish()
}
async function addToGlobalBuffer(playlist) {
playlist.channels.forEach(channel => {
const url = utils.removeProtocol(channel.url)
globalBuffer.push(url)
})
return playlist
}
async function removeDuplicates(playlist) {
const buffer = []
const channels = playlist.channels.filter(channel => {
const sameUrl = buffer.find(item => {
return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url)
})
if (sameUrl) return false
buffer.push(channel)
return true
})
if (playlist.channels.length !== channels.length) {
log.print('updated')
playlist.channels = channels
playlist.updated = true
}
return playlist
}
async function removeGlobalDuplicates(playlist) {
const channels = playlist.channels.filter(channel => {
const url = utils.removeProtocol(channel.url)
return !globalBuffer.includes(url)
})
if (channels.length !== playlist.channels.length) {
log.print('updated')
playlist.channels = channels
playlist.updated = true
}
return playlist
}
main()

View File

@ -1,41 +0,0 @@
const parser = require('./helpers/parser')
const utils = require('./helpers/utils')
const file = require('./helpers/file')
const log = require('./helpers/log')
async function main() {
log.start()
let files = await file.list()
if (!files.length) log.print(`No files is selected\n`)
files = files.filter(file => file !== 'channels/unsorted.m3u')
for (const file of files) {
log.print(`\nProcessing '${file}'...`)
await parser
.parsePlaylist(file)
.then(sortChannels)
.then(p => p.save())
}
log.print('\n')
log.finish()
}
async function sortChannels(playlist) {
let channels = [...playlist.channels]
channels = utils.sortBy(
channels,
['name', 'status', 'resolution.height', 'url'],
['asc', 'asc', 'desc', 'asc']
)
if (JSON.stringify(channels) !== JSON.stringify(playlist.channels)) {
log.print('updated')
playlist.channels = channels
playlist.updated = true
}
return playlist
}
main()

View File

@ -1,142 +0,0 @@
const utils = require('./helpers/utils')
const file = require('./helpers/file')
const log = require('./helpers/log')
const db = require('./helpers/db')
async function main() {
log.start()
await loadDatabase()
generateCategoriesTable()
generateCountriesTable()
generateLanguagesTable()
generateReadme()
log.finish()
}
async function loadDatabase() {
log.print('Loading database...\n')
await db.load()
}
function generateCategoriesTable() {
log.print('Generating categories table...\n')
const categories = []
for (const category of [...db.categories.all(), { name: 'Other', id: 'other' }]) {
categories.push({
category: category.name,
channels: db.channels.forCategory(category).removeOffline().removeDuplicates().count(),
playlist: `<code>https://iptv-org.github.io/iptv/categories/${category.id}.m3u</code>`
})
}
const table = generateTable(categories, {
columns: [
{ name: 'Category', align: 'left' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', align: 'left' }
]
})
file.create('./.readme/_categories.md', table)
}
function generateCountriesTable() {
log.print('Generating countries table...\n')
const countries = []
for (const country of [
...db.countries.sortBy(['name']).all(),
{ name: 'Undefined', code: 'undefined' }
]) {
let flag = utils.code2flag(country.code)
const prefix = flag ? `${flag}&nbsp;` : ''
countries.push({
country: prefix + country.name,
channels: db.channels
.forCountry(country)
.removeOffline()
.removeDuplicates()
.removeNSFW()
.count(),
playlist: `<code>https://iptv-org.github.io/iptv/countries/${country.code}.m3u</code>`
})
}
const table = generateTable(countries, {
columns: [
{ name: 'Country', align: 'left' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', align: 'left', nowrap: true }
]
})
file.create('./.readme/_countries.md', table)
}
function generateLanguagesTable() {
log.print('Generating languages table...\n')
const languages = []
for (const language of [
...db.languages.sortBy(['name']).all(),
{ name: 'Undefined', code: 'undefined' }
]) {
languages.push({
language: language.name,
channels: db.channels
.forLanguage(language)
.removeOffline()
.removeDuplicates()
.removeNSFW()
.count(),
playlist: `<code>https://iptv-org.github.io/iptv/languages/${language.code}.m3u</code>`
})
}
const table = generateTable(languages, {
columns: [
{ name: 'Language', align: 'left' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', align: 'left' }
]
})
file.create('./.readme/_languages.md', table)
}
function generateTable(data, options) {
let output = '<table>\n'
output += '\t<thead>\n\t\t<tr>'
for (let column of options.columns) {
output += `<th align="${column.align}">${column.name}</th>`
}
output += '</tr>\n\t</thead>\n'
output += '\t<tbody>\n'
for (let item of data) {
output += '\t\t<tr>'
let i = 0
for (let prop in item) {
const column = options.columns[i]
let nowrap = column.nowrap
let align = column.align
output += `<td align="${align}"${nowrap ? ' nowrap' : ''}>${item[prop]}</td>`
i++
}
output += '</tr>\n'
}
output += '\t</tbody>\n'
output += '</table>'
return output
}
function generateReadme() {
log.print('Generating README.md...\n')
file.compileMarkdown('.readme/config.json')
}
main()