Create scripts/core

This commit is contained in:
Aleksandr Statciuk 2021-12-12 07:10:18 +03:00
parent 213da67762
commit c6c3154522
11 changed files with 517 additions and 0 deletions

19
scripts/core/checker.js Normal file
View File

@ -0,0 +1,19 @@
const IPTVChecker = require('iptv-checker')
const checker = {}
checker.check = async function (item, config) {
const ic = new IPTVChecker(config)
const result = await ic.checkStream({ url: item.url, http: item.http })
return {
_id: item._id,
url: item.url,
http: item.http,
error: !result.status.ok ? result.status.reason : null,
streams: result.status.ok ? result.status.metadata.streams : [],
requests: result.status.ok ? result.status.metadata.requests : []
}
}
module.exports = checker

61
scripts/core/db.js Normal file
View File

@ -0,0 +1,61 @@
const Database = require('nedb-promises')
const file = require('./file')
const DB_FILEPATH = process.env.DB_FILEPATH || './scripts/channels.db'
const nedb = Database.create({
filename: file.resolve(DB_FILEPATH),
autoload: true,
onload(err) {
if (err) console.error(err)
},
compareStrings: (a, b) => {
a = a.replace(/\s/g, '_')
b = b.replace(/\s/g, '_')
return a.localeCompare(b, undefined, {
sensitivity: 'accent',
numeric: true
})
}
})
const db = {}
db.removeIndex = function (field) {
return nedb.removeIndex(field)
}
db.addIndex = function (options) {
return nedb.ensureIndex(options)
}
db.compact = function () {
return nedb.persistence.compactDatafile()
}
db.reset = function () {
return file.clear(DB_FILEPATH)
}
db.count = function (query) {
return nedb.count(query)
}
db.insert = function (doc) {
return nedb.insert(doc)
}
db.update = function (query, update) {
return nedb.update(query, update)
}
db.find = function (query) {
return nedb.find(query)
}
db.remove = function (query, options) {
return nedb.remove(query, options)
}
module.exports = db

67
scripts/core/file.js Normal file
View File

@ -0,0 +1,67 @@
const path = require('path')
const glob = require('glob')
const fs = require('mz/fs')
const file = {}
file.list = function (pattern) {
return new Promise(resolve => {
glob(pattern, function (err, files) {
resolve(files)
})
})
}
file.getFilename = function (filepath) {
return path.parse(filepath).name
}
file.createDir = async function (dir) {
if (await file.exists(dir)) return
return fs.mkdir(dir, { recursive: true }).catch(console.error)
}
file.exists = function (filepath) {
return fs.exists(path.resolve(filepath))
}
file.read = function (filepath) {
return fs.readFile(path.resolve(filepath), { encoding: 'utf8' }).catch(console.error)
}
file.append = function (filepath, data) {
return fs.appendFile(path.resolve(filepath), data).catch(console.error)
}
file.create = function (filepath, data = '') {
filepath = path.resolve(filepath)
const dir = path.dirname(filepath)
return file
.createDir(dir)
.then(() => fs.writeFile(filepath, data, { encoding: 'utf8', flag: 'w' }))
.catch(console.error)
}
file.write = function (filepath, data = '') {
return fs.writeFile(path.resolve(filepath), data).catch(console.error)
}
file.clear = function (filepath) {
return file.write(filepath, '')
}
file.resolve = function (filepath) {
return path.resolve(filepath)
}
file.dirname = function (filepath) {
return path.dirname(filepath)
}
file.basename = function (filepath) {
return path.basename(filepath)
}
module.exports = file

114
scripts/core/generator.js Normal file
View File

@ -0,0 +1,114 @@
const { create: createPlaylist } = require('./playlist')
const store = require('./store')
const file = require('./file')
const logger = require('./logger')
const db = require('./db')
const _ = require('lodash')
const generator = {}
generator.generate = async function (filepath, query = {}, options = {}) {
options = {
...{
format: 'm3u',
saveEmpty: false,
includeNSFW: false,
includeGuides: true,
includeBroken: false,
onLoad: r => r,
uniqBy: item => item.id || _.uniqueId(),
sortBy: null
},
...options
}
query['is_nsfw'] = options.includeNSFW ? { $in: [true, false] } : false
query['is_broken'] = options.includeBroken ? { $in: [true, false] } : false
let items = await db
.find(query)
.sort({ name: 1, 'status.level': 1, 'resolution.height': -1, url: 1 })
items = _.uniqBy(items, 'url')
if (!options.saveEmpty && !items.length) return { filepath, query, options, count: 0 }
if (options.uniqBy) items = _.uniqBy(items, options.uniqBy)
items = options.onLoad(items)
if (options.sortBy) items = _.sortBy(items, options.sortBy)
switch (options.format) {
case 'json':
await saveAsJSON(filepath, items, options)
break
case 'm3u':
default:
await saveAsM3U(filepath, items, options)
break
}
return { filepath, query, options, count: items.length }
}
async function saveAsM3U(filepath, items, options) {
const playlist = await createPlaylist(filepath)
const header = {}
if (options.includeGuides) {
let guides = items.map(item => item.guides)
guides = _.uniq(_.flatten(guides)).sort().join(',')
header['x-tvg-url'] = guides
}
await playlist.header(header)
for (const item of items) {
const stream = store.create(item)
await playlist.link(
stream.get('url'),
stream.get('title'),
{
'tvg-id': stream.get('tvg_id'),
'tvg-country': stream.get('tvg_country'),
'tvg-language': stream.get('tvg_language'),
'tvg-logo': stream.get('tvg_logo'),
// 'tvg-url': stream.get('tvg_url') || undefined,
'user-agent': stream.get('http.user-agent') || undefined,
'group-title': stream.get('group_title')
},
{
'http-referrer': stream.get('http.referrer') || undefined,
'http-user-agent': stream.get('http.user-agent') || undefined
}
)
}
}
async function saveAsJSON(filepath, items, options) {
const output = items.map(item => {
const stream = store.create(item)
const categories = stream.get('categories').map(c => ({ name: c.name, slug: c.slug }))
const countries = stream.get('countries').map(c => ({ name: c.name, code: c.code }))
return {
name: stream.get('name'),
logo: stream.get('logo'),
url: stream.get('url'),
categories,
countries,
languages: stream.get('languages'),
tvg: {
id: stream.get('tvg_id'),
name: stream.get('name'),
url: stream.get('tvg_url')
}
}
})
await file.create(filepath, JSON.stringify(output))
}
generator.saveAsM3U = saveAsM3U
generator.saveAsJSON = saveAsJSON
module.exports = generator

10
scripts/core/index.js Normal file
View File

@ -0,0 +1,10 @@
exports.db = require('./db')
exports.logger = require('./logger')
exports.file = require('./file')
exports.timer = require('./timer')
exports.parser = require('./parser')
exports.checker = require('./checker')
exports.generator = require('./generator')
exports.playlist = require('./playlist')
exports.store = require('./store')
exports.markdown = require('./markdown')

42
scripts/core/logger.js Normal file
View File

@ -0,0 +1,42 @@
const { createLogger, format, transports, addColors } = require('winston')
const { combine, timestamp, printf } = format
const consoleFormat = ({ level, message, timestamp }) => {
if (typeof message === 'object') return JSON.stringify(message)
return message
}
const config = {
levels: {
error: 0,
warn: 1,
info: 2,
failed: 3,
success: 4,
http: 5,
verbose: 6,
debug: 7,
silly: 8
},
colors: {
info: 'white',
success: 'green',
failed: 'red'
}
}
const t = [
new transports.Console({
format: format.combine(format.printf(consoleFormat))
})
]
const logger = createLogger({
transports: t,
levels: config.levels,
level: 'verbose'
})
addColors(config.colors)
module.exports = logger

39
scripts/core/markdown.js Normal file
View File

@ -0,0 +1,39 @@
const markdownInclude = require('markdown-include')
const file = require('./file')
const markdown = {}
markdown.createTable = function (data, cols) {
let output = '<table>\n'
output += ' <thead>\n <tr>'
for (let column of cols) {
output += `<th align="${column.align}">${column.name}</th>`
}
output += '</tr>\n </thead>\n'
output += ' <tbody>\n'
for (let item of data) {
output += ' <tr>'
let i = 0
for (let prop in item) {
const column = cols[i]
let nowrap = column.nowrap
let align = column.align
output += `<td align="${align}"${nowrap ? ' nowrap' : ''}>${item[prop]}</td>`
i++
}
output += '</tr>\n'
}
output += ' </tbody>\n'
output += '</table>'
return output
}
markdown.compile = function (filepath) {
markdownInclude.compileFiles(file.resolve(filepath))
}
module.exports = markdown

31
scripts/core/parser.js Normal file
View File

@ -0,0 +1,31 @@
const ipp = require('iptv-playlist-parser')
const logger = require('./logger')
const file = require('./file')
const parser = {}
parser.parsePlaylist = async function (filepath) {
const content = await file.read(filepath)
const playlist = ipp.parse(content)
return playlist.items
}
parser.parseLogs = async function (filepath) {
const content = await file.read(filepath)
if (!content) return []
const lines = content.split('\n')
return lines.map(line => (line ? JSON.parse(line) : null)).filter(l => l)
}
parser.parseNumber = function (string) {
const parsed = parseInt(string)
if (isNaN(parsed)) {
logger.error('Not a number')
}
return parsed
}
module.exports = parser

49
scripts/core/playlist.js Normal file
View File

@ -0,0 +1,49 @@
const file = require('./file')
const playlist = {}
playlist.create = async function (filepath) {
playlist.filepath = filepath
const dir = file.dirname(filepath)
file.createDir(dir)
await file.create(filepath, '')
return playlist
}
playlist.header = async function (attrs) {
let header = `#EXTM3U`
for (const name in attrs) {
const value = attrs[name]
header += ` ${name}="${value}"`
}
header += `\n`
await file.append(playlist.filepath, header)
return playlist
}
playlist.link = async function (url, title, attrs, vlcOpts) {
let link = `#EXTINF:-1`
for (const name in attrs) {
const value = attrs[name]
if (value !== undefined) {
link += ` ${name}="${value}"`
}
}
link += `,${title}\n`
for (const name in vlcOpts) {
const value = vlcOpts[name]
if (value !== undefined) {
link += `#EXTVLCOPT:${name}=${value}\n`
}
}
link += `${url}\n`
await file.append(playlist.filepath, link)
return playlist
}
module.exports = playlist

56
scripts/core/store.js Normal file
View File

@ -0,0 +1,56 @@
const _ = require('lodash')
const logger = require('./logger')
const setters = require('../store/setters')
const getters = require('../store/getters')
module.exports = {
create(state = {}) {
return {
state,
changed: false,
set: function (prop, value) {
const prevState = JSON.stringify(this.state)
const setter = setters[prop]
if (typeof setter === 'function') {
try {
this.state[prop] = setter.bind()(value)
} catch (error) {
logger.error(`store/setters/${prop}.js: ${error.message}`)
}
} else if (typeof value === 'object') {
this.state[prop] = value[prop]
} else {
this.state[prop] = value
}
const newState = JSON.stringify(this.state)
if (prevState !== newState) {
this.changed = true
}
return this
},
get: function (prop) {
const getter = getters[prop]
if (typeof getter === 'function') {
try {
return getter.bind(this.state)()
} catch (error) {
logger.error(`store/getters/${prop}.js: ${error.message}`)
}
} else {
return prop.split('.').reduce((o, i) => (o ? o[i] : undefined), this.state)
}
},
has: function (prop) {
const value = this.get(prop)
return !_.isEmpty(value)
},
data: function () {
return this.state
}
}
}
}

29
scripts/core/timer.js Normal file
View File

@ -0,0 +1,29 @@
const { performance } = require('perf_hooks')
const dayjs = require('dayjs')
const duration = require('dayjs/plugin/duration')
const relativeTime = require('dayjs/plugin/relativeTime')
dayjs.extend(relativeTime)
dayjs.extend(duration)
const timer = {}
let t0 = 0
timer.start = function () {
t0 = performance.now()
}
timer.format = function (f) {
let t1 = performance.now()
return dayjs.duration(t1 - t0).format(f)
}
timer.humanize = function (suffix = true) {
let t1 = performance.now()
return dayjs.duration(t1 - t0).humanize(suffix)
}
module.exports = timer