iptv/scripts/commands/update-database.js

242 lines
5.8 KiB
JavaScript

const _ = require('lodash')
const statuses = require('../data/statuses')
const languages = require('../data/languages')
const { db, store, parser, file, logger } = require('../core')
let epgCodes = []
let streams = []
let checkResults = {}
const origins = {}
const items = []
const LOGS_PATH = process.env.LOGS_PATH || 'scripts/logs'
const EPG_CODES_FILEPATH = process.env.EPG_CODES_FILEPATH || 'scripts/data/codes.json'
async function main() {
await setUp()
await loadDatabase()
await removeDuplicates()
await loadCheckResults()
await findStreamOrigins()
await updateStreams()
await updateDatabase()
}
main()
async function loadDatabase() {
logger.info('Loading database...')
streams = await db.find({})
logger.info(`Found ${streams.length} streams`)
}
async function removeDuplicates() {
logger.info('Removing duplicates...')
const before = streams.length
streams = _.uniqBy(streams, 'id')
const after = streams.length
logger.info(`Removed ${before - after} links`)
}
async function loadCheckResults() {
logger.info('Loading check results from logs/...')
const files = await file.list(`${LOGS_PATH}/check-streams/cluster_*.log`)
for (const filepath of files) {
const results = await parser.parseLogs(filepath)
for (const result of results) {
checkResults[result._id] = result
}
}
logger.info(`Found ${Object.values(checkResults).length} results`)
}
async function findStreamOrigins() {
logger.info('Searching for stream origins...')
for (const { error, requests } of Object.values(checkResults)) {
if (error || !Array.isArray(requests) || !requests.length) continue
let origin = requests.shift()
origin = new URL(origin.url)
for (const request of requests) {
const curr = new URL(request.url)
const key = curr.href.replace(/(^\w+:|^)/, '')
if (!origins[key] && curr.host === origin.host) {
origins[key] = origin.href
}
}
}
logger.info(`Found ${_.uniq(Object.values(origins)).length} origins`)
}
async function updateStreams() {
logger.info('Updating streams...')
let updated = 0
for (const item of streams) {
const stream = store.create(item)
const result = checkResults[item._id]
if (result) {
const { error, streams, requests } = result
const resolution = parseResolution(streams)
const origin = findOrigin(requests)
let status = parseStatus(error)
if (status) {
const prevStatus = item.status
if (prevStatus.code === 'not_247') // not_247 -> * = not_247
status = item.status
else if (prevStatus.code === 'geo_blocked') // geo_blocked -> * = geo_blocked
status = item.status
else if(prevStatus.code === 'offline' && status.code === 'online') // offline -> online = not_247
status = statuses['not_247']
stream.set('status', { status })
stream.set('is_broken', { status: stream.get('status') })
}
if (resolution) {
stream.set('resolution', { resolution })
}
if (origin) {
stream.set('url', { url: origin })
}
}
if (!stream.has('logo')) {
const logo = findLogo(stream.get('id'))
stream.set('logo', { logo })
}
if (!stream.has('guides')) {
const guides = findGuides(stream.get('id'))
stream.set('guides', { guides })
}
if (!stream.has('countries') && stream.get('src_country')) {
const countries = [stream.get('src_country')]
stream.set('countries', { countries })
}
if (!stream.has('languages')) {
const languages = findLanguages(stream.get('countries'), stream.get('src_country'))
stream.set('languages', { languages })
}
if (stream.changed) {
stream.set('updated', true)
items.push(stream.data())
updated++
}
}
logger.info(`Updated ${updated} items`)
}
async function updateDatabase() {
logger.info('Updating database...')
for (const item of items) {
await db.update({ _id: item._id }, item)
}
db.compact()
logger.info('Done')
}
async function setUp() {
try {
const codes = await file.read(EPG_CODES_FILEPATH)
epgCodes = JSON.parse(codes)
} catch (err) {
logger.error(err.message)
}
}
function findLanguages(countries, src_country) {
if (countries && Array.isArray(countries)) {
let codes = countries.map(country => country.lang)
codes = _.uniq(codes)
return codes.map(code => languages.find(l => l.code === code)).filter(l => l)
}
if (src_country) {
const code = src_country.lang
const lang = languages.find(l => l.code === code)
return lang ? [lang] : []
}
return []
}
function findOrigin(requests) {
if (origins && Array.isArray(requests)) {
requests = requests.map(r => r.url.replace(/(^\w+:|^)/, ''))
for (const url of requests) {
if (origins[url]) {
return origins[url]
}
}
}
return null
}
function parseResolution(streams) {
const resolution = streams
.filter(s => s.codec_type === 'video')
.reduce(
(acc, curr) => {
if (curr.height > acc.height) return { width: curr.width, height: curr.height }
return acc
},
{ width: 0, height: 0 }
)
if (resolution.width > 0 && resolution.height > 0) return resolution
return null
}
function parseStatus(error) {
if (error) {
if (error.includes('timed out')) {
return statuses['timeout']
} else if (error.includes('403')) {
return statuses['geo_blocked']
}
return statuses['offline']
}
return statuses['online']
}
function findLogo(id) {
const item = epgCodes.find(i => i.tvg_id === id)
if (item && item.logo) {
return item.logo
}
return null
}
function findGuides(id) {
const item = epgCodes.find(i => i.tvg_id === id)
if (item && Array.isArray(item.guides)) {
return item.guides
}
return []
}