ixixy
/
iptv
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'create-playlist-cleaner'

This commit is contained in:
Aleksandr Statciuk 2022-03-11 19:43:32 +03:00
parent 66f692185a 02b27d247f
commit 311629968c
9 changed files with 140 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -12,6 +12,7 @@
"playlist:generate": "node scripts/commands/playlist/generate.js",
"playlist:update": "node scripts/commands/playlist/update.js",
"playlist:lint": "npx m3u-linter -c m3u-linter.json",
"playlist:cleaner": "node scripts/commands/playlist/cleaner.js",
"readme:update": "node scripts/commands/readme/update.js",
"test": "jest --runInBand"
},

4
scripts/commands/database/create.js
View File

@ -33,8 +33,8 @@ async function findStreams() {
const streams = []
const files = await file.list(`${options.inputDir}/**/*.m3u`)
for (const filepath of files) {
const items = await parser.parsePlaylist(filepath)
for (const item of items) {
const playlist = await parser.parsePlaylist(filepath)
for (const item of playlist.items) {
item.filepath = filepath
const stream = store.create()

90
scripts/commands/playlist/cleaner.js Normal file
View File

@ -0,0 +1,90 @@
const { file, parser, logger, checker, m3u } = require('../../core')
const { program } = require('commander')
program
.argument('[filepath]', 'Path to file to validate')
.option('-t, --timeout <timeout>', 'Set timeout for each request', parser.parseNumber, 60000)
.option('-d, --delay <delay>', 'Set delay for each request', parser.parseNumber, 0)
.option('--debug', 'Enable debug mode')
.parse(process.argv)
const options = program.opts()
async function main() {
const files = program.args.length ? program.args : await file.list('streams/*.m3u')
for (const filepath of files) {
if (!filepath.endsWith('.m3u')) continue
logger.info(`${filepath}`)
const playlist = await parser.parsePlaylist(filepath)
const before = playlist.items.length
for (const stream of playlist.items) {
if (options.debug) logger.info(stream.url)
const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null]
stream.status = status
if (status === 'error' && /^(http|https)/.test(stream.url) && !/\[.*\]$/.test(stream.name)) {
const result = await checkStream(stream)
const newStatus = parseStatus(result.error)
if (status === newStatus) {
stream.remove = true
logger.info(`removed "${stream.name}"`)
}
}
}
const items = playlist.items
.filter(i => !i.remove)
.map(item => ({
attrs: {
'tvg-id': item.tvg.id,
status: item.status,
'user-agent': item.http['user-agent'] || undefined
},
title: item.name,
url: item.url,
vlcOpts: {
'http-referrer': item.http.referrer || undefined,
'http-user-agent': item.http['user-agent'] || undefined
}
}))
if (before !== items.length) {
const output = m3u.create(items)
await file.create(filepath, output)
logger.info(`saved`)
}
}
}
main()
async function checkStream(item) {
const config = {
timeout: options.timeout,
delay: options.delay,
debug: options.debug
}
const request = {
url: item.url,
http: {
referrer: item.http.referrer,
'user-agent': item.http['user-agent']
}
}
return checker.check(request, config)
}
function parseStatus(error) {
if (!error) return 'online'
switch (error) {
case 'Operation timed out':
return 'timeout'
case 'Server returned 403 Forbidden (access denied)':
return 'blocked'
default:
return 'error'
}
}

6
scripts/commands/playlist/validate.js
View File

@ -6,7 +6,7 @@ const _ = require('lodash')
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
async function main() {
const files = program.args.length ? program.args : await file.list('channels/*.m3u')
const files = program.args.length ? program.args : await file.list('streams/*.m3u')
logger.info(`loading blocklist...`)
await api.channels.load()
@ -31,8 +31,8 @@ async function main() {
const [__, country] = basename.match(/([a-z]{2})(|_.*)\.m3u/i) || [null, null]
const fileLog = []
const items = await parser.parsePlaylist(filepath)
for (const item of items) {
const playlist = await parser.parsePlaylist(filepath)
for (const item of playlist.items) {
if (item.tvg.id && !api.channels.find({ id: item.tvg.id })) {
fileLog.push({
type: 'warning',

1
scripts/core/index.js
View File

@ -10,3 +10,4 @@ exports.store = require('./store')
exports.markdown = require('./markdown')
exports.api = require('./api')
exports.id = require('./id')
exports.m3u = require('./m3u')

34
scripts/core/m3u.js Normal file
View File

@ -0,0 +1,34 @@
const m3u = {}
m3u.create = function (links = [], header = {}) {
let output = `#EXTM3U`
for (const attr in header) {
const value = header[attr]
output += ` ${attr}="${value}"`
}
output += `\n`
for (const link of links) {
output += `#EXTINF:-1`
for (const name in link.attrs) {
const value = link.attrs[name]
if (value !== undefined) {
output += ` ${name}="${value}"`
}
}
output += `,${link.title}\n`
for (const name in link.vlcOpts) {
const value = link.vlcOpts[name]
if (value !== undefined) {
output += `#EXTVLCOPT:${name}=${value}\n`
}
}
output += `${link.url}\n`
}
return output
}
module.exports = m3u

3
scripts/core/parser.js
View File

@ -6,9 +6,8 @@ const parser = {}
parser.parsePlaylist = async function (filepath) {
const content = await file.read(filepath)
const playlist = ipp.parse(content)
return playlist.items
return ipp.parse(content)
}
parser.parseLogs = async function (filepath) {

30
scripts/core/playlist.js
View File

@ -1,4 +1,5 @@
const store = require('./store')
const m3u = require('./m3u')
const _ = require('lodash')
const playlist = {}
@ -50,34 +51,7 @@ class Playlist {
}
toString() {
let output = `#EXTM3U`
for (const attr in this.header) {
const value = this.header[attr]
output += ` ${attr}="${value}"`
}
output += `\n`
for (const link of this.links) {
output += `#EXTINF:-1`
for (const name in link.attrs) {
const value = link.attrs[name]
if (value !== undefined) {
output += ` ${name}="${value}"`
}
}
output += `,${link.title}\n`
for (const name in link.vlcOpts) {
const value = link.vlcOpts[name]
if (value !== undefined) {
output += `#EXTVLCOPT:${name}=${value}\n`
}
}
output += `${link.url}\n`
}
return output
return m3u.create(this.links, this.header)
}
}

10
tests/__data__/expected/database/db_create.streams.db
View File

@ -1,4 +1,6 @@
{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"ZJejfvbOVTyuf6Gk"}
{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"gnjGLZU1CEz79gcp"}
{"channel":"ATV.ad","title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"9r9qmYRa2kxiirl0"}
{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"unOCFJtsDCbJupxR"}
{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"TyQaTTYos0fr2q0P"}
{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"yNDfQt0ITDrOGGV2"}
{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"asTdyPDWW77mXDLZ"}
{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"1gBgkVYcwsNJQlso"}
{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"8F6RyHFzpOe20huV"}
{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"9DY8CqVcKyp8jqiA"}