Merge branch 'master' into LaneSh4d0w-patch-1
This commit is contained in:
commit
536418d69d
|
@ -17,8 +17,9 @@ jobs:
|
|||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
branch: 'bot/remove-broken-links'
|
||||
test:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
needs: create-branch
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
@ -193,6 +194,8 @@ jobs:
|
|||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/remove-broken-links
|
||||
- name: Setup FFmpeg
|
||||
uses: FedericoCarboni/setup-ffmpeg@v1
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Remove Broken Links
|
||||
|
@ -204,7 +207,7 @@ jobs:
|
|||
path: channels/${{ matrix.country }}.m3u
|
||||
commit-changes:
|
||||
runs-on: ubuntu-latest
|
||||
needs: test
|
||||
needs: check
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
|
@ -215,13 +218,14 @@ jobs:
|
|||
- name: Commit Changes
|
||||
uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
commit_message: '[Bot] Update playlists'
|
||||
commit_message: '[Bot] Remove broken links'
|
||||
commit_user_name: iptv-bot
|
||||
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
|
||||
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
|
||||
branch: bot/remove-broken-links
|
||||
file_pattern: channels/*
|
||||
pull-request:
|
||||
if: ${{ github.ref == 'refs/heads/master' }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: commit-changes
|
||||
steps:
|
||||
|
@ -244,9 +248,9 @@ jobs:
|
|||
pr_body: |
|
||||
This pull request is created by [clean][1] workflow.
|
||||
|
||||
The script checks each link and removes only those that return a HTTP 404 code (Not Found). Also, the script ignores links with labels `[Geo-blocked]` and `[Not 24/7]` in the title.
|
||||
The script checks all links except those with labels `[Geo-blocked]`, `[Offline]` or `[Not 24/7]` in the title.
|
||||
|
||||
**IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. Working links should be marked as `[Not 24/7]` so that next time the script will not delete them.
|
||||
**IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. If the link works for you but occasionally return an HTTP code 403 (Forbidden) then it should be marked as `[Geo-blocked]`. If the link does not work but has no alternative, you can mark it as `[Offline]` to save it in the playlist along with a description. Working links should be marked as `[Not 24/7]` so that the script will skip them next time.
|
||||
|
||||
[1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }}
|
||||
pr_draft: true
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -15,6 +15,7 @@
|
|||
"chalk": "^4.1.1",
|
||||
"commander": "^7.0.0",
|
||||
"escape-string-regexp": "^2.0.0",
|
||||
"iptv-checker": "^0.20.2",
|
||||
"iptv-playlist-parser": "^0.5.4",
|
||||
"m3u-linter": "^0.1.3",
|
||||
"markdown-include": "^0.4.3",
|
||||
|
|
|
@ -1,93 +1,65 @@
|
|||
const IPTVChecker = require('iptv-checker')
|
||||
const { program } = require('commander')
|
||||
const ProgressBar = require('progress')
|
||||
const axios = require('axios')
|
||||
const https = require('https')
|
||||
const chalk = require('chalk')
|
||||
const parser = require('./helpers/parser')
|
||||
const utils = require('./helpers/utils')
|
||||
const log = require('./helpers/log')
|
||||
|
||||
program
|
||||
.usage('[OPTIONS]...')
|
||||
.option('-d, --debug', 'Enable debug mode')
|
||||
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
|
||||
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
|
||||
.option('--delay <delay>', 'Delay between parser requests', 1000)
|
||||
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
|
||||
.parse(process.argv)
|
||||
|
||||
let bar
|
||||
const config = program.opts()
|
||||
const offlineStatusCodes = [404, 410, 451, 500, 501]
|
||||
const ignoreStatus = ['Geo-blocked', 'Not 24/7', 'Offline']
|
||||
const instance = axios.create({
|
||||
timeout: config.timeout,
|
||||
maxContentLength: 200000,
|
||||
httpsAgent: new https.Agent({
|
||||
rejectUnauthorized: false
|
||||
}),
|
||||
validateStatus: function (status) {
|
||||
return !offlineStatusCodes.includes(status)
|
||||
}
|
||||
const checker = new IPTVChecker({
|
||||
timeout: config.timeout
|
||||
})
|
||||
|
||||
let broken = 0
|
||||
|
||||
async function main() {
|
||||
log.start()
|
||||
|
||||
log.print(`Parsing 'index.m3u'...`)
|
||||
if (config.debug) log.print(`Debug mode enabled\n`)
|
||||
|
||||
let playlists = parser.parseIndex()
|
||||
playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
|
||||
for (const playlist of playlists) {
|
||||
await parser
|
||||
.parsePlaylist(playlist.url)
|
||||
.then(checkStatus)
|
||||
.then(checkPlaylist)
|
||||
.then(p => p.save())
|
||||
}
|
||||
|
||||
log.finish()
|
||||
}
|
||||
|
||||
async function checkStatus(playlist) {
|
||||
let bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
async function checkPlaylist(playlist) {
|
||||
if (!config.debug) {
|
||||
bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
}
|
||||
const channels = []
|
||||
const total = playlist.channels.length
|
||||
for (const [index, channel] of playlist.channels.entries()) {
|
||||
const current = index + 1
|
||||
const counter = chalk.gray(`[${current}/${total}]`)
|
||||
const skipChannel =
|
||||
channel.status &&
|
||||
ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
|
||||
bar.tick()
|
||||
if (
|
||||
skipChannel ||
|
||||
(!channel.url.startsWith('http://') && !channel.url.startsWith('https://'))
|
||||
) {
|
||||
if (skipChannel) {
|
||||
channels.push(channel)
|
||||
} else {
|
||||
const CancelToken = axios.CancelToken
|
||||
const source = CancelToken.source()
|
||||
const timeout = setTimeout(() => {
|
||||
source.cancel()
|
||||
}, config.timeout)
|
||||
|
||||
await instance
|
||||
.get(channel.url, { cancelToken: source.token })
|
||||
.then(() => {
|
||||
clearTimeout(timeout)
|
||||
channels.push(channel)
|
||||
})
|
||||
.then(utils.sleep(config.delay))
|
||||
.catch(err => {
|
||||
clearTimeout(timeout)
|
||||
if (err.response && offlineStatusCodes.includes(err.response.status)) {
|
||||
broken++
|
||||
} else {
|
||||
channels.push(channel)
|
||||
}
|
||||
})
|
||||
const result = await checker.checkStream(channel.data)
|
||||
if (result.status.ok || result.status.reason.includes('timed out')) {
|
||||
channels.push(channel)
|
||||
} else {
|
||||
if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`)
|
||||
}
|
||||
}
|
||||
if (!config.debug) bar.tick()
|
||||
}
|
||||
|
||||
if (playlist.channels.length !== channels.length) {
|
||||
|
|
|
@ -7,6 +7,7 @@ const nsfwCategories = categories.filter(c => c.nsfw).map(c => c.name)
|
|||
|
||||
module.exports = class Channel {
|
||||
constructor(data) {
|
||||
this.data = data
|
||||
this.raw = data.raw
|
||||
this.tvg = data.tvg
|
||||
this.http = data.http
|
||||
|
|
Loading…
Reference in New Issue