Merge pull request #4187 from iptv-org/update-format-js

Update format.js
2021-08-19 15:48:44 +03:00 · 2021-08-19 15:48:44 +03:00 · 417d119583
parent f7b67dd44a 4d2c065e88
commit 417d119583
5 changed files with 144 additions and 93 deletions
--- a/.github/workflows/auto-update.yml
+++ b/.github/workflows/auto-update.yml
@ -197,7 +197,7 @@ jobs:
      - name: Install Dependencies
        run: npm install
      - name: Format Playlists
-        run: node scripts/format.js --country=${{ matrix.country }} --status --resolution --debug
+        run: node scripts/format.js --country=${{ matrix.country }} --debug
      - name: Upload Artifact
        uses: actions/upload-artifact@v2
        with:
--- a/package-lock.json
+++ b/package-lock.json
@ -8,7 +8,7 @@
      "dependencies": {
        "@freearhey/iso-639-3": "^1.0.0",
        "commander": "^7.0.0",
-        "iptv-checker": "^0.20.2",
+        "iptv-checker": "^0.21.0",
        "iptv-playlist-parser": "^0.5.4",
        "m3u-linter": "^0.2.1",
        "markdown-include": "^0.4.3",
@ -1904,9 +1904,9 @@
      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
    },
    "node_modules/iptv-checker": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/iptv-checker/-/iptv-checker-0.20.2.tgz",
-      "integrity": "sha512-3lzN3q1l3TApHvsP6Rt4ALrs8PE+Zb2gmuykwPzQNtlfazIUR73Yeq/ypwpqRwYJxSq5WMoEdQHwV4KcR7y5Ig==",
+      "version": "0.21.0",
+      "resolved": "https://registry.npmjs.org/iptv-checker/-/iptv-checker-0.21.0.tgz",
+      "integrity": "sha512-qWqq8mLl+qRgV0/fmBpZm8/Cf7FwSnIA1gt9PFlYbzilRDFtPoTRDm0eP9fCWHVwwZZsImE1aMLx8qkikL5Aog==",
      "dependencies": {
        "axios": "^0.21.1",
        "colors": "^1.4.0",
@ -5252,9 +5252,9 @@
      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
    },
    "iptv-checker": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/iptv-checker/-/iptv-checker-0.20.2.tgz",
-      "integrity": "sha512-3lzN3q1l3TApHvsP6Rt4ALrs8PE+Zb2gmuykwPzQNtlfazIUR73Yeq/ypwpqRwYJxSq5WMoEdQHwV4KcR7y5Ig==",
+      "version": "0.21.0",
+      "resolved": "https://registry.npmjs.org/iptv-checker/-/iptv-checker-0.21.0.tgz",
+      "integrity": "sha512-qWqq8mLl+qRgV0/fmBpZm8/Cf7FwSnIA1gt9PFlYbzilRDFtPoTRDm0eP9fCWHVwwZZsImE1aMLx8qkikL5Aog==",
      "requires": {
        "axios": "^0.21.1",
        "colors": "^1.4.0",
--- a/package.json
+++ b/package.json
@ -12,7 +12,7 @@
  "dependencies": {
    "@freearhey/iso-639-3": "^1.0.0",
    "commander": "^7.0.0",
-    "iptv-checker": "^0.20.2",
+    "iptv-checker": "^0.21.0",
    "iptv-playlist-parser": "^0.5.4",
    "m3u-linter": "^0.2.1",
    "markdown-include": "^0.4.3",
--- a/scripts/format.js
+++ b/scripts/format.js
@ -1,105 +1,153 @@
 const IPTVChecker = require('iptv-checker')
 const normalize = require('normalize-url')
 const { program } = require('commander')
-const ProgressBar = require('progress')
 const parser = require('./helpers/parser')
 const utils = require('./helpers/utils')
 const file = require('./helpers/file')
 const log = require('./helpers/log')

+const ignoreStatus = ['Geo-blocked', 'Not 24/7']
+
 program
  .usage('[OPTIONS]...')
-  .option('-d, --debug', 'Enable debug mode')
-  .option('-s, --status', 'Update stream status')
-  .option('-r, --resolution', 'Detect stream resolution')
+  .option('--debug', 'Enable debug mode')
+  .option('-d, --delay <delay>', 'Set delay for each request', parseNumber, 0)
+  .option('-t, --timeout <timeout>', 'Set timeout for each request', parseNumber, 5000)
  .option('-c, --country <country>', 'Comma-separated list of country codes', '')
  .option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
-  .option('--delay <delay>', 'Set delay for each request', 0)
-  .option('--timeout <timeout>', 'Set timeout for each request', 5000)
  .parse(process.argv)

-let bar
-const ignoreStatus = ['Geo-blocked', 'Not 24/7']
 const config = program.opts()
 const checker = new IPTVChecker({
  timeout: config.timeout
 })

+let buffer, origins
 async function main() {
  log.start()

-  if (config.debug) log.print(`Debug mode enabled\n`)
-  if (config.status) log.print(`Status check enabled\n`)
-  if (config.resolution) log.print(`Resolution detection enabled\n`)
-
  let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
  playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
  if (!playlists.length) log.print(`No playlist is selected\n`)
  for (const playlist of playlists) {
-    await parser
-      .parsePlaylist(playlist.url)
-      .then(updatePlaylist)
-      .then(playlist => {
-        if (file.read(playlist.url) !== playlist.toString()) {
-          log.print(`File '${playlist.url}' has been updated\n`)
-          playlist.updated = true
-        }
-
-        playlist.save()
-      })
+    await parser.parsePlaylist(playlist.url).then(updatePlaylist).then(savePlaylist)
  }

  log.finish()
 }

-async function updatePlaylist(playlist) {
-  if (!config.debug) {
-    bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, {
-      total: playlist.channels.length
-    })
-  } else {
-    log.print(`Processing '${playlist.url}'...\n`)
+function savePlaylist(playlist) {
+  if (file.read(playlist.url) !== playlist.toString()) {
+    log.print(`File '${playlist.url}' has been updated\n`)
+    playlist.updated = true
  }

-  for (const channel of playlist.channels) {
-    addMissingData(channel, playlist)
-    updateGroupTitle(channel)
+  playlist.save()
+}
+
+async function updatePlaylist(playlist) {
+  const total = playlist.channels.length
+  log.print(`Processing '${playlist.url}'...\n`)
+
+  buffer = {}
+  origins = {}
+  for (const [i, channel] of playlist.channels.entries()) {
+    const curr = i + 1
+    updateDescription(channel, playlist)
    normalizeUrl(channel)

-    const checkOnline = config.status || config.resolution
-    const skipChannel =
-      channel.status &&
-      ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
-    if (checkOnline && !skipChannel) {
-      await checker
-        .checkStream(channel.data)
-        .then(result => {
-          const status = parseStatus(result.status)
-
-          if (config.status) {
-            updateStatus(channel, status)
-          }
-
-          if (config.resolution && status === 'online') {
-            updateResolution(channel, result.status.metadata)
-          }
-
-          if (config.debug && status === 'offline') {
-            log.print(`  ERR: ${channel.url} (${result.status.reason})\n`)
-          }
-        })
-        .catch(err => {
-          if (config.debug) log.print(`  ERR: ${channel.url} (${err.message})\n`)
-        })
-
-      await utils.sleep(config.delay)
+    if (ignoreStatus.includes(channel.status)) {
+      continue
    }
-    if (!config.debug) bar.tick()
+
+    await checker
+      .checkStream(channel.data)
+      .then(parseResult)
+      .then(result => {
+        updateStatus(channel, result.status)
+        if (result.status === 'online') {
+          buffer[i] = result
+          updateOrigins(channel, result.requests)
+          updateResolution(channel, result.resolution)
+        } else {
+          buffer[i] = null
+          if (config.debug) {
+            log.print(`  INFO: ${channel.url} (${result.error})\n`)
+          }
+        }
+      })
+      .catch(err => {
+        buffer[i] = null
+        if (config.debug) {
+          log.print(`  ERR: ${channel.data.url} (${err.message})\n`)
+        }
+      })
+  }
+
+  for (const [i, channel] of playlist.channels.entries()) {
+    if (!buffer[i]) continue
+    const { requests } = buffer[i]
+    updateUrl(channel, requests)
  }

  return playlist
 }

+function updateOrigins(channel, requests) {
+  if (!requests) return
+  const origin = new URL(channel.url)
+  const target = new URL(requests[0])
+  const type = origin.host === target.host ? 'origin' : 'redirect'
+  requests.forEach(url => {
+    const key = utils.removeProtocol(url)
+    if (!origins[key] && type === 'origin') {
+      origins[key] = channel.url
+    }
+  })
+}
+
+function updateStatus(channel, status) {
+  switch (status) {
+    case 'online':
+      channel.status = null
+      break
+    case 'offline':
+      channel.status = 'Offline'
+      break
+  }
+}
+
+function updateResolution(channel, resolution) {
+  if (!channel.resolution.height && resolution) {
+    channel.resolution = resolution
+  }
+}
+
+function updateUrl(channel, requests) {
+  for (const request of requests) {
+    let key = utils.removeProtocol(channel.url)
+    if (origins[key]) {
+      channel.updateUrl(origins[key])
+      break
+    }
+
+    key = utils.removeProtocol(request)
+    if (origins[key]) {
+      channel.updateUrl(origins[key])
+      break
+    }
+  }
+}
+
+function parseResult(result) {
+  return {
+    status: parseStatus(result.status),
+    resolution: result.status.ok ? parseResolution(result.status.metadata.streams) : null,
+    requests: result.status.ok ? parseRequests(result.status.metadata.requests) : [],
+    error: !result.status.ok ? result.status.reason : null
+  }
+}
+
 function parseStatus(status) {
  if (status.ok) {
    return 'online'
@ -114,18 +162,28 @@ function parseStatus(status) {
  }
 }

-function updateStatus(channel, status) {
-  switch (status) {
-    case 'online':
-      channel.status = null
-      break
-    case 'offline':
-      channel.status = 'Offline'
-      break
-  }
+function parseResolution(streams) {
+  const resolution = streams
+    .filter(stream => stream.codec_type === 'video')
+    .reduce(
+      (acc, curr) => {
+        if (curr.height > acc.height) return { width: curr.width, height: curr.height }
+        return acc
+      },
+      { width: 0, height: 0 }
+    )
+
+  return resolution.width > 0 && resolution.height > 0 ? resolution : null
 }

-function addMissingData(channel, playlist) {
+function parseRequests(requests) {
+  requests = requests.map(r => r.url)
+  requests.shift()
+
+  return requests
+}
+
+function updateDescription(channel, playlist) {
  const code = playlist.country.code
  // tvg-name
  if (!channel.tvg.name && channel.name) {
@ -142,9 +200,7 @@ function addMissingData(channel, playlist) {
    channel.countries = name ? [{ code, name }] : []
    channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';')
  }
-}
-
-function updateGroupTitle(channel) {
+  // group-title
  channel.group.title = channel.category
 }

@ -154,14 +210,8 @@ function normalizeUrl(channel) {
  channel.updateUrl(decoded)
 }

-function updateResolution(channel, metadata) {
-  const streams = metadata ? metadata.streams.filter(stream => stream.codec_type === 'video') : []
-  if (!channel.resolution.height && streams.length) {
-    channel.resolution = streams.reduce((acc, curr) => {
-      if (curr.height > acc.height) return { width: curr.width, height: curr.height }
-      return acc
-    })
-  }
+function parseNumber(str) {
+  return parseInt(str)
 }

 main()
--- a/scripts/helpers/log.js
+++ b/scripts/helpers/log.js
@ -1,7 +1,8 @@
 const log = {}

-log.print = function (string) {
-  process.stdout.write(string)
+log.print = function (message) {
+  if (typeof message === 'object') message = JSON.stringify(message, null, 2)
+  process.stdout.write(message)
 }

 log.start = function () {