Merge pull request #8145 from iptv-org/patch-2022.08

Patch 2022.08
2022-08-21 09:21:35 +02:00
parent f28c7f318f 7683587c3e
commit e82e3ddea7
18 changed files with 167 additions and 116 deletions
--- a/.github/workflows/auto-update.yml
+++ b/.github/workflows/auto-update.yml
@@ -90,17 +90,18 @@ jobs:
          path: scripts/logs
      - run: npm install
      - run: npm run db:update
+      - run: npm run db:clear
      - uses: actions/upload-artifact@v2
        with:
          name: database
          path: scripts/database
-      - run: npm run playlist:update
-      - run: npm run playlist:generate
      - run: npm run db:export
      - uses: actions/upload-artifact@v2
        with:
          name: api
          path: .api
+      - run: npm run playlist:update
+      - run: npm run playlist:generate
      - run: npm run readme:update
      - uses: actions/upload-artifact@v2
        with:
--- a/package.json
+++ b/package.json
@@ -4,6 +4,7 @@
    "act:auto-update": "act workflow_dispatch -W .github/workflows/auto-update.yml --artifact-server-path=.artifacts",
    "act:check": "act pull_request -W .github/workflows/check.yml",
    "api:load": "./scripts/commands/api/load.sh",
+    "db:clear": "node scripts/commands/database/clear.js",
    "db:create": "node scripts/commands/database/create.js",
    "db:matrix": "node scripts/commands/database/matrix.js",
    "db:update": "node scripts/commands/database/update.js",
@@ -13,7 +14,6 @@
    "playlist:generate": "node scripts/commands/playlist/generate.js",
    "playlist:update": "node scripts/commands/playlist/update.js",
    "playlist:lint": "npx m3u-linter -c m3u-linter.json",
-    "playlist:cleaner": "node scripts/commands/playlist/cleaner.js",
    "readme:update": "node scripts/commands/readme/update.js",
    "test": "jest --runInBand"
  },
--- a/scripts/commands/database/clear.js
+++ b/scripts/commands/database/clear.js
@@ -0,0 +1,35 @@
+const { logger, parser, db, date } = require('../../core')
+const { program } = require('commander')
+
+const options = program
+  .option(
+    '-t, --threshold <threshold>',
+    'Number of days after which the stream should be deleted',
+    parser.parseNumber,
+    7
+  )
+  .option('--input-dir <input-dir>', 'Set path to input directory', 'streams')
+  .parse(process.argv)
+  .opts()
+
+async function main() {
+  await db.streams.load()
+
+  const streams = await db.streams.all()
+
+  let total = 0
+  for (const stream of streams) {
+    if (
+      stream.status === 'error' &&
+      date.utc().diff(stream.updated_at, 'day') >= options.threshold
+    ) {
+      total += await db.streams.remove({ url: stream.url }, { multi: true })
+    }
+  }
+
+  await db.streams.compact()
+
+  logger.info(`removed ${total} streams`)
+}
+
+main()
--- a/scripts/commands/database/create.js
+++ b/scripts/commands/database/create.js
@@ -28,6 +28,7 @@ async function findStreams() {
  logger.info(`looking for streams...`)

  await api.channels.load()
+  await api.streams.load()
  await db.streams.load()

  const streams = []
@@ -39,6 +40,7 @@ async function findStreams() {

      const stream = store.create()
      const channel = await api.channels.find({ id: item.tvg.id })
+      const cached = (await api.streams.find({ url: item.url })) || {}

      stream.set('channel', { channel: channel ? channel.id : null })
      stream.set('title', { title: item.name })
@@ -46,6 +48,14 @@ async function findStreams() {
      stream.set('url', { url: item.url })
      stream.set('http_referrer', { http_referrer: item.http.referrer })
      stream.set('user_agent', { user_agent: item.http['user-agent'] })
+      stream.set('status', { status: cached.status })
+      stream.set('width', { width: cached.width })
+      stream.set('height', { height: cached.height })
+      stream.set('bitrate', { bitrate: cached.bitrate })
+      stream.set('frame_rate', { frame_rate: cached.frame_rate })
+      stream.set('added_at', { added_at: cached.added_at })
+      stream.set('updated_at', { updated_at: cached.updated_at })
+      stream.set('checked_at', { checked_at: cached.checked_at })

      streams.push(stream)
    }
--- a/scripts/commands/playlist/cleaner.js
+++ b/scripts/commands/playlist/cleaner.js
@@ -1,90 +0,0 @@
-const { file, parser, logger, checker, m3u } = require('../../core')
-const { program } = require('commander')
-
-program
-  .argument('[filepath]', 'Path to file to validate')
-  .option('-t, --timeout <timeout>', 'Set timeout for each request', parser.parseNumber, 60000)
-  .option('-d, --delay <delay>', 'Set delay for each request', parser.parseNumber, 0)
-  .option('--debug', 'Enable debug mode')
-  .parse(process.argv)
-
-const options = program.opts()
-
-async function main() {
-  const files = program.args.length ? program.args : await file.list('streams/*.m3u')
-
-  for (const filepath of files) {
-    if (!filepath.endsWith('.m3u')) continue
-    logger.info(`${filepath}`)
-    const playlist = await parser.parsePlaylist(filepath)
-    const before = playlist.items.length
-    for (const stream of playlist.items) {
-      if (options.debug) logger.info(stream.url)
-      const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null]
-      stream.status = status
-      if (status === 'error' && /^(http|https)/.test(stream.url)) {
-        const result = await checkStream(stream)
-        const newStatus = parseStatus(result.error)
-        if (status === newStatus) {
-          stream.remove = true
-          logger.info(`removed "${stream.name}"`)
-        }
-      }
-    }
-
-    const items = playlist.items
-      .filter(i => !i.remove)
-      .map(item => ({
-        attrs: {
-          'tvg-id': item.tvg.id,
-          status: item.status,
-          'user-agent': item.http['user-agent'] || undefined
-        },
-        title: item.name,
-        url: item.url,
-        vlcOpts: {
-          'http-referrer': item.http.referrer || undefined,
-          'http-user-agent': item.http['user-agent'] || undefined
-        }
-      }))
-
-    if (before !== items.length) {
-      const output = m3u.create(items)
-      await file.create(filepath, output)
-      logger.info(`saved`)
-    }
-  }
-}
-
-main()
-
-async function checkStream(item) {
-  const config = {
-    timeout: options.timeout,
-    delay: options.delay,
-    debug: options.debug
-  }
-
-  const request = {
-    url: item.url,
-    http: {
-      referrer: item.http.referrer,
-      'user-agent': item.http['user-agent']
-    }
-  }
-
-  return checker.check(request, config)
-}
-
-function parseStatus(error) {
-  if (!error) return 'online'
-
-  switch (error) {
-    case 'Operation timed out':
-      return 'timeout'
-    case 'Server returned 403 Forbidden (access denied)':
-      return 'blocked'
-    default:
-      return 'error'
-  }
-}
--- a/scripts/core/date.js
+++ b/scripts/core/date.js
@@ -0,0 +1,12 @@
+const dayjs = require('dayjs')
+const utc = require('dayjs/plugin/utc')
+
+dayjs.extend(utc)
+
+const date = {}
+
+date.utc = d => {
+	return dayjs.utc(d)
+}
+
+module.exports = date
--- a/scripts/core/db.js
+++ b/scripts/core/db.js
@@ -63,6 +63,10 @@ class Database {
    return this.db.find(query)
  }

+  all() {
+    return this.find({})
+  }
+
  remove(query, options) {
    return this.db.remove(query, options)
  }
--- a/scripts/core/index.js
+++ b/scripts/core/index.js
@@ -11,3 +11,4 @@ exports.markdown = require('./markdown')
 exports.api = require('./api')
 exports.id = require('./id')
 exports.m3u = require('./m3u')
+exports.date = require('./date')
--- a/tests/data/expected/database/db_clear.streams.db
+++ b/tests/data/expected/database/db_clear.streams.db
@@ -0,0 +1,6 @@
+{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"}
+{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"}
+{"title":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
+{"title":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"tests/__data__/output/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
+{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"}
+{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"}
--- a/tests/data/expected/database/db_create.streams.db
+++ b/tests/data/expected/database/db_create.streams.db
@@ -1,6 +1,7 @@
-{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"TyQaTTYos0fr2q0P"}
-{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"yNDfQt0ITDrOGGV2"}
-{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"asTdyPDWW77mXDLZ"}
-{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"1gBgkVYcwsNJQlso"}
-{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"8F6RyHFzpOe20huV"}
-{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"9DY8CqVcKyp8jqiA"}
+{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"UAYbCvjOWvqXHH95"}
+{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"IjKHoixx2crsuOlE"}
+{"channel":"LibyasChannel.ly","title":"Libyas Channel","filepath":"tests/__data__/input/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","updated_at":"2022-07-07T00:00:00Z","checked_at":"2022-07-07T00:00:00Z","cluster_id":1,"_id":"I0rJlwp3rZEy2SnG"}
+{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"tdzk1IN7wLJxfGab"}
+{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error","added_at":"2022-07-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z","cluster_id":1,"_id":"3evMHt3nFsZjhzas"}
+{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"agJlTEr8wwpbWgw0"}
+{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"FpMhoWjtpnVuYlO9"}
--- a/tests/data/input/data/channels.json
+++ b/tests/data/input/data/channels.json
@@ -123,6 +123,23 @@
    "is_nsfw": false,
    "logo": "https://iptvx.one/icn/ldpr-tv.png"
  },
+  {
+    "id": "LibyasChannel.ly",
+    "name": "Libyas Channel",
+    "network": null,
+    "country": "LY",
+    "subdivision": null,
+    "city": null,
+    "broadcast_area": [
+      "c/LY"
+    ],
+    "languages": [
+      "eng"
+    ],
+    "categories": [],
+    "is_nsfw": false,
+    "logo": "https://i.imgur.com/RD9wbNF.jpg"
+  },
  {
    "id": "MeteoMedia.ca",
    "name": "MétéoMédia",
--- a/tests/data/input/database/db_clear.streams.db
+++ b/tests/data/input/database/db_clear.streams.db
@@ -0,0 +1,7 @@
+{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"}
+{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"}
+{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz","status":"error","added_at":"2022-07-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z"}
+{"title":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
+{"title":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"tests/__data__/output/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
+{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"}
+{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"}
--- a/tests/data/input/streams/ly.m3u
+++ b/tests/data/input/streams/ly.m3u
@@ -0,0 +1,3 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="LibyasChannel.ly",Libyas Channel
+https://master.starmena-cloud.com/hls/libyas.m3u8
--- a/tests/commands/database/clear.test.js
+++ b/tests/commands/database/clear.test.js
@@ -0,0 +1,48 @@
+const fs = require('fs-extra')
+const path = require('path')
+const { execSync } = require('child_process')
+
+beforeEach(() => {
+  fs.emptyDirSync('tests/__data__/output')
+  fs.mkdirSync('tests/__data__/output/database')
+  fs.copyFileSync(
+    'tests/__data__/input/database/db_clear.streams.db',
+    'tests/__data__/output/database/streams.db'
+  )
+
+  const stdout = execSync(
+    'DB_DIR=tests/__data__/output/database npm run db:clear -- --threshold 7',
+    {
+      encoding: 'utf8'
+    }
+  )
+})
+
+it('can clear database', () => {
+  let output = content('tests/__data__/output/database/streams.db')
+  let expected = content('tests/__data__/expected/database/db_clear.streams.db')
+
+  output = output.map(i => {
+    i._id = null
+    return i
+  })
+  expected = expected.map(i => {
+    i._id = null
+    return i
+  })
+
+  expect(output).toMatchObject(expected)
+})
+
+function content(filepath) {
+  const data = fs.readFileSync(path.resolve(filepath), {
+    encoding: 'utf8'
+  })
+
+  return data
+    .split('\n')
+    .filter(l => l)
+    .map(l => {
+      return JSON.parse(l)
+    })
+}
--- a/tests/commands/database/create.test.js
+++ b/tests/commands/database/create.test.js
@@ -4,9 +4,10 @@ const { execSync } = require('child_process')

 beforeEach(() => {
  fs.emptyDirSync('tests/__data__/output')
+  fs.mkdirSync('tests/__data__/output/database')

  const stdout = execSync(
-    'DB_DIR=tests/__data__/output/database npm run db:create -- --input-dir=tests/__data__/input/streams --max-clusters=1',
+    'DB_DIR=tests/__data__/output/database DATA_DIR=tests/__data__/input/data npm run db:create -- --input-dir=tests/__data__/input/streams --max-clusters=1',
    { encoding: 'utf8' }
  )
 })
@@ -24,14 +25,7 @@ it('can create database', () => {
    return i
  })

-  expect(output).toEqual(
-    expect.arrayContaining([
-      expect.objectContaining(expected[0]),
-      expect.objectContaining(expected[1]),
-      expect.objectContaining(expected[2]),
-      expect.objectContaining(expected[3])
-    ])
-  )
+  expect(output).toMatchObject(expect.arrayContaining(expected))
 })

 function content(filepath) {
--- a/tests/commands/database/export.test.js
+++ b/tests/commands/database/export.test.js
@@ -5,13 +5,14 @@ const dayjs = require('dayjs')

 beforeEach(() => {
  fs.emptyDirSync('tests/__data__/output')
+  fs.mkdirSync('tests/__data__/output/database')
  fs.copyFileSync(
    'tests/__data__/input/database/db_export.streams.db',
-    'tests/__data__/output/streams.db'
+    'tests/__data__/output/database/streams.db'
  )

  const stdout = execSync(
-    'DB_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.api npm run db:export',
+    'DB_DIR=tests/__data__/output/database DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.api npm run db:export',
    { encoding: 'utf8' }
  )
 })
--- a/tests/commands/database/matrix.test.js
+++ b/tests/commands/database/matrix.test.js
@@ -4,15 +4,15 @@ const { execSync } = require('child_process')

 beforeEach(() => {
  fs.emptyDirSync('tests/__data__/output')
-
+  fs.mkdirSync('tests/__data__/output/database')
  fs.copyFileSync(
    'tests/__data__/input/database/db_matrix.streams.db',
-    'tests/__data__/output/streams.db'
+    'tests/__data__/output/database/streams.db'
  )
 })

 it('can create valid matrix', () => {
-  const result = execSync('DB_DIR=tests/__data__/output npm run db:matrix', {
+  const result = execSync('DB_DIR=tests/__data__/output/database npm run db:matrix', {
    encoding: 'utf8'
  })
  expect(result).toBe(
--- a/tests/commands/database/update.test.js
+++ b/tests/commands/database/update.test.js
@@ -4,15 +4,16 @@ const path = require('path')

 beforeEach(() => {
  fs.emptyDirSync('tests/__data__/output')
+  fs.mkdirSync('tests/__data__/output/database')
  fs.copyFileSync(
    'tests/__data__/input/database/db_update.streams.db',
-    'tests/__data__/output/streams.db'
+    'tests/__data__/output/database/streams.db'
  )
 })

 it('can save results', () => {
  const stdout = execSync(
-    'DB_DIR=tests/__data__/output LOGS_DIR=tests/__data__/input/logs/cluster/load npm run db:update',
+    'DB_DIR=tests/__data__/output/database LOGS_DIR=tests/__data__/input/logs/cluster/load npm run db:update',
    { encoding: 'utf8' }
  )
  expect(stdout).toEqual(`
@@ -31,7 +32,7 @@ removed 1 duplicates
 done
 `)

-  expect(content('tests/__data__/output/streams.db')).toEqual(
+  expect(content('tests/__data__/output/database/streams.db')).toEqual(
    content('tests/__data__/expected/database/db_update.streams.db')
  )
 })