Merge pull request #8145 from iptv-org/patch-2022.08

Patch 2022.08

.github/workflows/auto-update.yml

@@ -90,17 +90,18 @@ jobs:
           path: scripts/logs
       - run: npm install
       - run: npm run db:update
+      - run: npm run db:clear
       - uses: actions/upload-artifact@v2
         with:
           name: database
           path: scripts/database
-      - run: npm run playlist:update
-      - run: npm run playlist:generate
       - run: npm run db:export
       - uses: actions/upload-artifact@v2
         with:
           name: api
           path: .api
+      - run: npm run playlist:update
+      - run: npm run playlist:generate
       - run: npm run readme:update
       - uses: actions/upload-artifact@v2
         with:

package.json

@@ -4,6 +4,7 @@
     "act:auto-update": "act workflow_dispatch -W .github/workflows/auto-update.yml --artifact-server-path=.artifacts",
     "act:check": "act pull_request -W .github/workflows/check.yml",
     "api:load": "./scripts/commands/api/load.sh",
+    "db:clear": "node scripts/commands/database/clear.js",
     "db:create": "node scripts/commands/database/create.js",
     "db:matrix": "node scripts/commands/database/matrix.js",
     "db:update": "node scripts/commands/database/update.js",
@@ -13,7 +14,6 @@
     "playlist:generate": "node scripts/commands/playlist/generate.js",
     "playlist:update": "node scripts/commands/playlist/update.js",
     "playlist:lint": "npx m3u-linter -c m3u-linter.json",
-    "playlist:cleaner": "node scripts/commands/playlist/cleaner.js",
     "readme:update": "node scripts/commands/readme/update.js",
     "test": "jest --runInBand"
   },

scripts/commands/database/clear.js

@@ -0,0 +1,35 @@
+const { logger, parser, db, date } = require('../../core')
+const { program } = require('commander')
+
+const options = program
+  .option(
+    '-t, --threshold <threshold>',
+    'Number of days after which the stream should be deleted',
+    parser.parseNumber,
+    7
+  )
+  .option('--input-dir <input-dir>', 'Set path to input directory', 'streams')
+  .parse(process.argv)
+  .opts()
+
+async function main() {
+  await db.streams.load()
+
+  const streams = await db.streams.all()
+
+  let total = 0
+  for (const stream of streams) {
+    if (
+      stream.status === 'error' &&
+      date.utc().diff(stream.updated_at, 'day') >= options.threshold
+    ) {
+      total += await db.streams.remove({ url: stream.url }, { multi: true })
+    }
+  }
+
+  await db.streams.compact()
+
+  logger.info(`removed ${total} streams`)
+}
+
+main()

scripts/commands/database/create.js

@@ -28,6 +28,7 @@ async function findStreams() {
   logger.info(`looking for streams...`)
 
   await api.channels.load()
+  await api.streams.load()
   await db.streams.load()
 
   const streams = []
@@ -39,6 +40,7 @@ async function findStreams() {
 
       const stream = store.create()
       const channel = await api.channels.find({ id: item.tvg.id })
+      const cached = (await api.streams.find({ url: item.url })) || {}
 
       stream.set('channel', { channel: channel ? channel.id : null })
       stream.set('title', { title: item.name })
@@ -46,6 +48,14 @@ async function findStreams() {
       stream.set('url', { url: item.url })
       stream.set('http_referrer', { http_referrer: item.http.referrer })
       stream.set('user_agent', { user_agent: item.http['user-agent'] })
+      stream.set('status', { status: cached.status })
+      stream.set('width', { width: cached.width })
+      stream.set('height', { height: cached.height })
+      stream.set('bitrate', { bitrate: cached.bitrate })
+      stream.set('frame_rate', { frame_rate: cached.frame_rate })
+      stream.set('added_at', { added_at: cached.added_at })
+      stream.set('updated_at', { updated_at: cached.updated_at })
+      stream.set('checked_at', { checked_at: cached.checked_at })
 
       streams.push(stream)
     }

scripts/commands/playlist/cleaner.js

@@ -1,90 +0,0 @@
-const { file, parser, logger, checker, m3u } = require('../../core')
-const { program } = require('commander')
-
-program
-  .argument('[filepath]', 'Path to file to validate')
-  .option('-t, --timeout <timeout>', 'Set timeout for each request', parser.parseNumber, 60000)
-  .option('-d, --delay <delay>', 'Set delay for each request', parser.parseNumber, 0)
-  .option('--debug', 'Enable debug mode')
-  .parse(process.argv)
-
-const options = program.opts()
-
-async function main() {
-  const files = program.args.length ? program.args : await file.list('streams/*.m3u')
-
-  for (const filepath of files) {
-    if (!filepath.endsWith('.m3u')) continue
-    logger.info(`${filepath}`)
-    const playlist = await parser.parsePlaylist(filepath)
-    const before = playlist.items.length
-    for (const stream of playlist.items) {
-      if (options.debug) logger.info(stream.url)
-      const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null]
-      stream.status = status
-      if (status === 'error' && /^(http|https)/.test(stream.url)) {
-        const result = await checkStream(stream)
-        const newStatus = parseStatus(result.error)
-        if (status === newStatus) {
-          stream.remove = true
-          logger.info(`removed "${stream.name}"`)
-        }
-      }
-    }
-
-    const items = playlist.items
-      .filter(i => !i.remove)
-      .map(item => ({
-        attrs: {
-          'tvg-id': item.tvg.id,
-          status: item.status,
-          'user-agent': item.http['user-agent'] || undefined
-        },
-        title: item.name,
-        url: item.url,
-        vlcOpts: {
-          'http-referrer': item.http.referrer || undefined,
-          'http-user-agent': item.http['user-agent'] || undefined
-        }
-      }))
-
-    if (before !== items.length) {
-      const output = m3u.create(items)
-      await file.create(filepath, output)
-      logger.info(`saved`)
-    }
-  }
-}
-
-main()
-
-async function checkStream(item) {
-  const config = {
-    timeout: options.timeout,
-    delay: options.delay,
-    debug: options.debug
-  }
-
-  const request = {
-    url: item.url,
-    http: {
-      referrer: item.http.referrer,
-      'user-agent': item.http['user-agent']
-    }
-  }
-
-  return checker.check(request, config)
-}
-
-function parseStatus(error) {
-  if (!error) return 'online'
-
-  switch (error) {
-    case 'Operation timed out':
-      return 'timeout'
-    case 'Server returned 403 Forbidden (access denied)':
-      return 'blocked'
-    default:
-      return 'error'
-  }
-}

scripts/core/date.js

@@ -0,0 +1,12 @@
+const dayjs = require('dayjs')
+const utc = require('dayjs/plugin/utc')
+
+dayjs.extend(utc)
+
+const date = {}
+
+date.utc = d => {
+	return dayjs.utc(d)
+}
+
+module.exports = date

scripts/core/db.js

@@ -63,6 +63,10 @@ class Database {
     return this.db.find(query)
   }
 
+  all() {
+    return this.find({})
+  }
+
   remove(query, options) {
     return this.db.remove(query, options)
   }

scripts/core/index.js

@@ -11,3 +11,4 @@ exports.markdown = require('./markdown')
 exports.api = require('./api')
 exports.id = require('./id')
 exports.m3u = require('./m3u')
+exports.date = require('./date')

tests/__data__/expected/database/db_clear.streams.db

@@ -0,0 +1,6 @@
+{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"}
+{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"}
+{"title":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
+{"title":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"tests/__data__/output/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
+{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"}
+{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"}

tests/__data__/expected/database/db_create.streams.db

@@ -1,6 +1,7 @@
-{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"TyQaTTYos0fr2q0P"}
+{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"UAYbCvjOWvqXHH95"}
-{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"yNDfQt0ITDrOGGV2"}
+{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"IjKHoixx2crsuOlE"}
-{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"asTdyPDWW77mXDLZ"}
+{"channel":"LibyasChannel.ly","title":"Libyas Channel","filepath":"tests/__data__/input/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","updated_at":"2022-07-07T00:00:00Z","checked_at":"2022-07-07T00:00:00Z","cluster_id":1,"_id":"I0rJlwp3rZEy2SnG"}
-{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"1gBgkVYcwsNJQlso"}
+{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"tdzk1IN7wLJxfGab"}
-{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"8F6RyHFzpOe20huV"}
+{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error","added_at":"2022-07-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z","cluster_id":1,"_id":"3evMHt3nFsZjhzas"}
-{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"9DY8CqVcKyp8jqiA"}
+{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"agJlTEr8wwpbWgw0"}
+{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"FpMhoWjtpnVuYlO9"}

tests/__data__/input/data/channels.json

@@ -123,6 +123,23 @@
     "is_nsfw": false,
     "logo": "https://iptvx.one/icn/ldpr-tv.png"
   },
+  {
+    "id": "LibyasChannel.ly",
+    "name": "Libyas Channel",
+    "network": null,
+    "country": "LY",
+    "subdivision": null,
+    "city": null,
+    "broadcast_area": [
+      "c/LY"
+    ],
+    "languages": [
+      "eng"
+    ],
+    "categories": [],
+    "is_nsfw": false,
+    "logo": "https://i.imgur.com/RD9wbNF.jpg"
+  },
   {
     "id": "MeteoMedia.ca",
     "name": "MétéoMédia",

tests/__data__/input/database/db_clear.streams.db

@@ -0,0 +1,7 @@
+{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"}
+{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"}
+{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz","status":"error","added_at":"2022-07-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z"}
+{"title":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
+{"title":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"tests/__data__/output/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
+{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"}
+{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"}

tests/__data__/input/streams/ly.m3u

@@ -0,0 +1,3 @@
+#EXTM3U
+#EXTINF:-1 tvg-id="LibyasChannel.ly",Libyas Channel
+https://master.starmena-cloud.com/hls/libyas.m3u8

tests/commands/database/clear.test.js

@@ -0,0 +1,48 @@
+const fs = require('fs-extra')
+const path = require('path')
+const { execSync } = require('child_process')
+
+beforeEach(() => {
+  fs.emptyDirSync('tests/__data__/output')
+  fs.mkdirSync('tests/__data__/output/database')
+  fs.copyFileSync(
+    'tests/__data__/input/database/db_clear.streams.db',
+    'tests/__data__/output/database/streams.db'
+  )
+
+  const stdout = execSync(
+    'DB_DIR=tests/__data__/output/database npm run db:clear -- --threshold 7',
+    {
+      encoding: 'utf8'
+    }
+  )
+})
+
+it('can clear database', () => {
+  let output = content('tests/__data__/output/database/streams.db')
+  let expected = content('tests/__data__/expected/database/db_clear.streams.db')
+
+  output = output.map(i => {
+    i._id = null
+    return i
+  })
+  expected = expected.map(i => {
+    i._id = null
+    return i
+  })
+
+  expect(output).toMatchObject(expected)
+})
+
+function content(filepath) {
+  const data = fs.readFileSync(path.resolve(filepath), {
+    encoding: 'utf8'
+  })
+
+  return data
+    .split('\n')
+    .filter(l => l)
+    .map(l => {
+      return JSON.parse(l)
+    })
+}

tests/commands/database/create.test.js

@@ -4,9 +4,10 @@ const { execSync } = require('child_process')
 
 beforeEach(() => {
   fs.emptyDirSync('tests/__data__/output')
+  fs.mkdirSync('tests/__data__/output/database')
 
   const stdout = execSync(
-    'DB_DIR=tests/__data__/output/database npm run db:create -- --input-dir=tests/__data__/input/streams --max-clusters=1',
+    'DB_DIR=tests/__data__/output/database DATA_DIR=tests/__data__/input/data npm run db:create -- --input-dir=tests/__data__/input/streams --max-clusters=1',
     { encoding: 'utf8' }
   )
 })
@@ -24,14 +25,7 @@ it('can create database', () => {
     return i
   })
 
-  expect(output).toEqual(
+  expect(output).toMatchObject(expect.arrayContaining(expected))
-    expect.arrayContaining([
-      expect.objectContaining(expected[0]),
-      expect.objectContaining(expected[1]),
-      expect.objectContaining(expected[2]),
-      expect.objectContaining(expected[3])
-    ])
-  )
 })
 
 function content(filepath) {

tests/commands/database/export.test.js

@@ -5,13 +5,14 @@ const dayjs = require('dayjs')
 
 beforeEach(() => {
   fs.emptyDirSync('tests/__data__/output')
+  fs.mkdirSync('tests/__data__/output/database')
   fs.copyFileSync(
     'tests/__data__/input/database/db_export.streams.db',
-    'tests/__data__/output/streams.db'
+    'tests/__data__/output/database/streams.db'
   )
 
   const stdout = execSync(
-    'DB_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.api npm run db:export',
+    'DB_DIR=tests/__data__/output/database DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.api npm run db:export',
     { encoding: 'utf8' }
   )
 })

tests/commands/database/matrix.test.js

@@ -4,15 +4,15 @@ const { execSync } = require('child_process')
 
 beforeEach(() => {
   fs.emptyDirSync('tests/__data__/output')
-
+  fs.mkdirSync('tests/__data__/output/database')
   fs.copyFileSync(
     'tests/__data__/input/database/db_matrix.streams.db',
-    'tests/__data__/output/streams.db'
+    'tests/__data__/output/database/streams.db'
   )
 })
 
 it('can create valid matrix', () => {
-  const result = execSync('DB_DIR=tests/__data__/output npm run db:matrix', {
+  const result = execSync('DB_DIR=tests/__data__/output/database npm run db:matrix', {
     encoding: 'utf8'
   })
   expect(result).toBe(

tests/commands/database/update.test.js

@@ -4,15 +4,16 @@ const path = require('path')
 
 beforeEach(() => {
   fs.emptyDirSync('tests/__data__/output')
+  fs.mkdirSync('tests/__data__/output/database')
   fs.copyFileSync(
     'tests/__data__/input/database/db_update.streams.db',
-    'tests/__data__/output/streams.db'
+    'tests/__data__/output/database/streams.db'
   )
 })
 
 it('can save results', () => {
   const stdout = execSync(
-    'DB_DIR=tests/__data__/output LOGS_DIR=tests/__data__/input/logs/cluster/load npm run db:update',
+    'DB_DIR=tests/__data__/output/database LOGS_DIR=tests/__data__/input/logs/cluster/load npm run db:update',
     { encoding: 'utf8' }
   )
   expect(stdout).toEqual(`
@@ -31,7 +32,7 @@ removed 1 duplicates
 done
 `)
 
-  expect(content('tests/__data__/output/streams.db')).toEqual(
+  expect(content('tests/__data__/output/database/streams.db')).toEqual(
     content('tests/__data__/expected/database/db_update.streams.db')
   )
 })