Merge branch 'master' into patch-24

2022-08-13 00:26:37 +02:00 · 2022-08-13 00:26:37 +02:00 · 08d65d28b4
parent e3eddbdfac ba6fa5adb5
commit 08d65d28b4
13 changed files with 103 additions and 24 deletions
--- a/.github/workflows/auto-update.yml
+++ b/.github/workflows/auto-update.yml
@ -97,6 +97,10 @@ jobs:
      - run: npm run playlist:update
      - run: npm run playlist:generate
      - run: npm run db:export
+      - uses: actions/upload-artifact@v2
+        with:
+          name: api
+          path: .api
      - run: npm run readme:update
      - uses: actions/upload-artifact@v2
        with:
--- a/package-lock.json
+++ b/package-lock.json
@ -1259,13 +1259,19 @@
      }
    },
    "node_modules/caniuse-lite": {
-      "version": "1.0.30001312",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz",
-      "integrity": "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ==",
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/browserslist"
-      }
+      "version": "1.0.30001374",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz",
+      "integrity": "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw==",
+      "funding": [
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/browserslist"
+        },
+        {
+          "type": "tidelift",
+          "url": "https://tidelift.com/funding/github/npm/caniuse-lite"
+        }
+      ]
    },
    "node_modules/chalk": {
      "version": "4.1.2",
@ -5200,9 +5206,9 @@
      "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg=="
    },
    "caniuse-lite": {
-      "version": "1.0.30001312",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz",
-      "integrity": "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ=="
+      "version": "1.0.30001374",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz",
+      "integrity": "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw=="
    },
    "chalk": {
      "version": "4.1.2",
--- a/scripts/commands/api/load.sh
+++ b/scripts/commands/api/load.sh
@ -4,6 +4,7 @@ mkdir -p scripts/data
 curl -L -o scripts/data/blocklist.json https://iptv-org.github.io/api/blocklist.json 
 curl -L -o scripts/data/categories.json https://iptv-org.github.io/api/categories.json 
 curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json 
+curl -L -o scripts/data/streams.json https://iptv-org.github.io/api/streams.json 
 curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json 
 curl -L -o scripts/data/guides.json https://iptv-org.github.io/api/guides.json 
 curl -L -o scripts/data/languages.json https://iptv-org.github.io/api/languages.json 
--- a/scripts/commands/database/export.js
+++ b/scripts/commands/database/export.js
@ -1,14 +1,19 @@
-const { logger, db, file } = require('../../core')
+const { logger, db, api, file } = require('../../core')
 const _ = require('lodash')
+const dayjs = require('dayjs')
+const utc = require('dayjs/plugin/utc')
+dayjs.extend(utc)

 const PUBLIC_DIR = process.env.PUBLIC_DIR || '.api'

 async function main() {
+  await api.streams.load()
  await db.streams.load()
+  const now = dayjs.utc().format()
  let streams = await db.streams.find({})
  streams = _.sortBy(streams, 'channel')
  streams = streams.map(stream => {
-    return {
+    const data = {
      channel: stream.channel,
      url: stream.url,
      http_referrer: stream.http_referrer,
@ -19,6 +24,26 @@ async function main() {
      bitrate: stream.bitrate,
      frame_rate: stream.frame_rate
    }
+
+    let addedAt = now
+    let updatedAt = now
+    let found = api.streams.find({ url: stream.url })
+    if (found) {
+      normalized = _.omit(found, ['added_at', 'updated_at', 'checked_at'])
+      if (_.isEqual(data, normalized)) {
+        addedAt = found.added_at || now
+        updatedAt = found.updated_at || now
+      } else {
+        addedAt = found.added_at || now
+        updatedAt = now
+      }
+    }
+
+    data.added_at = addedAt
+    data.updated_at = updatedAt
+    data.checked_at = now
+
+    return data
  })

  await file.create(`${PUBLIC_DIR}/streams.json`, JSON.stringify(streams))
--- a/scripts/core/api.js
+++ b/scripts/core/api.js
@ -29,6 +29,7 @@ class API {
 const api = {}

 api.channels = new API(`${DATA_DIR}/channels.json`)
+api.streams = new API(`${DATA_DIR}/streams.json`)
 api.countries = new API(`${DATA_DIR}/countries.json`)
 api.guides = new API(`${DATA_DIR}/guides.json`)
 api.categories = new API(`${DATA_DIR}/categories.json`)
--- a/streams/dk.m3u
+++ b/streams/dk.m3u
@ -1,4 +1,10 @@
 #EXTM3U
+#EXTINF:-1 tvg-id="DR1.dk",DR1 (1080p) [Geo-blocked]
+https://drlive01hls.akamaized.net/hls/live/2014185/drlive01/master.m3u8
+#EXTINF:-1 tvg-id="DR2.dk",DR2 (1080p) [Geo-blocked]
+https://drlive02hls.akamaized.net/hls/live/2014187/drlive02/master.m3u8
+#EXTINF:-1 tvg-id="DRRamaSjang.dk",DR Ramasjang (720p) [Geo-blocked]
+https://drlive03hls.akamaized.net/hls/live/2014190/drlive03/master.m3u8
 #EXTINF:-1 tvg-id="KanalHovedstaden.dk" status="online",Kanal Hovedstaden (720p)
 https://59b954022ec35.streamlock.net/liveTV2/smil:liveTVstream2.transcoder.smil/playlist.m3u8
 #EXTINF:-1 tvg-id="KKRtv.dk" status="online",KKRtv (720p)
--- a/streams/py.m3u
+++ b/streams/py.m3u
@ -61,3 +61,7 @@ http://unicanal.com.py/live/unicanal.m3u8
 http://45.55.127.106/live/unicanal_mid.m3u8
 #EXTINF:-1 tvg-id="Uniradio.py" status="online",Uniradio
 https://59ce1298bfb98.streamlock.net/uniradiotv/uniradiotv/playlist.m3u8
+#EXTINF:-1 tvg-id="SenadoTV.py" status="online",Senado TV
+https://59ce1298bfb98.streamlock.net/tvsenadopy/tvsenadopy/playlist.m3u8
+#EXTINF:-1 tvg-id="SomosdelEste.py" status="online",Somos Del Este
+https://59ce1298bfb98.streamlock.net/somosdeleste/somosdeleste/playlist.m3u8
--- a/tests/data/expected/.api/streams.json
+++ b/tests/data/expected/.api/streams.json
@ -1 +1 @@
-[{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error"},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"blocked"},{"channel":"BBCNewsHD.ad","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25},{"channel":"KayhanTV.af","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"Sharq.af","url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1280,"height":720,"bitrate":2226543,"frame_rate":25}]
+[{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error","added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"blocked","added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"BBCNewsHD.uk","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"KayhanTV.af","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-07-07T00:00:00Z"},{"channel":"Sharq.af","url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1280,"height":720,"bitrate":2226543,"frame_rate":25,"added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"}]
--- a/tests/data/input/data/streams.json
+++ b/tests/data/input/data/streams.json
@ -0,0 +1 @@
+[{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1920,"height":1080,"bitrate":6542556,"frame_rate":50,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z"},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-07-07T00:00:00Z","updated_at":"2022-07-07T00:00:00Z"}]
--- a/tests/data/input/database/db_export.streams.db
+++ b/tests/data/input/database/db_export.streams.db
@ -1,6 +1,7 @@
 {"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"}
 {"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"}
 {"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz","status":"error"}
-{"title":"BBC News HD","channel":"BBCNewsHD.ad","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
+{"title":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
 {"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"}
 {"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"}
+{"title":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"tests/__data__/output/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
--- a/tests/commands/database/export.test.js
+++ b/tests/commands/database/export.test.js
@ -1,5 +1,7 @@
 const { execSync } = require('child_process')
 const fs = require('fs-extra')
+const _ = require('lodash')
+const dayjs = require('dayjs')

 beforeEach(() => {
  fs.emptyDirSync('tests/__data__/output')
@ -9,17 +11,45 @@ beforeEach(() => {
  )

  const stdout = execSync(
-    'DB_DIR=tests/__data__/output PUBLIC_DIR=tests/__data__/output/.api npm run db:export',
+    'DB_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.api npm run db:export',
    { encoding: 'utf8' }
  )
 })

 it('can create streams.json', () => {
-  expect(content(`output/.api/streams.json`)).toBe(content(`expected/.api/streams.json`))
+  let api = content('input/data/streams.json')
+  let output = content(`output/.api/streams.json`)
+  let expected = content(`expected/.api/streams.json`)
+
+  const updatedUrl = 'https://master.starmena-cloud.com/hls/libyas.m3u8'
+  let outputData = output.find(i => i.url === updatedUrl)
+  let savedData = api.find(i => i.url === updatedUrl)
+  expect(outputData.added_at).toBe(savedData.added_at)
+  expect(outputData.updated_at).toBe(savedData.updated_at)
+  expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0)
+
+  const sameUrl = 'http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8'
+  outputData = output.find(i => i.url === sameUrl)
+  savedData = api.find(i => i.url === sameUrl)
+  expect(outputData.added_at).toBe(savedData.added_at)
+  expect(dayjs().diff(outputData.updated_at, 'h')).toBe(0)
+  expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0)
+
+  const addedUrl = 'https://master.starmena-cloud.com/hls/bbc.m3u8'
+  outputData = output.find(i => i.url === addedUrl)
+  expect(dayjs().diff(outputData.added_at, 'h')).toBe(0)
+  expect(dayjs().diff(outputData.updated_at, 'h')).toBe(0)
+  expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0)
+
+  expect(output.map(item => _.omit(item, ['added_at', 'updated_at', 'checked_at']))).toMatchObject(
+    expected.map(item => _.omit(item, ['added_at', 'updated_at', 'checked_at']))
+  )
 })

 function content(filepath) {
-  return fs.readFileSync(`tests/__data__/${filepath}`, {
-    encoding: 'utf8'
-  })
+  return JSON.parse(
+    fs.readFileSync(`tests/__data__/${filepath}`, {
+      encoding: 'utf8'
+    })
+  )
 }
--- a/tests/commands/playlist/validate.test.js
+++ b/tests/commands/playlist/validate.test.js
@ -13,7 +13,7 @@ it('show an error if channel name in the blocklist', () => {
  } catch (err) {
    expect(err.status).toBe(1)
    expect(err.stdout).toBe(
-      `\n> playlist:validate\n> node scripts/commands/playlist/validate.js "tests/__data__/input/streams/us_blocked.m3u"\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/us_blocked.m3u\n 2     error    "Fox Sports 2 Asia" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n`
+      `\n> playlist:validate\n> node scripts/commands/playlist/validate.js\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/us_blocked.m3u\n 2     error    "Fox Sports 2 Asia" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n`
    )
  }
 })
@ -27,6 +27,6 @@ it('show a warning if channel has wrong id', () => {
  )

  expect(stdout).toBe(
-    `\n> playlist:validate\n> node scripts/commands/playlist/validate.js "tests/__data__/input/streams/wrong_id.m3u"\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/wrong_id.m3u\n 2     warning  "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n`
+    `\n> playlist:validate\n> node scripts/commands/playlist/validate.js\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/wrong_id.m3u\n 2     warning  "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n`
  )
 })
--- a/yarn.lock
+++ b/yarn.lock
@ -848,9 +848,9 @@
  "version" "6.3.0"

 "caniuse-lite@^1.0.30001286":
-  "integrity" "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ=="
-  "resolved" "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz"
-  "version" "1.0.30001312"
+  "integrity" "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw=="
+  "resolved" "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz"
+  "version" "1.0.30001374"

 "chalk@^2.0.0":
  "integrity" "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ=="
				`@ -0,0 +1 @@`
				[{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1920,"height":1080,"bitrate":6542556,"frame_rate":50,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z"},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-07-07T00:00:00Z","updated_at":"2022-07-07T00:00:00Z"}]