Merge pull request #8003 from iptv-org/update-streams-json

Add new properties to streams.json

.github/workflows/auto-update.yml

@@ -97,6 +97,10 @@ jobs:
       - run: npm run playlist:update
       - run: npm run playlist:generate
       - run: npm run db:export
+      - uses: actions/upload-artifact@v2
+        with:
+          name: api
+          path: .api
       - run: npm run readme:update
       - uses: actions/upload-artifact@v2
         with:

package-lock.json

@@ -1259,13 +1259,19 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001312",
+      "version": "1.0.30001374",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz",
-      "integrity": "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ==",
+      "integrity": "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw==",
-      "funding": {
+      "funding": [
+        {
           "type": "opencollective",
           "url": "https://opencollective.com/browserslist"
+        },
+        {
+          "type": "tidelift",
+          "url": "https://tidelift.com/funding/github/npm/caniuse-lite"
         }
+      ]
     },
     "node_modules/chalk": {
       "version": "4.1.2",
@@ -5200,9 +5206,9 @@
       "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg=="
     },
     "caniuse-lite": {
-      "version": "1.0.30001312",
+      "version": "1.0.30001374",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz",
-      "integrity": "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ=="
+      "integrity": "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw=="
     },
     "chalk": {
       "version": "4.1.2",

scripts/commands/api/load.sh

@@ -4,6 +4,7 @@ mkdir -p scripts/data
 curl -L -o scripts/data/blocklist.json https://iptv-org.github.io/api/blocklist.json 
 curl -L -o scripts/data/categories.json https://iptv-org.github.io/api/categories.json 
 curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json 
+curl -L -o scripts/data/streams.json https://iptv-org.github.io/api/streams.json 
 curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json 
 curl -L -o scripts/data/guides.json https://iptv-org.github.io/api/guides.json 
 curl -L -o scripts/data/languages.json https://iptv-org.github.io/api/languages.json 

scripts/commands/database/export.js

@@ -1,14 +1,19 @@
-const { logger, db, file } = require('../../core')
+const { logger, db, api, file } = require('../../core')
 const _ = require('lodash')
+const dayjs = require('dayjs')
+const utc = require('dayjs/plugin/utc')
+dayjs.extend(utc)
 
 const PUBLIC_DIR = process.env.PUBLIC_DIR || '.api'
 
 async function main() {
+  await api.streams.load()
   await db.streams.load()
+  const now = dayjs.utc().format()
   let streams = await db.streams.find({})
   streams = _.sortBy(streams, 'channel')
   streams = streams.map(stream => {
-    return {
+    const data = {
       channel: stream.channel,
       url: stream.url,
       http_referrer: stream.http_referrer,
@@ -19,6 +24,26 @@ async function main() {
       bitrate: stream.bitrate,
       frame_rate: stream.frame_rate
     }
+
+    let addedAt = now
+    let updatedAt = now
+    let found = api.streams.find({ url: stream.url })
+    if (found) {
+      normalized = _.omit(found, ['added_at', 'updated_at', 'checked_at'])
+      if (_.isEqual(data, normalized)) {
+        addedAt = found.added_at || now
+        updatedAt = found.updated_at || now
+      } else {
+        addedAt = found.added_at || now
+        updatedAt = now
+      }
+    }
+
+    data.added_at = addedAt
+    data.updated_at = updatedAt
+    data.checked_at = now
+
+    return data
   })
 
   await file.create(`${PUBLIC_DIR}/streams.json`, JSON.stringify(streams))

scripts/core/api.js

@@ -29,6 +29,7 @@ class API {
 const api = {}
 
 api.channels = new API(`${DATA_DIR}/channels.json`)
+api.streams = new API(`${DATA_DIR}/streams.json`)
 api.countries = new API(`${DATA_DIR}/countries.json`)
 api.guides = new API(`${DATA_DIR}/guides.json`)
 api.categories = new API(`${DATA_DIR}/categories.json`)

tests/__data__/expected/.api/streams.json

@@ -1 +1 @@
-[{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error"},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"blocked"},{"channel":"BBCNewsHD.ad","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25},{"channel":"KayhanTV.af","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"Sharq.af","url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1280,"height":720,"bitrate":2226543,"frame_rate":25}]
+[{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error","added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"blocked","added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"BBCNewsHD.uk","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"KayhanTV.af","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-07-07T00:00:00Z"},{"channel":"Sharq.af","url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1280,"height":720,"bitrate":2226543,"frame_rate":25,"added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"}]

tests/__data__/input/data/streams.json

@@ -0,0 +1 @@
+[{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1920,"height":1080,"bitrate":6542556,"frame_rate":50,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z"},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-07-07T00:00:00Z","updated_at":"2022-07-07T00:00:00Z"}]

tests/__data__/input/database/db_export.streams.db

@@ -1,6 +1,7 @@
 {"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"}
 {"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"}
 {"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz","status":"error"}
-{"title":"BBC News HD","channel":"BBCNewsHD.ad","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
+{"title":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
 {"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"}
 {"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"}
+{"title":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"tests/__data__/output/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}

tests/commands/database/export.test.js

@@ -1,5 +1,7 @@
 const { execSync } = require('child_process')
 const fs = require('fs-extra')
+const _ = require('lodash')
+const dayjs = require('dayjs')
 
 beforeEach(() => {
   fs.emptyDirSync('tests/__data__/output')
@@ -9,17 +11,45 @@ beforeEach(() => {
   )
 
   const stdout = execSync(
-    'DB_DIR=tests/__data__/output PUBLIC_DIR=tests/__data__/output/.api npm run db:export',
+    'DB_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.api npm run db:export',
     { encoding: 'utf8' }
   )
 })
 
 it('can create streams.json', () => {
-  expect(content(`output/.api/streams.json`)).toBe(content(`expected/.api/streams.json`))
+  let api = content('input/data/streams.json')
+  let output = content(`output/.api/streams.json`)
+  let expected = content(`expected/.api/streams.json`)
+
+  const updatedUrl = 'https://master.starmena-cloud.com/hls/libyas.m3u8'
+  let outputData = output.find(i => i.url === updatedUrl)
+  let savedData = api.find(i => i.url === updatedUrl)
+  expect(outputData.added_at).toBe(savedData.added_at)
+  expect(outputData.updated_at).toBe(savedData.updated_at)
+  expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0)
+
+  const sameUrl = 'http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8'
+  outputData = output.find(i => i.url === sameUrl)
+  savedData = api.find(i => i.url === sameUrl)
+  expect(outputData.added_at).toBe(savedData.added_at)
+  expect(dayjs().diff(outputData.updated_at, 'h')).toBe(0)
+  expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0)
+
+  const addedUrl = 'https://master.starmena-cloud.com/hls/bbc.m3u8'
+  outputData = output.find(i => i.url === addedUrl)
+  expect(dayjs().diff(outputData.added_at, 'h')).toBe(0)
+  expect(dayjs().diff(outputData.updated_at, 'h')).toBe(0)
+  expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0)
+
+  expect(output.map(item => _.omit(item, ['added_at', 'updated_at', 'checked_at']))).toMatchObject(
+    expected.map(item => _.omit(item, ['added_at', 'updated_at', 'checked_at']))
+  )
 })
 
 function content(filepath) {
-  return fs.readFileSync(`tests/__data__/${filepath}`, {
+  return JSON.parse(
+    fs.readFileSync(`tests/__data__/${filepath}`, {
       encoding: 'utf8'
     })
+  )
 }

tests/commands/playlist/validate.test.js

@@ -13,7 +13,7 @@ it('show an error if channel name in the blocklist', () => {
   } catch (err) {
     expect(err.status).toBe(1)
     expect(err.stdout).toBe(
-      `\n> playlist:validate\n> node scripts/commands/playlist/validate.js "tests/__data__/input/streams/us_blocked.m3u"\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/us_blocked.m3u\n 2     error    "Fox Sports 2 Asia" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n`
+      `\n> playlist:validate\n> node scripts/commands/playlist/validate.js\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/us_blocked.m3u\n 2     error    "Fox Sports 2 Asia" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n`
     )
   }
 })
@@ -27,6 +27,6 @@ it('show a warning if channel has wrong id', () => {
   )
 
   expect(stdout).toBe(
-    `\n> playlist:validate\n> node scripts/commands/playlist/validate.js "tests/__data__/input/streams/wrong_id.m3u"\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/wrong_id.m3u\n 2     warning  "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n`
+    `\n> playlist:validate\n> node scripts/commands/playlist/validate.js\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/wrong_id.m3u\n 2     warning  "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n`
   )
 })

yarn.lock

@@ -848,9 +848,9 @@
   "version" "6.3.0"
 
 "caniuse-lite@^1.0.30001286":
-  "integrity" "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ=="
+  "integrity" "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw=="
-  "resolved" "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz"
+  "resolved" "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz"
-  "version" "1.0.30001312"
+  "version" "1.0.30001374"
 
 "chalk@^2.0.0":
   "integrity" "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ=="