From 6c71c4a55ad1faa2f7e9ebfcc1944649d4bc72a2 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 03:39:44 +0300 Subject: [PATCH 01/11] Update streams.json --- tests/__data__/expected/.api/streams.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__data__/expected/.api/streams.json b/tests/__data__/expected/.api/streams.json index fd650e3261..5901fa753a 100644 --- a/tests/__data__/expected/.api/streams.json +++ b/tests/__data__/expected/.api/streams.json @@ -1 +1 @@ -[{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error"},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"blocked"},{"channel":"BBCNewsHD.ad","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25},{"channel":"KayhanTV.af","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"Sharq.af","url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1280,"height":720,"bitrate":2226543,"frame_rate":25}] \ No newline at end of file +[{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error","added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"blocked","added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"BBCNewsHD.uk","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"KayhanTV.af","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-07-07T00:00:00Z"},{"channel":"Sharq.af","url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1280,"height":720,"bitrate":2226543,"frame_rate":25,"added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"}] \ No newline at end of file From e4a4e390c942d01a4523f53130985c383b236d8f Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 03:39:49 +0300 Subject: [PATCH 02/11] Create streams.json --- tests/__data__/input/data/streams.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/__data__/input/data/streams.json diff --git a/tests/__data__/input/data/streams.json b/tests/__data__/input/data/streams.json new file mode 100644 index 0000000000..69f43a964f --- /dev/null +++ b/tests/__data__/input/data/streams.json @@ -0,0 +1 @@ +[{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1920,"height":1080,"bitrate":6542556,"frame_rate":50,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z"},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-07-07T00:00:00Z","updated_at":"2022-07-07T00:00:00Z"}] \ No newline at end of file From bb83b8cf2ef4f17a355394831609286926a833f8 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 03:39:56 +0300 Subject: [PATCH 03/11] Update db_export.streams.db --- tests/__data__/input/database/db_export.streams.db | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/__data__/input/database/db_export.streams.db b/tests/__data__/input/database/db_export.streams.db index 1c4b73c92d..b6e975cf01 100644 --- a/tests/__data__/input/database/db_export.streams.db +++ b/tests/__data__/input/database/db_export.streams.db @@ -1,6 +1,7 @@ {"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"} {"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"} {"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz","status":"error"} -{"title":"BBC News HD","channel":"BBCNewsHD.ad","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576} +{"title":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576} {"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"} {"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"} +{"title":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"tests/__data__/output/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576} From 7fb0a570f473d0d67c746e2e7bf3a55048e030ff Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 03:40:07 +0300 Subject: [PATCH 04/11] Update load.sh --- scripts/commands/api/load.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/commands/api/load.sh b/scripts/commands/api/load.sh index 0364dccf21..133d2eac0b 100755 --- a/scripts/commands/api/load.sh +++ b/scripts/commands/api/load.sh @@ -4,6 +4,7 @@ mkdir -p scripts/data curl -L -o scripts/data/blocklist.json https://iptv-org.github.io/api/blocklist.json curl -L -o scripts/data/categories.json https://iptv-org.github.io/api/categories.json curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json +curl -L -o scripts/data/streams.json https://iptv-org.github.io/api/streams.json curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json curl -L -o scripts/data/guides.json https://iptv-org.github.io/api/guides.json curl -L -o scripts/data/languages.json https://iptv-org.github.io/api/languages.json From 27b348efb0d2a1001086b004c6ba127e9e5b6c0b Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 03:40:15 +0300 Subject: [PATCH 05/11] Update api.js --- scripts/core/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/core/api.js b/scripts/core/api.js index 770f3a7eac..901f0ef26e 100644 --- a/scripts/core/api.js +++ b/scripts/core/api.js @@ -29,6 +29,7 @@ class API { const api = {} api.channels = new API(`${DATA_DIR}/channels.json`) +api.streams = new API(`${DATA_DIR}/streams.json`) api.countries = new API(`${DATA_DIR}/countries.json`) api.guides = new API(`${DATA_DIR}/guides.json`) api.categories = new API(`${DATA_DIR}/categories.json`) From c4c91400950f696e64800b39c11ee7613f567dc2 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 03:41:06 +0300 Subject: [PATCH 06/11] Update export.js --- scripts/commands/database/export.js | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/scripts/commands/database/export.js b/scripts/commands/database/export.js index 12990a0820..675a848479 100644 --- a/scripts/commands/database/export.js +++ b/scripts/commands/database/export.js @@ -1,14 +1,19 @@ -const { logger, db, file } = require('../../core') +const { logger, db, api, file } = require('../../core') const _ = require('lodash') +const dayjs = require('dayjs') +const utc = require('dayjs/plugin/utc') +dayjs.extend(utc) const PUBLIC_DIR = process.env.PUBLIC_DIR || '.api' async function main() { + await api.streams.load() await db.streams.load() + const now = dayjs.utc().format() let streams = await db.streams.find({}) streams = _.sortBy(streams, 'channel') streams = streams.map(stream => { - return { + const data = { channel: stream.channel, url: stream.url, http_referrer: stream.http_referrer, @@ -19,6 +24,26 @@ async function main() { bitrate: stream.bitrate, frame_rate: stream.frame_rate } + + let addedAt = now + let updatedAt = now + let found = api.streams.find({ url: stream.url }) + if (found) { + normalized = _.omit(found, ['added_at', 'updated_at', 'checked_at']) + if (_.isEqual(data, normalized)) { + addedAt = found.added_at || now + updatedAt = found.updated_at || now + } else { + addedAt = found.added_at || now + updatedAt = now + } + } + + data.added_at = addedAt + data.updated_at = updatedAt + data.checked_at = now + + return data }) await file.create(`${PUBLIC_DIR}/streams.json`, JSON.stringify(streams)) From 1b7823bb5c68cc006344a73416351f32a76da69c Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 03:41:21 +0300 Subject: [PATCH 07/11] Update export.test.js --- tests/commands/database/export.test.js | 40 ++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/tests/commands/database/export.test.js b/tests/commands/database/export.test.js index b0ab866ed7..05a3364961 100644 --- a/tests/commands/database/export.test.js +++ b/tests/commands/database/export.test.js @@ -1,5 +1,7 @@ const { execSync } = require('child_process') const fs = require('fs-extra') +const _ = require('lodash') +const dayjs = require('dayjs') beforeEach(() => { fs.emptyDirSync('tests/__data__/output') @@ -9,17 +11,45 @@ beforeEach(() => { ) const stdout = execSync( - 'DB_DIR=tests/__data__/output PUBLIC_DIR=tests/__data__/output/.api npm run db:export', + 'DB_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.api npm run db:export', { encoding: 'utf8' } ) }) it('can create streams.json', () => { - expect(content(`output/.api/streams.json`)).toBe(content(`expected/.api/streams.json`)) + let api = content('input/data/streams.json') + let output = content(`output/.api/streams.json`) + let expected = content(`expected/.api/streams.json`) + + const updatedUrl = 'https://master.starmena-cloud.com/hls/libyas.m3u8' + let outputData = output.find(i => i.url === updatedUrl) + let savedData = api.find(i => i.url === updatedUrl) + expect(outputData.added_at).toBe(savedData.added_at) + expect(outputData.updated_at).toBe(savedData.updated_at) + expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0) + + const sameUrl = 'http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8' + outputData = output.find(i => i.url === sameUrl) + savedData = api.find(i => i.url === sameUrl) + expect(outputData.added_at).toBe(savedData.added_at) + expect(dayjs().diff(outputData.updated_at, 'h')).toBe(0) + expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0) + + const addedUrl = 'https://master.starmena-cloud.com/hls/bbc.m3u8' + outputData = output.find(i => i.url === addedUrl) + expect(dayjs().diff(outputData.added_at, 'h')).toBe(0) + expect(dayjs().diff(outputData.updated_at, 'h')).toBe(0) + expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0) + + expect(output.map(item => _.omit(item, ['added_at', 'updated_at', 'checked_at']))).toMatchObject( + expected.map(item => _.omit(item, ['added_at', 'updated_at', 'checked_at'])) + ) }) function content(filepath) { - return fs.readFileSync(`tests/__data__/${filepath}`, { - encoding: 'utf8' - }) + return JSON.parse( + fs.readFileSync(`tests/__data__/${filepath}`, { + encoding: 'utf8' + }) + ) } From 3413121a811dfb1c46ae509ae4e3cfad2a5a1bce Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 03:41:29 +0300 Subject: [PATCH 08/11] Update package-lock.json --- package-lock.json | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index a463bf60e6..603c5be151 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1259,13 +1259,19 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001312", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz", - "integrity": "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ==", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - } + "version": "1.0.30001374", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz", + "integrity": "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + } + ] }, "node_modules/chalk": { "version": "4.1.2", @@ -5200,9 +5206,9 @@ "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" }, "caniuse-lite": { - "version": "1.0.30001312", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz", - "integrity": "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ==" + "version": "1.0.30001374", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz", + "integrity": "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw==" }, "chalk": { "version": "4.1.2", From dee3c217fc77fba4b516aaeb6c10e7cae84702f8 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 03:41:32 +0300 Subject: [PATCH 09/11] Update yarn.lock --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index d60a42abfd..4328fba8d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -848,9 +848,9 @@ "version" "6.3.0" "caniuse-lite@^1.0.30001286": - "integrity" "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ==" - "resolved" "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz" - "version" "1.0.30001312" + "integrity" "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw==" + "resolved" "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz" + "version" "1.0.30001374" "chalk@^2.0.0": "integrity" "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==" From c26ea4bd1c1bc820144fdeb369f8b3c3bb68ae28 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 03:41:37 +0300 Subject: [PATCH 10/11] Update validate.test.js --- tests/commands/playlist/validate.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commands/playlist/validate.test.js b/tests/commands/playlist/validate.test.js index 28089e010b..9b5aefb492 100644 --- a/tests/commands/playlist/validate.test.js +++ b/tests/commands/playlist/validate.test.js @@ -13,7 +13,7 @@ it('show an error if channel name in the blocklist', () => { } catch (err) { expect(err.status).toBe(1) expect(err.stdout).toBe( - `\n> playlist:validate\n> node scripts/commands/playlist/validate.js "tests/__data__/input/streams/us_blocked.m3u"\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/us_blocked.m3u\n 2 error "Fox Sports 2 Asia" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n` + `\n> playlist:validate\n> node scripts/commands/playlist/validate.js\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/us_blocked.m3u\n 2 error "Fox Sports 2 Asia" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n` ) } }) @@ -27,6 +27,6 @@ it('show a warning if channel has wrong id', () => { ) expect(stdout).toBe( - `\n> playlist:validate\n> node scripts/commands/playlist/validate.js "tests/__data__/input/streams/wrong_id.m3u"\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/wrong_id.m3u\n 2 warning "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n` + `\n> playlist:validate\n> node scripts/commands/playlist/validate.js\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/wrong_id.m3u\n 2 warning "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n` ) }) From df189a80622ceef3571d539843b1c09107c8e143 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 10 Aug 2022 04:39:14 +0300 Subject: [PATCH 11/11] Update auto-update.yml --- .github/workflows/auto-update.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 19fa304744..b074c265fe 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -97,6 +97,10 @@ jobs: - run: npm run playlist:update - run: npm run playlist:generate - run: npm run db:export + - uses: actions/upload-artifact@v2 + with: + name: api + path: .api - run: npm run readme:update - uses: actions/upload-artifact@v2 with: