From a0e4d60dc15d01181aa0522833afa7848ab6c6a7 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 15 Aug 2022 02:21:03 +0300 Subject: [PATCH 01/12] Upgrade iptv-playlist-parser package --- package-lock.json | 43 +++++++++++++++++++++++++++++++++++++------ yarn.lock | 13 +++++++++++++ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 603c5be151..93e9455df4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "dayjs": "^1.10.7", "fs-extra": "^10.0.0", "iptv-checker": "^0.24.5", - "iptv-playlist-parser": "^0.11.0", + "iptv-playlist-parser": "^0.12.1", "jest": "^27.5.1", "jest-expect-message": "^1.0.2", "lodash": "^4.17.21", @@ -2041,7 +2041,7 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" }, - "node_modules/iptv-playlist-parser": { + "node_modules/iptv-checker/node_modules/iptv-playlist-parser": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/iptv-playlist-parser/-/iptv-playlist-parser-0.11.0.tgz", "integrity": "sha512-kUqayPVW8WVhDCodEOBzUlPpBTtCR3k7uvIHfmK/xB8TJCOnbMpvaxzHl0W3ImRS9qDVkBtzcIyfutdLXUr2KQ==", @@ -2050,6 +2050,15 @@ "valid-url": "^1.0.9" } }, + "node_modules/iptv-playlist-parser": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/iptv-playlist-parser/-/iptv-playlist-parser-0.12.1.tgz", + "integrity": "sha512-N0sJFsV8+FBZiR/kl7F5YfFQsUxFXrABP9+xyHPFEQjHJmXSBbvyHLlaW0GjROTd+iXVHA9glEeEwlzJimt5NA==", + "dependencies": { + "is-valid-path": "^0.1.1", + "validator": "^13.7.0" + } + }, "node_modules/is-core-module": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz", @@ -4061,6 +4070,14 @@ "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz", "integrity": "sha1-HBRHm0DxOXp1eC8RXkCGRHQzogA=" }, + "node_modules/validator": { + "version": "13.7.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-13.7.0.tgz", + "integrity": "sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw==", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/w3c-hr-time": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", @@ -5770,16 +5787,25 @@ "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "iptv-playlist-parser": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/iptv-playlist-parser/-/iptv-playlist-parser-0.11.0.tgz", + "integrity": "sha512-kUqayPVW8WVhDCodEOBzUlPpBTtCR3k7uvIHfmK/xB8TJCOnbMpvaxzHl0W3ImRS9qDVkBtzcIyfutdLXUr2KQ==", + "requires": { + "is-valid-path": "^0.1.1", + "valid-url": "^1.0.9" + } } } }, "iptv-playlist-parser": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/iptv-playlist-parser/-/iptv-playlist-parser-0.11.0.tgz", - "integrity": "sha512-kUqayPVW8WVhDCodEOBzUlPpBTtCR3k7uvIHfmK/xB8TJCOnbMpvaxzHl0W3ImRS9qDVkBtzcIyfutdLXUr2KQ==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/iptv-playlist-parser/-/iptv-playlist-parser-0.12.1.tgz", + "integrity": "sha512-N0sJFsV8+FBZiR/kl7F5YfFQsUxFXrABP9+xyHPFEQjHJmXSBbvyHLlaW0GjROTd+iXVHA9glEeEwlzJimt5NA==", "requires": { "is-valid-path": "^0.1.1", - "valid-url": "^1.0.9" + "validator": "^13.7.0" } }, "is-core-module": { @@ -7279,6 +7305,11 @@ "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz", "integrity": "sha1-HBRHm0DxOXp1eC8RXkCGRHQzogA=" }, + "validator": { + "version": "13.7.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-13.7.0.tgz", + "integrity": "sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw==" + }, "w3c-hr-time": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", diff --git a/yarn.lock b/yarn.lock index 4328fba8d8..7115c1a28d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1414,6 +1414,14 @@ "is-valid-path" "^0.1.1" "valid-url" "^1.0.9" +"iptv-playlist-parser@^0.12.1": + "integrity" "sha512-N0sJFsV8+FBZiR/kl7F5YfFQsUxFXrABP9+xyHPFEQjHJmXSBbvyHLlaW0GjROTd+iXVHA9glEeEwlzJimt5NA==" + "resolved" "https://registry.npmjs.org/iptv-playlist-parser/-/iptv-playlist-parser-0.12.1.tgz" + "version" "0.12.1" + dependencies: + "is-valid-path" "^0.1.1" + "validator" "^13.7.0" + "is-arrayish@^0.2.1": "integrity" "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=" "resolved" "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz" @@ -2746,6 +2754,11 @@ "resolved" "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz" "version" "1.0.9" +"validator@^13.7.0": + "integrity" "sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw==" + "resolved" "https://registry.npmjs.org/validator/-/validator-13.7.0.tgz" + "version" "13.7.0" + "w3c-hr-time@^1.0.2": "integrity" "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==" "resolved" "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz" From 0b2ab738e8972bb6c5aeca81a29fb4342556e44c Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 15 Aug 2022 02:21:26 +0300 Subject: [PATCH 02/12] Update tests/__data__ --- .../expected/database/db_clear.streams.db | 6 ++++++ .../expected/database/db_create.streams.db | 13 +++++++------ tests/__data__/input/data/channels.json | 17 +++++++++++++++++ .../__data__/input/database/db_clear.streams.db | 7 +++++++ tests/__data__/input/streams/ly.m3u | 3 +++ 5 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 tests/__data__/expected/database/db_clear.streams.db create mode 100644 tests/__data__/input/database/db_clear.streams.db create mode 100644 tests/__data__/input/streams/ly.m3u diff --git a/tests/__data__/expected/database/db_clear.streams.db b/tests/__data__/expected/database/db_clear.streams.db new file mode 100644 index 0000000000..214264c24b --- /dev/null +++ b/tests/__data__/expected/database/db_clear.streams.db @@ -0,0 +1,6 @@ +{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"} +{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"} +{"title":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576} +{"title":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"tests/__data__/output/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576} +{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"} +{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"} diff --git a/tests/__data__/expected/database/db_create.streams.db b/tests/__data__/expected/database/db_create.streams.db index 454e77577d..fe943bd50c 100644 --- a/tests/__data__/expected/database/db_create.streams.db +++ b/tests/__data__/expected/database/db_create.streams.db @@ -1,6 +1,7 @@ -{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"TyQaTTYos0fr2q0P"} -{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"yNDfQt0ITDrOGGV2"} -{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"asTdyPDWW77mXDLZ"} -{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"1gBgkVYcwsNJQlso"} -{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"8F6RyHFzpOe20huV"} -{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"9DY8CqVcKyp8jqiA"} +{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"UAYbCvjOWvqXHH95"} +{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"IjKHoixx2crsuOlE"} +{"channel":"LibyasChannel.ly","title":"Libyas Channel","filepath":"tests/__data__/input/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","updated_at":"2022-07-07T00:00:00Z","checked_at":"2022-07-07T00:00:00Z","cluster_id":1,"_id":"I0rJlwp3rZEy2SnG"} +{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"tdzk1IN7wLJxfGab"} +{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error","added_at":"2022-07-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z","cluster_id":1,"_id":"3evMHt3nFsZjhzas"} +{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"agJlTEr8wwpbWgw0"} +{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"FpMhoWjtpnVuYlO9"} diff --git a/tests/__data__/input/data/channels.json b/tests/__data__/input/data/channels.json index 6b1e051c0c..c7b4d6a853 100644 --- a/tests/__data__/input/data/channels.json +++ b/tests/__data__/input/data/channels.json @@ -123,6 +123,23 @@ "is_nsfw": false, "logo": "https://iptvx.one/icn/ldpr-tv.png" }, + { + "id": "LibyasChannel.ly", + "name": "Libyas Channel", + "network": null, + "country": "LY", + "subdivision": null, + "city": null, + "broadcast_area": [ + "c/LY" + ], + "languages": [ + "eng" + ], + "categories": [], + "is_nsfw": false, + "logo": "https://i.imgur.com/RD9wbNF.jpg" + }, { "id": "MeteoMedia.ca", "name": "MétéoMédia", diff --git a/tests/__data__/input/database/db_clear.streams.db b/tests/__data__/input/database/db_clear.streams.db new file mode 100644 index 0000000000..852379a3f6 --- /dev/null +++ b/tests/__data__/input/database/db_clear.streams.db @@ -0,0 +1,7 @@ +{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"} +{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"} +{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz","status":"error","added_at":"2022-07-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z"} +{"title":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576} +{"title":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"tests/__data__/output/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576} +{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"} +{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"} diff --git a/tests/__data__/input/streams/ly.m3u b/tests/__data__/input/streams/ly.m3u new file mode 100644 index 0000000000..8dd73939a7 --- /dev/null +++ b/tests/__data__/input/streams/ly.m3u @@ -0,0 +1,3 @@ +#EXTM3U +#EXTINF:-1 tvg-id="LibyasChannel.ly",Libyas Channel +https://master.starmena-cloud.com/hls/libyas.m3u8 From 2b84b126c9e4aa1e5a70ad408583364854250153 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 15 Aug 2022 02:22:01 +0300 Subject: [PATCH 03/12] Update tests/commands --- tests/commands/database/clear.test.js | 48 ++++++++++++++++++++++++++ tests/commands/database/create.test.js | 12 ++----- tests/commands/database/export.test.js | 5 +-- tests/commands/database/matrix.test.js | 6 ++-- tests/commands/database/update.test.js | 7 ++-- 5 files changed, 61 insertions(+), 17 deletions(-) create mode 100644 tests/commands/database/clear.test.js diff --git a/tests/commands/database/clear.test.js b/tests/commands/database/clear.test.js new file mode 100644 index 0000000000..3dea67cc12 --- /dev/null +++ b/tests/commands/database/clear.test.js @@ -0,0 +1,48 @@ +const fs = require('fs-extra') +const path = require('path') +const { execSync } = require('child_process') + +beforeEach(() => { + fs.emptyDirSync('tests/__data__/output') + fs.mkdirSync('tests/__data__/output/database') + fs.copyFileSync( + 'tests/__data__/input/database/db_clear.streams.db', + 'tests/__data__/output/database/streams.db' + ) + + const stdout = execSync( + 'DB_DIR=tests/__data__/output/database npm run db:clear -- --threshold 7', + { + encoding: 'utf8' + } + ) +}) + +it('can clear database', () => { + let output = content('tests/__data__/output/database/streams.db') + let expected = content('tests/__data__/expected/database/db_clear.streams.db') + + output = output.map(i => { + i._id = null + return i + }) + expected = expected.map(i => { + i._id = null + return i + }) + + expect(output).toMatchObject(expected) +}) + +function content(filepath) { + const data = fs.readFileSync(path.resolve(filepath), { + encoding: 'utf8' + }) + + return data + .split('\n') + .filter(l => l) + .map(l => { + return JSON.parse(l) + }) +} diff --git a/tests/commands/database/create.test.js b/tests/commands/database/create.test.js index 056b50cbd3..79f7b07b32 100644 --- a/tests/commands/database/create.test.js +++ b/tests/commands/database/create.test.js @@ -4,9 +4,10 @@ const { execSync } = require('child_process') beforeEach(() => { fs.emptyDirSync('tests/__data__/output') + fs.mkdirSync('tests/__data__/output/database') const stdout = execSync( - 'DB_DIR=tests/__data__/output/database npm run db:create -- --input-dir=tests/__data__/input/streams --max-clusters=1', + 'DB_DIR=tests/__data__/output/database DATA_DIR=tests/__data__/input/data npm run db:create -- --input-dir=tests/__data__/input/streams --max-clusters=1', { encoding: 'utf8' } ) }) @@ -24,14 +25,7 @@ it('can create database', () => { return i }) - expect(output).toEqual( - expect.arrayContaining([ - expect.objectContaining(expected[0]), - expect.objectContaining(expected[1]), - expect.objectContaining(expected[2]), - expect.objectContaining(expected[3]) - ]) - ) + expect(output).toMatchObject(expect.arrayContaining(expected)) }) function content(filepath) { diff --git a/tests/commands/database/export.test.js b/tests/commands/database/export.test.js index 04fba3d36e..f121a1453b 100644 --- a/tests/commands/database/export.test.js +++ b/tests/commands/database/export.test.js @@ -5,13 +5,14 @@ const dayjs = require('dayjs') beforeEach(() => { fs.emptyDirSync('tests/__data__/output') + fs.mkdirSync('tests/__data__/output/database') fs.copyFileSync( 'tests/__data__/input/database/db_export.streams.db', - 'tests/__data__/output/streams.db' + 'tests/__data__/output/database/streams.db' ) const stdout = execSync( - 'DB_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.api npm run db:export', + 'DB_DIR=tests/__data__/output/database DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.api npm run db:export', { encoding: 'utf8' } ) }) diff --git a/tests/commands/database/matrix.test.js b/tests/commands/database/matrix.test.js index 1d2ae7a342..1266584a24 100644 --- a/tests/commands/database/matrix.test.js +++ b/tests/commands/database/matrix.test.js @@ -4,15 +4,15 @@ const { execSync } = require('child_process') beforeEach(() => { fs.emptyDirSync('tests/__data__/output') - + fs.mkdirSync('tests/__data__/output/database') fs.copyFileSync( 'tests/__data__/input/database/db_matrix.streams.db', - 'tests/__data__/output/streams.db' + 'tests/__data__/output/database/streams.db' ) }) it('can create valid matrix', () => { - const result = execSync('DB_DIR=tests/__data__/output npm run db:matrix', { + const result = execSync('DB_DIR=tests/__data__/output/database npm run db:matrix', { encoding: 'utf8' }) expect(result).toBe( diff --git a/tests/commands/database/update.test.js b/tests/commands/database/update.test.js index 3701dbaa10..a0cc342c06 100644 --- a/tests/commands/database/update.test.js +++ b/tests/commands/database/update.test.js @@ -4,15 +4,16 @@ const path = require('path') beforeEach(() => { fs.emptyDirSync('tests/__data__/output') + fs.mkdirSync('tests/__data__/output/database') fs.copyFileSync( 'tests/__data__/input/database/db_update.streams.db', - 'tests/__data__/output/streams.db' + 'tests/__data__/output/database/streams.db' ) }) it('can save results', () => { const stdout = execSync( - 'DB_DIR=tests/__data__/output LOGS_DIR=tests/__data__/input/logs/cluster/load npm run db:update', + 'DB_DIR=tests/__data__/output/database LOGS_DIR=tests/__data__/input/logs/cluster/load npm run db:update', { encoding: 'utf8' } ) expect(stdout).toEqual(` @@ -31,7 +32,7 @@ removed 1 duplicates done `) - expect(content('tests/__data__/output/streams.db')).toEqual( + expect(content('tests/__data__/output/database/streams.db')).toEqual( content('tests/__data__/expected/database/db_update.streams.db') ) }) From 2fef0866db61bfb0252619a920468cce58a8a519 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 15 Aug 2022 02:22:31 +0300 Subject: [PATCH 04/12] Create core/date.js --- scripts/core/date.js | 12 ++++++++++++ scripts/core/index.js | 1 + 2 files changed, 13 insertions(+) create mode 100644 scripts/core/date.js diff --git a/scripts/core/date.js b/scripts/core/date.js new file mode 100644 index 0000000000..9872b5a41f --- /dev/null +++ b/scripts/core/date.js @@ -0,0 +1,12 @@ +const dayjs = require('dayjs') +const utc = require('dayjs/plugin/utc') + +dayjs.extend(utc) + +const date = {} + +date.utc = d => { + return dayjs.utc(d) +} + +module.exports = date diff --git a/scripts/core/index.js b/scripts/core/index.js index d5a5b727cd..98fc70ae96 100644 --- a/scripts/core/index.js +++ b/scripts/core/index.js @@ -11,3 +11,4 @@ exports.markdown = require('./markdown') exports.api = require('./api') exports.id = require('./id') exports.m3u = require('./m3u') +exports.date = require('./date') From ec72bd823e092ccb1635fd2e44621fd2b0dd05fc Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 15 Aug 2022 02:22:36 +0300 Subject: [PATCH 05/12] Update db.js --- scripts/core/db.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/core/db.js b/scripts/core/db.js index fc3f117553..db3fd527f8 100644 --- a/scripts/core/db.js +++ b/scripts/core/db.js @@ -63,6 +63,10 @@ class Database { return this.db.find(query) } + all() { + return this.find({}) + } + remove(query, options) { return this.db.remove(query, options) } From aae44c900413b98864949844ebd95ad69f4c62be Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 15 Aug 2022 02:22:40 +0300 Subject: [PATCH 06/12] Delete cleaner.js --- scripts/commands/playlist/cleaner.js | 90 ---------------------------- 1 file changed, 90 deletions(-) delete mode 100644 scripts/commands/playlist/cleaner.js diff --git a/scripts/commands/playlist/cleaner.js b/scripts/commands/playlist/cleaner.js deleted file mode 100644 index 3389f5208e..0000000000 --- a/scripts/commands/playlist/cleaner.js +++ /dev/null @@ -1,90 +0,0 @@ -const { file, parser, logger, checker, m3u } = require('../../core') -const { program } = require('commander') - -program - .argument('[filepath]', 'Path to file to validate') - .option('-t, --timeout ', 'Set timeout for each request', parser.parseNumber, 60000) - .option('-d, --delay ', 'Set delay for each request', parser.parseNumber, 0) - .option('--debug', 'Enable debug mode') - .parse(process.argv) - -const options = program.opts() - -async function main() { - const files = program.args.length ? program.args : await file.list('streams/*.m3u') - - for (const filepath of files) { - if (!filepath.endsWith('.m3u')) continue - logger.info(`${filepath}`) - const playlist = await parser.parsePlaylist(filepath) - const before = playlist.items.length - for (const stream of playlist.items) { - if (options.debug) logger.info(stream.url) - const [_, status] = stream.raw.match(/status="([a-z]+)"/) || [null, null] - stream.status = status - if (status === 'error' && /^(http|https)/.test(stream.url)) { - const result = await checkStream(stream) - const newStatus = parseStatus(result.error) - if (status === newStatus) { - stream.remove = true - logger.info(`removed "${stream.name}"`) - } - } - } - - const items = playlist.items - .filter(i => !i.remove) - .map(item => ({ - attrs: { - 'tvg-id': item.tvg.id, - status: item.status, - 'user-agent': item.http['user-agent'] || undefined - }, - title: item.name, - url: item.url, - vlcOpts: { - 'http-referrer': item.http.referrer || undefined, - 'http-user-agent': item.http['user-agent'] || undefined - } - })) - - if (before !== items.length) { - const output = m3u.create(items) - await file.create(filepath, output) - logger.info(`saved`) - } - } -} - -main() - -async function checkStream(item) { - const config = { - timeout: options.timeout, - delay: options.delay, - debug: options.debug - } - - const request = { - url: item.url, - http: { - referrer: item.http.referrer, - 'user-agent': item.http['user-agent'] - } - } - - return checker.check(request, config) -} - -function parseStatus(error) { - if (!error) return 'online' - - switch (error) { - case 'Operation timed out': - return 'timeout' - case 'Server returned 403 Forbidden (access denied)': - return 'blocked' - default: - return 'error' - } -} From 6ad1f2e6826aa659696dfcbac017e1e2d7ab6d4c Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 15 Aug 2022 02:22:44 +0300 Subject: [PATCH 07/12] Create clear.js --- scripts/commands/database/clear.js | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 scripts/commands/database/clear.js diff --git a/scripts/commands/database/clear.js b/scripts/commands/database/clear.js new file mode 100644 index 0000000000..6ea623b488 --- /dev/null +++ b/scripts/commands/database/clear.js @@ -0,0 +1,35 @@ +const { logger, parser, db, date } = require('../../core') +const { program } = require('commander') + +const options = program + .option( + '-t, --threshold ', + 'Number of days after which the stream should be deleted', + parser.parseNumber, + 7 + ) + .option('--input-dir ', 'Set path to input directory', 'streams') + .parse(process.argv) + .opts() + +async function main() { + await db.streams.load() + + const streams = await db.streams.all() + + let total = 0 + for (const stream of streams) { + if ( + stream.status === 'error' && + date.utc().diff(stream.updated_at, 'day') >= options.threshold + ) { + total += await db.streams.remove({ url: stream.url }, { multi: true }) + } + } + + await db.streams.compact() + + logger.info(`removed ${total} streams`) +} + +main() From eb7ae04029f556b45ade510642ad6470679e6c02 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 15 Aug 2022 02:22:46 +0300 Subject: [PATCH 08/12] Update create.js --- scripts/commands/database/create.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/commands/database/create.js b/scripts/commands/database/create.js index d4f54666b8..41b38c67bb 100644 --- a/scripts/commands/database/create.js +++ b/scripts/commands/database/create.js @@ -28,6 +28,7 @@ async function findStreams() { logger.info(`looking for streams...`) await api.channels.load() + await api.streams.load() await db.streams.load() const streams = [] @@ -39,6 +40,7 @@ async function findStreams() { const stream = store.create() const channel = await api.channels.find({ id: item.tvg.id }) + const cached = (await api.streams.find({ url: item.url })) || {} stream.set('channel', { channel: channel ? channel.id : null }) stream.set('title', { title: item.name }) @@ -46,6 +48,14 @@ async function findStreams() { stream.set('url', { url: item.url }) stream.set('http_referrer', { http_referrer: item.http.referrer }) stream.set('user_agent', { user_agent: item.http['user-agent'] }) + stream.set('status', { status: cached.status }) + stream.set('width', { width: cached.width }) + stream.set('height', { height: cached.height }) + stream.set('bitrate', { bitrate: cached.bitrate }) + stream.set('frame_rate', { frame_rate: cached.frame_rate }) + stream.set('added_at', { added_at: cached.added_at }) + stream.set('updated_at', { updated_at: cached.updated_at }) + stream.set('checked_at', { checked_at: cached.checked_at }) streams.push(stream) } From e345be44ba1f3836c785f802e919535a3fe53827 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 15 Aug 2022 02:22:51 +0300 Subject: [PATCH 09/12] Update package.json --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index bd6ce5fcb4..b08052c46f 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "act:auto-update": "act workflow_dispatch -W .github/workflows/auto-update.yml --artifact-server-path=.artifacts", "act:check": "act pull_request -W .github/workflows/check.yml", "api:load": "./scripts/commands/api/load.sh", + "db:clear": "node scripts/commands/database/clear.js", "db:create": "node scripts/commands/database/create.js", "db:matrix": "node scripts/commands/database/matrix.js", "db:update": "node scripts/commands/database/update.js", @@ -13,7 +14,6 @@ "playlist:generate": "node scripts/commands/playlist/generate.js", "playlist:update": "node scripts/commands/playlist/update.js", "playlist:lint": "npx m3u-linter -c m3u-linter.json", - "playlist:cleaner": "node scripts/commands/playlist/cleaner.js", "readme:update": "node scripts/commands/readme/update.js", "test": "jest --runInBand" }, @@ -33,7 +33,7 @@ "dayjs": "^1.10.7", "fs-extra": "^10.0.0", "iptv-checker": "^0.24.5", - "iptv-playlist-parser": "^0.11.0", + "iptv-playlist-parser": "^0.12.1", "jest": "^27.5.1", "jest-expect-message": "^1.0.2", "lodash": "^4.17.21", From e2ab794e5ab53853588a0a4edc6645bb0f48dbfe Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Mon, 15 Aug 2022 02:22:55 +0300 Subject: [PATCH 10/12] Update auto-update.yml --- .github/workflows/auto-update.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index b074c265fe..1ff0815673 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -90,17 +90,18 @@ jobs: path: scripts/logs - run: npm install - run: npm run db:update + - run: npm run db:clear - uses: actions/upload-artifact@v2 with: name: database path: scripts/database - - run: npm run playlist:update - - run: npm run playlist:generate - run: npm run db:export - uses: actions/upload-artifact@v2 with: name: api path: .api + - run: npm run playlist:update + - run: npm run playlist:generate - run: npm run readme:update - uses: actions/upload-artifact@v2 with: From 320dc0044e0b6ad1efc676166022897fc40c3259 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sat, 20 Aug 2022 20:15:58 +0300 Subject: [PATCH 11/12] Update auto-update.yml for the test only --- .github/workflows/auto-update.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 1ff0815673..9cd393531b 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -34,9 +34,9 @@ jobs: continue-on-error: true strategy: fail-fast: false - matrix: ${{ fromJson(needs.setup.outputs.matrix) }} - # matrix: - # cluster_id: [1] + # matrix: ${{ fromJson(needs.setup.outputs.matrix) }} + matrix: + cluster_id: [1] steps: - uses: actions/checkout@v2 - uses: FedericoCarboni/setup-ffmpeg@v1 From 1462999a99eea6aef1d9e004735d7b7171e6e4ab Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sat, 20 Aug 2022 20:42:57 +0300 Subject: [PATCH 12/12] Update auto-update.yml --- .github/workflows/auto-update.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 9cd393531b..1ff0815673 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -34,9 +34,9 @@ jobs: continue-on-error: true strategy: fail-fast: false - # matrix: ${{ fromJson(needs.setup.outputs.matrix) }} - matrix: - cluster_id: [1] + matrix: ${{ fromJson(needs.setup.outputs.matrix) }} + # matrix: + # cluster_id: [1] steps: - uses: actions/checkout@v2 - uses: FedericoCarboni/setup-ffmpeg@v1