From 3c794d6eb76e853cbe39b4d4b194a04f124f9541 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 13 Jul 2021 18:14:49 +0300 Subject: [PATCH 1/6] Create filter.js --- scripts/filter.js | 63 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 scripts/filter.js diff --git a/scripts/filter.js b/scripts/filter.js new file mode 100644 index 0000000000..ff08aa1ade --- /dev/null +++ b/scripts/filter.js @@ -0,0 +1,63 @@ +const parser = require('./parser') +const utils = require('./utils') +const blacklist = require('./blacklist.json') + +let globalBuffer = [] + +async function main() { + const playlists = parseIndex() + for (const playlist of playlists) { + await loadPlaylist(playlist.url).then(removeBlacklisted).then(savePlaylist).then(done) + } + + finish() +} + +function parseIndex() { + console.info(`Parsing 'index.m3u'...`) + let playlists = parser.parseIndex() + console.info(`Found ${playlists.length} playlist(s)\n`) + + return playlists +} + +async function loadPlaylist(url) { + console.info(`Processing '${url}'...`) + return parser.parsePlaylist(url) +} + +async function removeBlacklisted(playlist) { + console.info(` Looking for blacklisted channels...`) + const list = blacklist.map(i => i.toLowerCase()) + playlist.channels = playlist.channels.filter(i => { + return !list.includes(i.name.toLowerCase()) + }) + + return playlist +} + +async function savePlaylist(playlist) { + console.info(` Saving playlist...`) + const original = utils.readFile(playlist.url) + const output = playlist.toString({ raw: true }) + + if (original === output) { + console.info(`No changes have been made.`) + return false + } else { + utils.createFile(playlist.url, output) + console.info(`Playlist has been updated.`) + } + + return true +} + +async function done() { + console.info(` `) +} + +function finish() { + console.info('Done.') +} + +main() From eca41ddc32b224de058c89a1bef87410f50707d9 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 13 Jul 2021 18:15:10 +0300 Subject: [PATCH 2/6] Remove filterChannels() function --- scripts/format.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index a407d334fd..38673dd6bd 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -1,5 +1,4 @@ const { program } = require('commander') -const blacklist = require('./blacklist') const parser = require('./parser') const utils = require('./utils') const axios = require('axios') @@ -35,7 +34,6 @@ async function main() { for (const playlist of playlists) { await loadPlaylist(playlist.url) .then(sortChannels) - .then(filterChannels) .then(detectResolution) .then(savePlaylist) .then(done) @@ -67,16 +65,6 @@ async function sortChannels(playlist) { return playlist } -async function filterChannels(playlist) { - console.info(` Filtering channels...`) - const list = blacklist.map(i => i.toLowerCase()) - playlist.channels = playlist.channels.filter(i => { - return !list.includes(i.name.toLowerCase()) - }) - - return playlist -} - async function detectResolution(playlist) { if (!config.resolution) return playlist console.log(' Detecting resolution...') From aed62aaa50f85cbb324c434b8f94e79b78fda65b Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 13 Jul 2021 19:10:21 +0300 Subject: [PATCH 3/6] Update blacklist.json --- scripts/blacklist.json | 445 ++++++++++++++++++++++++++++++++++------- 1 file changed, 378 insertions(+), 67 deletions(-) diff --git a/scripts/blacklist.json b/scripts/blacklist.json index b0410d49f4..8893f5de14 100644 --- a/scripts/blacklist.json +++ b/scripts/blacklist.json @@ -1,69 +1,380 @@ [ - "Animal Planet", - "Asian Food Network", - "Cooking Channel", - "Discovery Asia", - "Discovery Channel", - "Discovery Civiliztion", - "Discovery en Espanol", - "Discovery Family", - "Discovery Historia", - "Discovery History", - "Discovery Home and Health", - "Discovery Life", - "Discovery Science", - "Discovery Shed", - "Discovery Theater", - "Discovery Travel and Living", - "Discovery Turbo Xtra", - "Discovery World", - "Discovery", - "DIY Network", - "DKiss", - "DMax", - "Eurosport 1", - "Eurosport 2", - "Eurosport", - "eve", - "Familia Discovery", - "Fatafeat", - "FEM", - "Fine Living", - "Food Network", - "food tv", - "foodtv", - "Frisbee", - "Giallo", - "Golf TV", - "GolfTV", - "HGTV", - "ID Investigation Discovery", - "ID Investigation", - "K2", - "Living Channel", - "Mango", - "Motortrend", - "Nove", - "Oprah", - "OWN", - "Quest Red", - "Quest", - "Real Time", - "Sky TG24", - "TLC", - "Trvl Channel", - "TTV", - "TV Norge", - "Tvn 24 Bis", - "TVN 24", - "Tvn 7", - "TVN Extra", - "TVN Fabula", - "TVN Meteo", - "TVN Style", - "TVN Turbo", - "TVN Warszawa", - "TVN", - "Vox", - "World Discovery" + { + "name": "Animal Planet", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Asian Food Network", + "country": "sg", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Cooking Channel", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Asia", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Channel", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Civiliztion", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery en Espanol", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Family", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Historia", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery History", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Home and Health", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Life", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Science", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Shed", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Theater", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Travel and Living", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery Turbo Xtra", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery World", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Discovery", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "DIY Network", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "DKiss", + "country": "es", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "DMax", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Eurosport 1", + "country": "fr", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Eurosport 2", + "country": "fr", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Eurosport", + "country": "fr", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "eve", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Familia Discovery", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Fatafeat", + "country": "eg", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "FEM", + "country": "no", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Fine Living", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Food Network", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "food tv", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": ["foodtv"] + }, + { + "name": "Frisbee", + "country": "it", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Giallo", + "country": "it", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "GolfTV", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": ["Golf TV"] + }, + { + "name": "HGTV", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Investigation Discovery", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": ["ID Investigation Discovery", "ID Investigation", "ID"] + }, + { + "name": "K2", + "country": "it", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Living Channel", + "country": "nz", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Mango", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Motortrend", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Nove", + "country": "it", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "OWN", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": ["Oprah"] + }, + { + "name": "Quest Red", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Quest", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Real Time", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Sky TG24", + "country": "it", + "dmca_notice": "https://github.com/iptv-org/iptv/pull/2294", + "aliases": [] + }, + { + "name": "TLC", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Trvl Channel", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "TTV", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "TV Norge", + "country": "no", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Tvn 24 Bis", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "TVN 24", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Tvn 7", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "TVN Extra", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "TVN Fabula", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "TVN Meteo", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "TVN Style", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "TVN Turbo", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "TVN Warszawa", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "TVN", + "country": "pl", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "Vox", + "country": "no", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + }, + { + "name": "World Discovery", + "country": "us", + "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831", + "aliases": [] + } ] From 6f912ee5bf76ed14ace88d440d513caf85b7312d Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 13 Jul 2021 19:10:31 +0300 Subject: [PATCH 4/6] Update filter.js --- scripts/filter.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/filter.js b/scripts/filter.js index ff08aa1ade..ec90cab56b 100644 --- a/scripts/filter.js +++ b/scripts/filter.js @@ -2,8 +2,6 @@ const parser = require('./parser') const utils = require('./utils') const blacklist = require('./blacklist.json') -let globalBuffer = [] - async function main() { const playlists = parseIndex() for (const playlist of playlists) { @@ -28,9 +26,15 @@ async function loadPlaylist(url) { async function removeBlacklisted(playlist) { console.info(` Looking for blacklisted channels...`) - const list = blacklist.map(i => i.toLowerCase()) - playlist.channels = playlist.channels.filter(i => { - return !list.includes(i.name.toLowerCase()) + playlist.channels = playlist.channels.filter(channel => { + return !blacklist.find(i => { + const channelName = channel.name.toLowerCase() + return ( + (i.name.toLowerCase() === channelName || + i.aliases.map(i => i.toLowerCase()).includes(channelName)) && + i.country === channel.filename + ) + }) }) return playlist From ef9fa11b1a9da68d13e441a7946d7e601dbbfe57 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 13 Jul 2021 19:18:04 +0300 Subject: [PATCH 5/6] Update auto-update.yml --- .github/workflows/auto-update.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 1a29e3a444..4ed5160ca8 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -18,9 +18,26 @@ jobs: with: name: channels path: channels/ - format: + filter: runs-on: ubuntu-latest needs: remove-duplicates + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Download Artifacts + uses: actions/download-artifact@v2 + - name: Install Dependencies + run: npm install + - name: Filter Playlists + run: node scripts/filter.js + - name: Upload Artifact + uses: actions/upload-artifact@v2 + with: + name: channels + path: channels/ + format: + runs-on: ubuntu-latest + needs: filter steps: - name: Checkout uses: actions/checkout@v2 From 642fe8508712c1eaa9678431145ea238a425968f Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 13 Jul 2021 19:18:06 +0300 Subject: [PATCH 6/6] Update CONTRIBUTING.md --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fff6051069..3933d33a43 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -171,7 +171,8 @@ http://example.com/stream.m3u8 - `categories.json`: list of supported categories. - `clean.js`: used in GitHub Action to check all links and remove broken ones. - `db.js`: contains functions for retrieving and managing the channel list. - - `format.js`: used within GitHub Action to sort channels and remove duplicates from each playlist. + - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists. + - `format.js`: used within GitHub Action to format channel descriptions and sort playlists. - `generate.js`: used within GitHub Action to generate all additional playlists. - `parser.js`: contains functions for parsing playlists. - `regions.json`: list of supported region codes.