diff --git a/scripts/commands/database/cleanup.js b/scripts/commands/database/cleanup.js index 5b23f52332..cb527a3ad4 100644 --- a/scripts/commands/database/cleanup.js +++ b/scripts/commands/database/cleanup.js @@ -1,4 +1,5 @@ const { db, logger } = require('../../core') +const _ = require('lodash') async function main() { logger.info(`loading streams...`) @@ -7,16 +8,15 @@ async function main() { logger.info(`removing broken links...`) let removed = 0 - const buffer = {} - for (const stream of streams) { - const duplicate = buffer[stream.channel] - if (duplicate && !stream.is_online) { + const failed = _.filter(streams, { status: 'error' }) + for (const stream of failed) { + const hasDuplicate = _.find(streams, s => s.channel === stream.channel && s.status !== 'error') + if (hasDuplicate) { await db.streams.remove({ _id: stream._id }) removed++ - } else { - buffer[stream.channel] = stream } } + db.streams.compact() logger.info(`removed ${removed} links`) diff --git a/tests/__data__/expected/database/db_cleanup.streams.db b/tests/__data__/expected/database/db_cleanup.streams.db index 3751ae9d32..9dac2dd735 100644 --- a/tests/__data__/expected/database/db_cleanup.streams.db +++ b/tests/__data__/expected/database/db_cleanup.streams.db @@ -1,5 +1,5 @@ -{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0"} -{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn"} -{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"} -{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"} -{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"} +{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"2ST8btby3mmsgPF0"} +{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"} +{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":3,"_id":"WTbieV1ptnZVCIdn"} +{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"blocked","cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"} +{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","http_referrer":null,"user_agent":null,"status":"timeout","cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"} diff --git a/tests/__data__/input/database/db_cleanup.streams.db b/tests/__data__/input/database/db_cleanup.streams.db index f000c67752..492f667576 100644 --- a/tests/__data__/input/database/db_cleanup.streams.db +++ b/tests/__data__/input/database/db_cleanup.streams.db @@ -1,6 +1,6 @@ -{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0"} -{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn"} -{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"} -{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn"} -{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"} -{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"} +{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/channels/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"2ST8btby3mmsgPF0"} +{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"error","cluster_id":3,"_id":"3TbieV1ptnZVCIdn"} +{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/channels/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":null,"user_agent":null,"status":"online","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz"} +{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/channels/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","cluster_id":3,"_id":"WTbieV1ptnZVCIdn"} +{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"blocked","cluster_id":1,"_id":"cFFpFVzSn6xFMUF3"} +{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","http_referrer":null,"user_agent":null,"status":"timeout","cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ"}