Merge pull request #8003 from iptv-org/update-streams-json
Add new properties to streams.json
This commit is contained in:
Diego
GitHub
commit
558035a768
|
|
@ -97,6 +97,10 @@ jobs:
|
|||
- run: npm run playlist:update
|
||||
- run: npm run playlist:generate
|
||||
- run: npm run db:export
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: api
|
||||
path: .api
|
||||
- run: npm run readme:update
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -1259,13 +1259,19 @@
|
|||
}
|
||||
},
|
||||
"node_modules/caniuse-lite": {
|
||||
"version": "1.0.30001312",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz",
|
||||
"integrity": "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ==",
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/browserslist"
|
||||
}
|
||||
"version": "1.0.30001374",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz",
|
||||
"integrity": "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/browserslist"
|
||||
},
|
||||
{
|
||||
"type": "tidelift",
|
||||
"url": "https://tidelift.com/funding/github/npm/caniuse-lite"
|
||||
}
|
||||
]
|
||||
},
|
||||
"node_modules/chalk": {
|
||||
"version": "4.1.2",
|
||||
|
|
@ -5200,9 +5206,9 @@
|
|||
"integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg=="
|
||||
},
|
||||
"caniuse-lite": {
|
||||
"version": "1.0.30001312",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz",
|
||||
"integrity": "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ=="
|
||||
"version": "1.0.30001374",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz",
|
||||
"integrity": "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw=="
|
||||
},
|
||||
"chalk": {
|
||||
"version": "4.1.2",
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ mkdir -p scripts/data
|
|||
curl -L -o scripts/data/blocklist.json https://iptv-org.github.io/api/blocklist.json
|
||||
curl -L -o scripts/data/categories.json https://iptv-org.github.io/api/categories.json
|
||||
curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json
|
||||
curl -L -o scripts/data/streams.json https://iptv-org.github.io/api/streams.json
|
||||
curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json
|
||||
curl -L -o scripts/data/guides.json https://iptv-org.github.io/api/guides.json
|
||||
curl -L -o scripts/data/languages.json https://iptv-org.github.io/api/languages.json
|
||||
|
|
|
|||
|
|
@ -1,14 +1,19 @@
|
|||
const { logger, db, file } = require('../../core')
|
||||
const { logger, db, api, file } = require('../../core')
|
||||
const _ = require('lodash')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
dayjs.extend(utc)
|
||||
|
||||
const PUBLIC_DIR = process.env.PUBLIC_DIR || '.api'
|
||||
|
||||
async function main() {
|
||||
await api.streams.load()
|
||||
await db.streams.load()
|
||||
const now = dayjs.utc().format()
|
||||
let streams = await db.streams.find({})
|
||||
streams = _.sortBy(streams, 'channel')
|
||||
streams = streams.map(stream => {
|
||||
return {
|
||||
const data = {
|
||||
channel: stream.channel,
|
||||
url: stream.url,
|
||||
http_referrer: stream.http_referrer,
|
||||
|
|
@ -19,6 +24,26 @@ async function main() {
|
|||
bitrate: stream.bitrate,
|
||||
frame_rate: stream.frame_rate
|
||||
}
|
||||
|
||||
let addedAt = now
|
||||
let updatedAt = now
|
||||
let found = api.streams.find({ url: stream.url })
|
||||
if (found) {
|
||||
normalized = _.omit(found, ['added_at', 'updated_at', 'checked_at'])
|
||||
if (_.isEqual(data, normalized)) {
|
||||
addedAt = found.added_at || now
|
||||
updatedAt = found.updated_at || now
|
||||
} else {
|
||||
addedAt = found.added_at || now
|
||||
updatedAt = now
|
||||
}
|
||||
}
|
||||
|
||||
data.added_at = addedAt
|
||||
data.updated_at = updatedAt
|
||||
data.checked_at = now
|
||||
|
||||
return data
|
||||
})
|
||||
|
||||
await file.create(`${PUBLIC_DIR}/streams.json`, JSON.stringify(streams))
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ class API {
|
|||
const api = {}
|
||||
|
||||
api.channels = new API(`${DATA_DIR}/channels.json`)
|
||||
api.streams = new API(`${DATA_DIR}/streams.json`)
|
||||
api.countries = new API(`${DATA_DIR}/countries.json`)
|
||||
api.guides = new API(`${DATA_DIR}/guides.json`)
|
||||
api.categories = new API(`${DATA_DIR}/categories.json`)
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
[{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error"},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"blocked"},{"channel":"BBCNewsHD.ad","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25},{"channel":"KayhanTV.af","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"Sharq.af","url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1280,"height":720,"bitrate":2226543,"frame_rate":25}]
|
||||
[{"channel":"AndorraTV.ad","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","status":"error","added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"BBCNews.uk","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"status":"blocked","added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"BBCNewsHD.uk","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"},{"channel":"KayhanTV.af","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"error"},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-07-07T00:00:00Z"},{"channel":"Sharq.af","url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1280,"height":720,"bitrate":2226543,"frame_rate":25,"added_at":"2022-08-09T02:54:12Z","checked_at":"2022-08-09T02:54:12Z","updated_at":"2022-08-09T02:54:12Z"}]
|
||||
|
|
@ -0,0 +1 @@
|
|||
[{"channel":"LDPRTV.ru","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1920,"height":1080,"bitrate":6542556,"frame_rate":50,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-08-07T00:00:00Z","updated_at":"2022-08-07T00:00:00Z"},{"channel":"LibyasChannel.ly","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"status":"online","width":1024,"height":576,"bitrate":0,"frame_rate":25,"added_at":"2022-07-07T00:00:00Z","checked_at":"2022-07-07T00:00:00Z","updated_at":"2022-07-07T00:00:00Z"}]
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
{"title":"ЛДПР ТВ","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"2ST8btby3mmsgPF0","status":"error"}
|
||||
{"title":"BBC News HD","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"3TbieV1ptnZVCIdn","status":"blocked"}
|
||||
{"title":"ATV","channel":"AndorraTV.ad","filepath":"tests/__data__/output/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","cluster_id":1,"_id":"I6cjG2xCBRFFP4sz","status":"error"}
|
||||
{"title":"BBC News HD","channel":"BBCNewsHD.ad","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
|
||||
{"title":"BBC News HD","channel":"BBCNewsHD.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"https://master.starmena-cloud.com/hls/bbc.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnXVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
|
||||
{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"channels/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"cFFpFVzSn6xFMUF3","status":"error"}
|
||||
{"title":"Sharq","channel":"Sharq.af","filepath":"channels/af.m3u","bitrate":2226543,"frame_rate":25,"width":1280,"height":720,"url":"https://forerunnerrtmp.livestreamingcdn.com/output18/output18.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"cluster_id":1,"_id":"u7iyA6cjtf1iWWAZ","status":"online"}
|
||||
{"title":"Libyas Channel","channel":"LibyasChannel.ly","filepath":"tests/__data__/output/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"cluster_id":3,"_id":"WTbieV1ptnZVCIdn","status":"online","bitrate":0,"frame_rate":25,"width":1024,"height":576}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
const { execSync } = require('child_process')
|
||||
const fs = require('fs-extra')
|
||||
const _ = require('lodash')
|
||||
const dayjs = require('dayjs')
|
||||
|
||||
beforeEach(() => {
|
||||
fs.emptyDirSync('tests/__data__/output')
|
||||
|
|
@ -9,17 +11,45 @@ beforeEach(() => {
|
|||
)
|
||||
|
||||
const stdout = execSync(
|
||||
'DB_DIR=tests/__data__/output PUBLIC_DIR=tests/__data__/output/.api npm run db:export',
|
||||
'DB_DIR=tests/__data__/output DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output/.api npm run db:export',
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
})
|
||||
|
||||
it('can create streams.json', () => {
|
||||
expect(content(`output/.api/streams.json`)).toBe(content(`expected/.api/streams.json`))
|
||||
let api = content('input/data/streams.json')
|
||||
let output = content(`output/.api/streams.json`)
|
||||
let expected = content(`expected/.api/streams.json`)
|
||||
|
||||
const updatedUrl = 'https://master.starmena-cloud.com/hls/libyas.m3u8'
|
||||
let outputData = output.find(i => i.url === updatedUrl)
|
||||
let savedData = api.find(i => i.url === updatedUrl)
|
||||
expect(outputData.added_at).toBe(savedData.added_at)
|
||||
expect(outputData.updated_at).toBe(savedData.updated_at)
|
||||
expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0)
|
||||
|
||||
const sameUrl = 'http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8'
|
||||
outputData = output.find(i => i.url === sameUrl)
|
||||
savedData = api.find(i => i.url === sameUrl)
|
||||
expect(outputData.added_at).toBe(savedData.added_at)
|
||||
expect(dayjs().diff(outputData.updated_at, 'h')).toBe(0)
|
||||
expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0)
|
||||
|
||||
const addedUrl = 'https://master.starmena-cloud.com/hls/bbc.m3u8'
|
||||
outputData = output.find(i => i.url === addedUrl)
|
||||
expect(dayjs().diff(outputData.added_at, 'h')).toBe(0)
|
||||
expect(dayjs().diff(outputData.updated_at, 'h')).toBe(0)
|
||||
expect(dayjs().diff(outputData.checked_at, 'h')).toBe(0)
|
||||
|
||||
expect(output.map(item => _.omit(item, ['added_at', 'updated_at', 'checked_at']))).toMatchObject(
|
||||
expected.map(item => _.omit(item, ['added_at', 'updated_at', 'checked_at']))
|
||||
)
|
||||
})
|
||||
|
||||
function content(filepath) {
|
||||
return fs.readFileSync(`tests/__data__/${filepath}`, {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
return JSON.parse(
|
||||
fs.readFileSync(`tests/__data__/${filepath}`, {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ it('show an error if channel name in the blocklist', () => {
|
|||
} catch (err) {
|
||||
expect(err.status).toBe(1)
|
||||
expect(err.stdout).toBe(
|
||||
`\n> playlist:validate\n> node scripts/commands/playlist/validate.js "tests/__data__/input/streams/us_blocked.m3u"\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/us_blocked.m3u\n 2 error "Fox Sports 2 Asia" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n`
|
||||
`\n> playlist:validate\n> node scripts/commands/playlist/validate.js\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/us_blocked.m3u\n 2 error "Fox Sports 2 Asia" is on the blocklist due to claims of copyright holders (https://github.com/iptv-org/iptv/issues/0000)\n\n1 problems (1 errors, 0 warnings)\n`
|
||||
)
|
||||
}
|
||||
})
|
||||
|
|
@ -27,6 +27,6 @@ it('show a warning if channel has wrong id', () => {
|
|||
)
|
||||
|
||||
expect(stdout).toBe(
|
||||
`\n> playlist:validate\n> node scripts/commands/playlist/validate.js "tests/__data__/input/streams/wrong_id.m3u"\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/wrong_id.m3u\n 2 warning "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n`
|
||||
`\n> playlist:validate\n> node scripts/commands/playlist/validate.js\n\nloading blocklist...\nfound 4 records\n\ntests/__data__/input/streams/wrong_id.m3u\n 2 warning "qib22lAq1L.us" is not in the database\n\n1 problems (0 errors, 1 warnings)\n`
|
||||
)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -848,9 +848,9 @@
|
|||
"version" "6.3.0"
|
||||
|
||||
"caniuse-lite@^1.0.30001286":
|
||||
"integrity" "sha512-Wiz1Psk2MEK0pX3rUzWaunLTZzqS2JYZFzNKqAiJGiuxIjRPLgV6+VDPOg6lQOUxmDwhTlh198JsTTi8Hzw6aQ=="
|
||||
"resolved" "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001312.tgz"
|
||||
"version" "1.0.30001312"
|
||||
"integrity" "sha512-mWvzatRx3w+j5wx/mpFN5v5twlPrabG8NqX2c6e45LCpymdoGqNvRkRutFUqpRTXKFQFNQJasvK0YT7suW6/Hw=="
|
||||
"resolved" "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001374.tgz"
|
||||
"version" "1.0.30001374"
|
||||
|
||||
"chalk@^2.0.0":
|
||||
"integrity" "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ=="
|
||||
|
|
|
|||
Loading…
Reference in New Issue