2020-03-06 14:38:44 +01:00
|
|
|
import getUrls from 'get-urls'
|
|
|
|
import { asyncMap, execPipe, map, findOr } from 'iter-tools'
|
|
|
|
import { mapNullable } from '/services/misc.js'
|
|
|
|
|
2020-01-20 03:26:18 +01:00
|
|
|
const LINK_RE = /<(.+?)>; rel="(\w+)"/gi
|
|
|
|
|
2022-05-13 19:15:35 +02:00
|
|
|
const YoutubeInstances = [
|
|
|
|
'youtube.com', 'm.youtube.com', 'music.youtube.com',
|
|
|
|
'invidio.us', 'redirect.invidious.io',
|
|
|
|
'invidious.snopyta.org', 'yewtu.be', 'invidious.kavin.rocks', 'vid.puffyan.us', 'invidious.namazso.eu', 'inv.riverside.rocks', 'invidious.osi.kr', 'youtube.076.ne.jp', 'yt.artemislena.eu', 'tube.cthd.icu', 'invidious.flokinet.to', 'invidious.weblibre.org', 'invidious.esmailelbob.xyz', 'invidious.lunar.icu', 'invidious.mutahar.rocks', 'inv.bp.projectsegfau.lt', 'y.com.sb', 'invidious.sethforprivacy.com', 'invidious.tiekoetter.com', 'invidious.hub.ne.kr',
|
|
|
|
'videos.arci.me', 'octt.ddns.net:48864',
|
|
|
|
'c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid.onion', 'w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd.onion', 'kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad.onion', 'grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad.onion', 'osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd.onion', 'u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad.onion', 'euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd.onion'
|
|
|
|
]
|
|
|
|
|
2020-02-22 03:39:15 +01:00
|
|
|
function parseLinkHeader(linkHeader) {
|
|
|
|
const links = new Map()
|
2020-01-20 03:26:18 +01:00
|
|
|
|
2020-02-22 03:39:15 +01:00
|
|
|
for (const [ , url, name ] of linkHeader.matchAll(LINK_RE)) {
|
|
|
|
links.set(name, url)
|
2020-01-20 03:26:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return links
|
|
|
|
}
|
|
|
|
|
2020-02-16 17:02:39 +01:00
|
|
|
export const fetchStatus = (domain, id) => fetch(`https://${domain}/api/v1/statuses/${id}`)
|
|
|
|
.then(response => response.json())
|
2020-02-15 23:12:53 +01:00
|
|
|
|
2020-02-20 15:55:22 +01:00
|
|
|
export async function* statusIterator({ domain, id }) {
|
2020-03-07 19:00:39 +01:00
|
|
|
const partialTrack = processStatus(domain, await fetchStatus(domain, id))
|
2020-03-06 14:38:44 +01:00
|
|
|
|
|
|
|
if (partialTrack !== null) {
|
|
|
|
yield partialTrack
|
|
|
|
}
|
2020-02-20 15:55:22 +01:00
|
|
|
}
|
|
|
|
|
2020-03-06 20:23:44 +01:00
|
|
|
export const hashtagStreamingStatusesObservable = (domain, hashtag) => {
|
2020-01-20 03:26:18 +01:00
|
|
|
return new Observable(observer => {
|
2020-02-04 20:07:34 +01:00
|
|
|
const onOpen = () => {
|
|
|
|
console.log(`Streaming ${domain} #${hashtag} : open`)
|
|
|
|
}
|
|
|
|
|
|
|
|
const onStatus = event => {
|
|
|
|
const status = JSON.parse(event.data)
|
|
|
|
console.log(`Streaming ${domain} #${hashtag} : status ${status.id}`)
|
2020-03-06 20:23:44 +01:00
|
|
|
observer.next(status)
|
2020-02-04 20:07:34 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
const onError = error => {
|
|
|
|
console.error(`Streaming ${domain} #${hashtag} : error`)
|
|
|
|
console.error(error)
|
|
|
|
observer.error(error)
|
|
|
|
}
|
2020-01-20 03:26:18 +01:00
|
|
|
|
|
|
|
const eventSource = new EventSource(`https://${domain}/api/v1/streaming/hashtag?tag=${hashtag}`)
|
2020-02-04 20:07:34 +01:00
|
|
|
eventSource.addEventListener('open', onOpen)
|
2020-01-20 03:26:18 +01:00
|
|
|
eventSource.addEventListener('update', onStatus)
|
|
|
|
eventSource.addEventListener('error', onError)
|
|
|
|
|
|
|
|
return () => {
|
2020-02-21 23:31:30 +01:00
|
|
|
console.log(`Streaming ${domain} #${hashtag} : closed`)
|
2020-02-04 20:07:34 +01:00
|
|
|
eventSource.removeEventListener('open', onOpen)
|
2020-01-20 03:26:18 +01:00
|
|
|
eventSource.removeEventListener('update', onStatus)
|
|
|
|
eventSource.removeEventListener('error', onError)
|
2020-02-16 17:02:39 +01:00
|
|
|
eventSource.close()
|
2020-01-20 03:26:18 +01:00
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2020-03-06 20:23:44 +01:00
|
|
|
export const hashtagStreamingObservable = (domain, hashtag) => {
|
|
|
|
return new Observable(observer => {
|
|
|
|
const subscription = hashtagStreamingStatusesObservable(domain, hashtag).subscribe({
|
|
|
|
next: status => {
|
|
|
|
const partialMedia = processStatus(domain, status)
|
|
|
|
|
|
|
|
if (partialMedia !== null) {
|
|
|
|
observer.next(partialMedia)
|
|
|
|
}
|
|
|
|
},
|
|
|
|
error: observer.error,
|
|
|
|
complete: observer.complete
|
|
|
|
})
|
|
|
|
|
|
|
|
return () => {
|
|
|
|
subscription.unsubscribe()
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
// don't handle correctly complete
|
2020-02-21 23:31:30 +01:00
|
|
|
export const hashtagsStreamingObservable = (domain, hashtags) => {
|
|
|
|
return new Observable(observer => {
|
|
|
|
const subscriptions = hashtags
|
|
|
|
.map(hashtag => hashtagStreamingObservable(domain, hashtag))
|
|
|
|
.map(observable => observable.subscribe(observer))
|
|
|
|
|
|
|
|
return () => {
|
|
|
|
subscriptions.forEach(subscription => subscription.unsubscribe())
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2020-03-06 14:38:44 +01:00
|
|
|
|
|
|
|
export async function* hashtagTimelineStatusesIterator (domain, hashtag) {
|
2020-01-20 03:26:18 +01:00
|
|
|
let nextLink = `https://${domain}/api/v1/timelines/tag/${hashtag}?limit=40`
|
|
|
|
|
|
|
|
while (nextLink) {
|
|
|
|
const response = await fetch(nextLink)
|
|
|
|
|
|
|
|
nextLink = response.headers.has('link')
|
2020-02-22 03:39:15 +01:00
|
|
|
? parseLinkHeader(response.headers.get('link')).get('next')
|
2020-01-20 03:26:18 +01:00
|
|
|
: null
|
|
|
|
|
2020-03-06 14:38:44 +01:00
|
|
|
yield* await response.json()
|
|
|
|
}
|
|
|
|
}
|
2020-02-04 20:07:34 +01:00
|
|
|
|
2020-03-06 14:38:44 +01:00
|
|
|
export const hashtagTimelineIterator = (domain, hashtag) => execPipe(
|
|
|
|
hashtagTimelineStatusesIterator(domain, hashtag),
|
|
|
|
asyncMap(status => processStatus(domain, status)),
|
|
|
|
async function* (xs) {
|
|
|
|
let c = 0
|
2020-02-04 20:07:34 +01:00
|
|
|
|
2020-03-06 14:38:44 +01:00
|
|
|
for await (const x of xs) {
|
|
|
|
if (x === null) {
|
|
|
|
if (++c > 69) {
|
|
|
|
console.log(`Not found any viable media on #${hashtag}.`)
|
|
|
|
break
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
c = 0
|
|
|
|
yield x
|
|
|
|
}
|
|
|
|
}
|
2020-01-20 03:26:18 +01:00
|
|
|
}
|
2020-03-06 14:38:44 +01:00
|
|
|
)
|
2020-01-20 03:26:18 +01:00
|
|
|
|
2020-02-21 23:31:30 +01:00
|
|
|
export async function* hashtagsTimelineIterator (domain, hashtags) {
|
|
|
|
const iterators = hashtags.map(hashtag => hashtagTimelineIterator(domain, hashtag))
|
|
|
|
const promises = iterators.map(iterator => iterator.next())
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
const results = (await Promise.all(promises))
|
|
|
|
.map((result, index) => ({ index, result }))
|
|
|
|
.filter(({ result }) => !result.done)
|
|
|
|
|
|
|
|
if (results.length > 0) {
|
2020-03-06 17:54:58 +01:00
|
|
|
const sorted = results.sort((a, b) => b.result.value.referer.date - a.result.value.referer.date)
|
2020-02-21 23:31:30 +01:00
|
|
|
const { index, result: { value } } = sorted[0]
|
|
|
|
|
|
|
|
promises[index] = iterators[index].next()
|
|
|
|
yield value
|
|
|
|
} else {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-20 03:26:18 +01:00
|
|
|
}
|
|
|
|
|
2020-02-21 23:31:30 +01:00
|
|
|
export async function* hashtagsIterator(domain, hashtags) {
|
|
|
|
const buffer = []
|
2020-02-14 17:49:56 +01:00
|
|
|
|
2020-02-21 23:31:30 +01:00
|
|
|
const streamingSubscription = hashtagsStreamingObservable(domain, hashtags).subscribe({
|
|
|
|
next: value => buffer.push(value),
|
|
|
|
error: error => console.error(error),
|
|
|
|
complete: () => console.log('complete')
|
|
|
|
})
|
2020-02-14 17:49:56 +01:00
|
|
|
|
2020-02-21 23:31:30 +01:00
|
|
|
const timelineGenerator = hashtagsTimelineIterator(domain, hashtags)
|
2020-02-14 17:49:56 +01:00
|
|
|
|
2020-02-21 23:31:30 +01:00
|
|
|
try {
|
|
|
|
while (true) {
|
|
|
|
if (buffer.length > 0) {
|
|
|
|
yield buffer.pop()
|
|
|
|
} else {
|
2020-03-06 13:25:09 +01:00
|
|
|
const { done, value } = await timelineGenerator.next()
|
|
|
|
|
|
|
|
if (done) {
|
|
|
|
break
|
|
|
|
} else {
|
|
|
|
yield value
|
|
|
|
}
|
2020-02-21 23:31:30 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
} finally {
|
|
|
|
streamingSubscription.unsubscribe()
|
|
|
|
timelineGenerator.return()
|
2020-02-14 17:49:56 +01:00
|
|
|
}
|
|
|
|
}
|
2020-02-15 23:12:53 +01:00
|
|
|
|
2020-03-06 14:38:44 +01:00
|
|
|
const processStatus = (domain, status) => mapNullable(findMedia(status), partialMedia => ({
|
|
|
|
referer: {
|
|
|
|
username: status.account.username,
|
|
|
|
content: status.content,
|
|
|
|
date: new Date(status.created_at),
|
|
|
|
url: status.url,
|
|
|
|
credentials: { type: 'mastodon', domain, id: status.id }
|
|
|
|
},
|
|
|
|
partialMedia
|
|
|
|
}))
|
|
|
|
|
|
|
|
const findMedia = status => execPipe(
|
|
|
|
status.content,
|
|
|
|
getUrls,
|
|
|
|
map(url => {
|
|
|
|
const { hostname, pathname, searchParams } = new URL(url)
|
|
|
|
|
2022-05-13 19:15:35 +02:00
|
|
|
if (YoutubeInstances.includes(hostname) && searchParams.has('v')) {
|
2020-03-06 14:38:44 +01:00
|
|
|
return { url, credentials: { type: 'youtube', id: searchParams.get('v') } }
|
|
|
|
} else if (hostname === 'youtu.be') {
|
|
|
|
return { url, credentials: { type: 'youtube', id: pathname.substring(1) } }
|
|
|
|
} else {
|
|
|
|
return null
|
|
|
|
}
|
|
|
|
}),
|
|
|
|
findOr(null, x => x !== null)
|
|
|
|
)
|