2020-01-16 20:10:40 +01:00
|
|
|
import { writable } from 'svelte/store'
|
2020-01-09 20:31:12 +01:00
|
|
|
import getUrls from 'get-urls'
|
2020-01-11 16:57:59 +01:00
|
|
|
import { execPipe, asyncFilter, asyncMap } from 'iter-tools'
|
2020-01-10 03:02:46 +01:00
|
|
|
|
2020-01-16 20:10:40 +01:00
|
|
|
export const writableLocalStorage = (key, value) => {
|
|
|
|
const item = JSON.parse(localStorage.getItem(key))
|
|
|
|
const store = writable(item === null ? value : item)
|
|
|
|
|
|
|
|
store.subscribe(x => localStorage.setItem(key, JSON.stringify(x)))
|
|
|
|
|
|
|
|
return store
|
|
|
|
}
|
|
|
|
|
2020-01-11 16:57:59 +01:00
|
|
|
const millisecond = 1
|
|
|
|
const second = 1000 * millisecond
|
|
|
|
const minute = 60 * second
|
2020-01-13 14:39:00 +01:00
|
|
|
const hour = 60 * minute
|
|
|
|
|
|
|
|
export const secondsToElapsedTime = (seconds) => {
|
|
|
|
const parts = [
|
|
|
|
Math.floor(seconds / 3600),
|
|
|
|
Math.floor(seconds / 60) % 60,
|
|
|
|
Math.floor(seconds) % 60
|
|
|
|
]
|
|
|
|
|
|
|
|
return parts
|
|
|
|
.filter((value, index) => value > 0 || index > 0)
|
|
|
|
.map(value => value < 10 ? '0' + value : value)
|
|
|
|
.join(':')
|
|
|
|
}
|
2020-01-10 14:30:17 +01:00
|
|
|
|
2020-01-17 17:38:07 +01:00
|
|
|
export async function* mkStatusesIterator(domain, hashtag) {
|
|
|
|
console.log(`Initialize statuses iterator for #${hashtag} on ${domain}`)
|
|
|
|
const buffer = []
|
2020-01-11 16:57:59 +01:00
|
|
|
|
|
|
|
|
2020-01-14 20:22:09 +01:00
|
|
|
|
2020-01-17 17:38:07 +01:00
|
|
|
// streaming
|
|
|
|
const eventSource = new EventSource(`https://${domain}/api/v1/streaming/hashtag?tag=${hashtag}`)
|
2020-01-12 22:52:40 +01:00
|
|
|
|
2020-01-17 17:38:07 +01:00
|
|
|
eventSource.addEventListener('update', (e) => {
|
|
|
|
console.log(`Received new recent status for #${hashtag} on ${domain}`)
|
|
|
|
buffer.unshift(JSON.parse(e.data))
|
|
|
|
})
|
2020-01-12 22:52:40 +01:00
|
|
|
|
2020-01-17 17:38:07 +01:00
|
|
|
eventSource.onerror = (error) => console.log('onerror', error)
|
2020-01-11 16:57:59 +01:00
|
|
|
|
2020-01-17 17:38:07 +01:00
|
|
|
|
|
|
|
|
|
|
|
// timeline
|
|
|
|
let nextLink = `https://${domain}/api/v1/timelines/tag/${hashtag}?limit=40`
|
|
|
|
|
|
|
|
while (true) {
|
2020-01-11 16:57:59 +01:00
|
|
|
if (buffer.length === 0) {
|
2020-01-17 17:38:07 +01:00
|
|
|
console.log(`Fetch timeline for #${hashtag} on ${domain}`)
|
2020-01-11 16:57:59 +01:00
|
|
|
const next = await fetchTimeline(nextLink)
|
2020-01-12 22:52:40 +01:00
|
|
|
|
|
|
|
if (next.statuses.length) {
|
|
|
|
buffer.push(...next.statuses)
|
|
|
|
nextLink = next.links.next
|
|
|
|
}
|
2020-01-11 16:57:59 +01:00
|
|
|
}
|
2020-01-10 03:02:46 +01:00
|
|
|
|
2020-01-11 16:57:59 +01:00
|
|
|
yield buffer.shift()
|
2020-01-10 03:02:46 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-11 16:57:59 +01:00
|
|
|
export async function* mkTracksIterator(domain, hashtags) {
|
2020-01-11 17:36:57 +01:00
|
|
|
// const known = new Set()
|
|
|
|
const known = {}
|
2020-01-11 16:57:59 +01:00
|
|
|
const [hashtag] = hashtags
|
2020-01-10 03:02:46 +01:00
|
|
|
|
2020-01-17 17:38:07 +01:00
|
|
|
const statuses = mkStatusesIterator(domain, hashtag)
|
2020-01-11 16:57:59 +01:00
|
|
|
|
|
|
|
const tracks = execPipe(
|
|
|
|
statuses,
|
|
|
|
asyncMap(status => ({ status, data: mkData(status) })),
|
|
|
|
asyncFilter(({ data }) => {
|
|
|
|
if (data) {
|
2020-01-11 17:36:57 +01:00
|
|
|
// const found = known.has(data.id)
|
|
|
|
// known.add(data.id)
|
|
|
|
const found = known.hasOwnProperty(data.id)
|
|
|
|
known[data.id] = true
|
2020-01-11 16:57:59 +01:00
|
|
|
return !found
|
|
|
|
}
|
|
|
|
|
|
|
|
return false
|
|
|
|
}),
|
|
|
|
asyncMap(async ({ status, data }) => ({ status, data, metadata: await mkMetadata(data) }))
|
|
|
|
)
|
|
|
|
|
|
|
|
yield* tracks
|
|
|
|
}
|
2020-01-09 20:31:12 +01:00
|
|
|
|
2020-01-11 16:57:59 +01:00
|
|
|
export async function fetchTimeline(url) {
|
2020-01-12 22:52:40 +01:00
|
|
|
console.log(`fetching ${url}`)
|
2020-01-09 20:31:12 +01:00
|
|
|
const response = await fetch(url)
|
|
|
|
const statuses = await response.json()
|
|
|
|
|
2020-01-11 16:57:59 +01:00
|
|
|
const links = response.headers.has('link')
|
|
|
|
? parseLinkHeader(response.headers.get('link'))
|
|
|
|
: {}
|
|
|
|
|
|
|
|
return { statuses, links }
|
2020-01-09 20:31:12 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
const LINK_RE = /<(.+?)>; rel="(\w+)"/gi
|
|
|
|
|
2020-01-10 14:30:17 +01:00
|
|
|
function parseLinkHeader(link) {
|
2020-01-09 20:31:12 +01:00
|
|
|
const links = {}
|
|
|
|
|
|
|
|
for (const [ , url, name ] of link.matchAll(LINK_RE)) {
|
|
|
|
links[name] = url
|
|
|
|
}
|
|
|
|
|
|
|
|
return links
|
2020-01-10 14:30:17 +01:00
|
|
|
}
|
|
|
|
|
2020-01-11 16:57:59 +01:00
|
|
|
function mkData(status)
|
|
|
|
{
|
2020-01-10 14:30:17 +01:00
|
|
|
const urls = getUrls(status.content)
|
|
|
|
|
2020-01-11 16:57:59 +01:00
|
|
|
for (const urlAsString of urls) {
|
|
|
|
const url = new URL(urlAsString)
|
2020-01-10 14:30:17 +01:00
|
|
|
|
2020-01-11 16:57:59 +01:00
|
|
|
if (['youtube.com', 'music.youtube.com'].includes(url.hostname) && url.searchParams.has('v')) {
|
|
|
|
return { url: urlAsString, id: url.searchParams.get('v') }
|
|
|
|
} else if (url.hostname === 'youtu.be') {
|
|
|
|
return { url: urlAsString, id: url.pathname.substring(1) }
|
2020-01-10 14:30:17 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-11 16:57:59 +01:00
|
|
|
return null
|
2020-01-10 14:30:17 +01:00
|
|
|
}
|
|
|
|
|
2020-01-11 16:57:59 +01:00
|
|
|
async function mkMetadata(entry) {
|
|
|
|
return fetch(`https://noembed.com/embed?url=https://www.youtube.com/watch?v=${entry.id}`)
|
2020-01-10 14:30:17 +01:00
|
|
|
.then(response => response.json())
|
|
|
|
}
|
|
|
|
|
|
|
|
export function intersection(xs, ys) {
|
|
|
|
return xs.filter(x => ys.includes(x))
|
2020-01-07 19:23:49 +01:00
|
|
|
}
|