From 3fb3773dfb3eebab7f2fe5310b1c93c36e8eaf4f Mon Sep 17 00:00:00 2001 From: wryk Date: Sat, 11 Jan 2020 16:57:59 +0100 Subject: [PATCH] enhance queue entries --- src/components/App.svelte | 12 ++-- src/components/Viewer.svelte | 6 +- src/store.js | 7 +- src/util.js | 124 +++++++++++++++++++++-------------- 4 files changed, 89 insertions(+), 60 deletions(-) diff --git a/src/components/App.svelte b/src/components/App.svelte index bc2979a..e9cd57a 100644 --- a/src/components/App.svelte +++ b/src/components/App.svelte @@ -22,18 +22,18 @@
-
{entry.data.metadata.title}
+
{entry.metadata.title}
- {entry.url} - {entry.status.account.username} {entry.status.account.acct} + {entry.status.account.username} {entry.status.account.acct} + {entry.data.url}
{/each} - +
@@ -57,7 +57,7 @@ } onMount(() => { - const unsubscribe = entries.subscribe(async (xs) => { + const unsubscribe = entries.subscribe(async (xs) => { if (xs.length) { const [firstEntry] = xs currentEntry.set(firstEntry) @@ -65,7 +65,7 @@ } }) - entries.load(7) + entries.load(1) }) diff --git a/src/components/Viewer.svelte b/src/components/Viewer.svelte index 3b00813..18a1d56 100644 --- a/src/components/Viewer.svelte +++ b/src/components/Viewer.svelte @@ -28,7 +28,6 @@ $: updateVolume($volume) const updateViewerDurationCallback = () => { - console.log('update') if (player) { duration = player.getDuration() currentTime = player.getCurrentTime() @@ -97,6 +96,11 @@ console.log('unplayable', ...args) entry.next() }) + + player.on('error', (...args) => { + console.log('error', ...args) + entry.next() + }) }) onDestroy(() => { diff --git a/src/store.js b/src/store.js index 466b2f4..c225f9f 100644 --- a/src/store.js +++ b/src/store.js @@ -72,7 +72,6 @@ function entryStore(entries) { entries.load(1) } - return entriesList[nextIndex] }) } @@ -81,19 +80,19 @@ function entryStore(entries) { } function entriesStore(domain, hashtags) { - const entriesSteam = util.statusesToEntries(util.statusesStreaming(domain, hashtags)) + const tracksIterator = util.mkTracksIterator(domain, hashtags) const store = writable([]) const { update, subscribe } = store const load = async (number) => { for (let i = 0; i < number; i++) { - const iteratorResult = await entriesSteam.next() + const iteratorResult = await tracksIterator.next() if (iteratorResult.value) { - // console.log(iteratorResult.value) update(entries => [...entries, iteratorResult.value]) } else { + // iterator don't have new entries for now break } } diff --git a/src/util.js b/src/util.js index 6fce9b6..98e07f4 100644 --- a/src/util.js +++ b/src/util.js @@ -1,36 +1,81 @@ import getUrls from 'get-urls' -import { pipe, asyncFilter, asyncMap, asyncTap, asyncTake } from 'iter-tools' -import YouTubePlayer from 'yt-player' +import { execPipe, asyncFilter, asyncMap } from 'iter-tools' -export async function* statusesStreaming(domain, [hashtag]) { - const initialLink = `https://${domain}/api/v1/timelines/tag/${hashtag}?limit=40` +const millisecond = 1 +const second = 1000 * millisecond +const minute = 60 * second - let { statuses, nextLink, previousLink } = await fetchTimeline(initialLink) +export async function* mkStatusesIterator(initialLink) { + let buffer = [] + let { previousLink, nextLink } = initialLink - yield* statuses + console.log('fetch initial') + const initial = await fetchTimeline(initialLink) + let latestPreviousFetch = Date.now() - while (nextLink) { - const a = await fetchTimeline(nextLink) + if (initial.statuses.length > 0) { + buffer = [...initial.statuses] + previousLink = initial.links.prev + nextLink = initial.links.next + } - nextLink = a.nextLink - yield* a.statuses + yield buffer.shift() + + while (true) { + const now = Date.now() + if (latestPreviousFetch + 5 * minute < now) { + console.log('fetch newer') + const previous = await fetchTimeline(previousLink) + console.log(`${previous.length} newers`) + buffer.unshift(...previous.statuses) + previousLink = previous.links.prev + latestPreviousFetch = now + } + + if (buffer.length === 0) { + console.log('fetch older') + const next = await fetchTimeline(nextLink) + buffer.push(...next.statuses) + nextLink = next.links.next + } + + yield buffer.shift() } } -export const statusesToEntries = pipe( - asyncMap(statusToEntry), - asyncFilter(entry => entry.type !== 'unsupported') -) +export async function* mkTracksIterator(domain, hashtags) { + const known = new Set() + const [hashtag] = hashtags + + const statuses = mkStatusesIterator(`https://${domain}/api/v1/timelines/tag/${hashtag}?limit=40`) + + const tracks = execPipe( + statuses, + asyncMap(status => ({ status, data: mkData(status) })), + asyncFilter(({ data }) => { + if (data) { + const found = known.has(data.id) + known.add(data.id) + return !found + } + + return false + }), + asyncMap(async ({ status, data }) => ({ status, data, metadata: await mkMetadata(data) })) + ) + + yield* tracks +} export async function fetchTimeline(url) { - const urlBuilder = new URL(url) - urlBuilder.searchParams.set('limit', 40) - const response = await fetch(url) const statuses = await response.json() - const { next, previous } = parseLinkHeader(response.headers.get('link')) - return { statuses, nextLink: next, previousLink: previous } + const links = response.headers.has('link') + ? parseLinkHeader(response.headers.get('link')) + : {} + + return { statuses, links } } const LINK_RE = /<(.+?)>; rel="(\w+)"/gi @@ -45,44 +90,25 @@ function parseLinkHeader(link) { return links } -async function statusToEntry(status) { +function mkData(status) +{ const urls = getUrls(status.content) - for await (const url of urls) { - const { type, data } = await urlToEntry(url) + for (const urlAsString of urls) { + const url = new URL(urlAsString) - if (type !== 'unsupported') { - return { status, url, type, data } + if (['youtube.com', 'music.youtube.com'].includes(url.hostname) && url.searchParams.has('v')) { + return { url: urlAsString, id: url.searchParams.get('v') } + } else if (url.hostname === 'youtu.be') { + return { url: urlAsString, id: url.pathname.substring(1) } } } - return { type: 'unsupported' } + return null } -async function urlToEntry(urlAsString) { - const url = new URL(urlAsString) - - if (['youtube.com', 'music.youtube.com'].includes(url.hostname) && url.searchParams.has('v')) { - return await mkYoutubeEntry(url.searchParams.get('v')) - } else if (url.hostname === 'youtu.be') { - return await mkYoutubeEntry(url.pathname.substring(1)) - } else { - return { type: 'unsupported' } - } -} - -async function mkYoutubeEntry(id) { - return { - type: 'youtube', - data: { - id, - metadata: await fetchYoutubeMetadata(id) - } - } -} - -function fetchYoutubeMetadata(id) { - return fetch(`https://noembed.com/embed?url=https://www.youtube.com/watch?v=${id}`) +async function mkMetadata(entry) { + return fetch(`https://noembed.com/embed?url=https://www.youtube.com/watch?v=${entry.id}`) .then(response => response.json()) }