From 3c2146127a695db324f7fa27eb1cb37100af73c0 Mon Sep 17 00:00:00 2001 From: wryk Date: Sun, 16 Feb 2020 17:02:39 +0100 Subject: [PATCH] better router, add sharing support --- package-lock.json | 5 ++ package.json | 3 +- src/components/App.svelte | 13 ++- src/components/Queue.svelte | 4 +- src/components/Radio.svelte | 24 +++-- src/routes.js | 4 + src/routes/Home.svelte | 2 +- src/routes/Share.svelte | 16 ++-- src/services/deep-set.js | 41 +++++++++ src/services/mastodon.js | 15 ++-- src/services/misc.js | 170 +++++++++++++----------------------- src/services/radio.js | 9 ++ 12 files changed, 170 insertions(+), 136 deletions(-) create mode 100644 src/routes.js create mode 100644 src/services/deep-set.js create mode 100644 src/services/radio.js diff --git a/package-lock.json b/package-lock.json index f1c7189..cf01ea4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8047,6 +8047,11 @@ "inherits": "^2.0.1" } }, + "route-parser": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/route-parser/-/route-parser-0.0.5.tgz", + "integrity": "sha1-fR0J0zXkkJQDHqFpkaSnmwG74fQ=" + }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", diff --git a/package.json b/package.json index 926f423..136a8a1 100644 --- a/package.json +++ b/package.json @@ -27,13 +27,14 @@ "date-fns": "^2.9.0", "get-urls": "^9.2.0", "iter-tools": "^7.0.0-rc.0", + "route-parser": "0.0.5", "svelte-routing": "^1.4.0" }, "browserslist": [ "last 1 chrome versions" ], "staticFiles": { - "staticPath": "public", + "staticPath": "static", "watcherGlob": "**" } } diff --git a/src/components/App.svelte b/src/components/App.svelte index 9660ecf..77971b3 100644 --- a/src/components/App.svelte +++ b/src/components/App.svelte @@ -1,10 +1,19 @@ - - + {#each routes as { duplex, component }} + + {/each} \ No newline at end of file diff --git a/src/components/Queue.svelte b/src/components/Queue.svelte index 1c2fafa..d432a0e 100644 --- a/src/components/Queue.svelte +++ b/src/components/Queue.svelte @@ -4,7 +4,7 @@ {#if $next}
select($next)}>
{$next.title}
-
by {$next.username}
+
by {$next.referer.username}
{/if} @@ -18,7 +18,7 @@ {#each history as track}
select(track)}>
{track.title}
-
shared by {track.username}
+
shared by {track.referer.username}
{/each} diff --git a/src/components/Radio.svelte b/src/components/Radio.svelte index dbc13ed..178b3ad 100644 --- a/src/components/Radio.svelte +++ b/src/components/Radio.svelte @@ -21,26 +21,38 @@ \ No newline at end of file diff --git a/src/services/deep-set.js b/src/services/deep-set.js new file mode 100644 index 0000000..40b3186 --- /dev/null +++ b/src/services/deep-set.js @@ -0,0 +1,41 @@ +export default class DeepSet { + constructor() { + this.map = new Map() + this.set = new Set() + } + + _reduce(path) { + return path.reduce((context, key) => { + if (context.map.has(key)) { + return context.map.get(key) + } else { + const newContext = new DeepSet() + context.map.set(key, newContext) + return newContext + } + }, this).set + } + + has(values) { + const { keys, value } = destruct(values) + return this._reduce(keys).has(value) + } + + add(values) { + const { keys, value } = destruct(values) + return this._reduce(keys).add(value) + } +} + +const destruct = xs => { + switch (xs.length) { + case 0: + return { keys: [], value: undefined } + + case 1: + return { keys: [], value: xs[0] } + + default: + return { keys: xs.slice(0, xs.length - 1), value: xs.slice(-1)[0] } + } +} \ No newline at end of file diff --git a/src/services/mastodon.js b/src/services/mastodon.js index afb8812..b4f5ae2 100644 --- a/src/services/mastodon.js +++ b/src/services/mastodon.js @@ -1,5 +1,6 @@ import Observable from 'core-js-pure/features/observable' -import { observableToAsyncIterator, raceIterator } from '/services/misc.js' +import getUrls from 'get-urls' +import { observableToAsyncIterator, raceIterator, urlsToMedia } from '/services/misc.js' const LINK_RE = /<(.+?)>; rel="(\w+)"/gi @@ -13,7 +14,9 @@ function parseLinkHeader(link) { return links } -export const fetchStatus = (domain, id) => fetch(`https://${domain}/api/v1/statuses/${id}`).then(x => x.json()) +export const fetchStatus = (domain, id) => fetch(`https://${domain}/api/v1/statuses/${id}`) + .then(response => response.json()) + .then(status => processStatus(domain, status)) // Observable<{ domain : string, hashtag : string, status : Status}> export const hashtagStreamingObservable = (domain, hashtag) => { @@ -43,6 +46,7 @@ export const hashtagStreamingObservable = (domain, hashtag) => { eventSource.removeEventListener('open', onOpen) eventSource.removeEventListener('update', onStatus) eventSource.removeEventListener('error', onError) + eventSource.close() } }) } @@ -92,14 +96,13 @@ export async function* hashtagsIterator (domain, hashtags) { } const processStatus = (domain, status) => ({ - title: null, - username: status.account.username, + title: '', date: new Date(status.createdAt), - content: status.content, referer: { + username: status.account.username, url: status.url, credentials: { type: 'mastodon', domain, id: status.id } }, - media: null + media: urlsToMedia(getUrls(status.content)) }) diff --git a/src/services/misc.js b/src/services/misc.js index fbafd46..269157b 100644 --- a/src/services/misc.js +++ b/src/services/misc.js @@ -1,4 +1,3 @@ -import getUrls from 'get-urls' import { execPipe, asyncFilter, asyncMap, map, findOr } from 'iter-tools' export const tap = f => x => { @@ -92,129 +91,82 @@ export async function* raceIterator(iterators) { } } -const mkMapSet = () => ({ set: new Set(), children: new Map() }) - -const pathSet = () => { - const root = mkMapSet() - - const has = (keys, value) => { - let x = root - - for (const key of keys) { - if (x.children.has(key)) { - x = x.children.get(key) - } else { - return false - } - } - - return x.set.has(value) - } - - const add = (keys, value) => { - let x = root - - for (const key of keys) { - if (!x.children.has(key)) { - x.children.set(key, mkMapSet()) - } - - x = x.children.get(key) - } - - x.set.add(value) - } - - return { root, has, add } -} - -export async function* tracksIterator(statusesIterator) { - const known = pathSet() - +export async function* tracksIterator(statusesIterator, cache) { yield* execPipe( statusesIterator, - asyncFilter(knownByReferer(known)), - asyncMap(processReferer), - asyncFilter(knownByMedia(known)), - asyncMap(processMedia) + asyncFilter(track => track != null), // should not be necessary + asyncFilter(notKnown(cache)), + asyncMap(completeTrack) ) } -const knownByReferer = known => track => { +const notKnown = cache => track => { if (!track) { - console.error(`No status, should not happen here`) - return false - } else { - switch (track.referer.credentials.type) { - default: - throw new Error() - - case 'mastodon': - const path = [ - 'referer', - 'mastodon', - track.referer.credentials.domain - ] - - const id = track.referer.credentials.id - - if (known.has(path, id)) { - console.log(`Drop already processed referer ${id}`) - return false - } else { - known.add(path, id) - return true - } - } - } -} - -const knownByMedia = known => track => { - if (track !== null) { - switch (track.media.credentials.type) { - default: - throw new Error() - - case 'youtube': - const path = [ - 'media', - 'youtube' - ] - - const id = track.media.credentials.id - - if (known.has(path, id)) { - console.log(`Drop already processed media ${id}`) - return false - } else { - known.add(path, id) - return true - } - } - } else { + console.error(`No track, should not happen here`) return false } + + const isKnown = (values) => { + if (cache.has(values)) { + console.log(`Drop already processed ${values.join(':')}`) + return true + } else { + cache.add(values) + return false + } + } + + switch (track.referer.credentials.type) { + default: + throw new Error() + + case 'mastodon': + if (isKnown([ + 'referer', + 'mastodon', + track.referer.credentials.domain, + track.referer.credentials.id + ])) { + return false + } + + break; + } + + if (track.media == null) { + return false + } + + switch (track.media.credentials.type) { + default: + throw new Error() + + case 'youtube': + if (isKnown([ + 'media', + 'youtube', + track.media.credentials.id + ])) { + return false + } + + break + } + + return true } -const processReferer = track => { - const urls = getUrls(track.content) +const completeTrack = async track => { + const metadata = await fetchMetadata(track.media) + return { ...track, title: metadata.title } +} - const media = execPipe( +export const urlsToMedia = urls => { + return execPipe( urls, map(parseSource), findOr(null, x => x !== null) ) - - if (media) { - return { ...track, media } - } else { - return null - } -} - -const processMedia = async track => { - const metadata = await fetchMetadata(track.media) - return { ...track, title: metadata.title } } const parseSource = (url) => { diff --git a/src/services/radio.js b/src/services/radio.js new file mode 100644 index 0000000..31de42c --- /dev/null +++ b/src/services/radio.js @@ -0,0 +1,9 @@ +import { asyncPrepend } from 'iter-tools' +import { hashtagsIterator } from '/services/mastodon.js' +import { tracksIterator } from '/services/misc.js' + +export const radioIterator = (domain, hashtags, cache) => + tracksIterator(hashtagsIterator(domain, hashtags), cache) + +export const radioShareIterator = (track, domain, hashtags, cache) => + tracksIterator(asyncPrepend(track, hashtagsIterator(domain, hashtags)), cache) \ No newline at end of file