From e3b6a9b758f062495cebdbe3fe3e7fc8fe1f1fd4 Mon Sep 17 00:00:00 2001 From: Nolan Lawson Date: Thu, 8 Mar 2018 18:31:59 -0800 Subject: [PATCH] fix offline threads --- routes/_database/cleanup.js | 12 ++-- routes/_database/constants.js | 1 + routes/_database/databaseLifecycle.js | 7 ++- routes/_database/timelines.js | 82 ++++++++++++++++++--------- 4 files changed, 69 insertions(+), 33 deletions(-) diff --git a/routes/_database/cleanup.js b/routes/_database/cleanup.js index 2c966f01..3f4f2831 100644 --- a/routes/_database/cleanup.js +++ b/routes/_database/cleanup.js @@ -7,6 +7,7 @@ import { RELATIONSHIPS_STORE, STATUS_TIMELINES_STORE, STATUSES_STORE, + THREADS_STORE, TIMESTAMP } from './constants' import debounce from 'lodash/debounce' @@ -30,12 +31,13 @@ function batchedGetAll (callGetAll, callback) { nextBatch() } -function cleanupStatuses (statusesStore, statusTimelinesStore, cutoff) { +function cleanupStatuses (statusesStore, statusTimelinesStore, threadsStore, cutoff) { batchedGetAll( () => statusesStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE), results => { results.forEach(result => { statusesStore.delete(result.id) + threadsStore.delete(result.id) let req = statusTimelinesStore.index('statusId').getAll(IDBKeyRange.only(result.id)) req.onsuccess = e => { let results = e.target.result @@ -98,7 +100,8 @@ async function cleanup (instanceName) { NOTIFICATIONS_STORE, NOTIFICATION_TIMELINES_STORE, ACCOUNTS_STORE, - RELATIONSHIPS_STORE + RELATIONSHIPS_STORE, + THREADS_STORE ] await dbPromise(db, storeNames, 'readwrite', (stores) => { let [ @@ -107,12 +110,13 @@ async function cleanup (instanceName) { notificationsStore, notificationTimelinesStore, accountsStore, - relationshipsStore + relationshipsStore, + threadsStore ] = stores let cutoff = Date.now() - TIME_AGO - cleanupStatuses(statusesStore, statusTimelinesStore, cutoff) + cleanupStatuses(statusesStore, statusTimelinesStore, threadsStore, cutoff) cleanupNotifications(notificationsStore, notificationTimelinesStore, cutoff) cleanupAccounts(accountsStore, cutoff) cleanupRelationships(relationshipsStore, cutoff) diff --git a/routes/_database/constants.js b/routes/_database/constants.js index 87803be3..f7ec6654 100644 --- a/routes/_database/constants.js +++ b/routes/_database/constants.js @@ -6,6 +6,7 @@ export const RELATIONSHIPS_STORE = 'relationships' export const NOTIFICATIONS_STORE = 'notifications' export const NOTIFICATION_TIMELINES_STORE = 'notification_timelines' export const PINNED_STATUSES_STORE = 'pinned_statuses' +export const THREADS_STORE = 'threads' export const TIMESTAMP = '__pinafore_ts' export const ACCOUNT_ID = '__pinafore_acct_id' diff --git a/routes/_database/databaseLifecycle.js b/routes/_database/databaseLifecycle.js index 6da6ae2f..095b3cbb 100644 --- a/routes/_database/databaseLifecycle.js +++ b/routes/_database/databaseLifecycle.js @@ -7,13 +7,13 @@ import { NOTIFICATIONS_STORE, NOTIFICATION_TIMELINES_STORE, PINNED_STATUSES_STORE, - TIMESTAMP, REBLOG_ID + TIMESTAMP, REBLOG_ID, THREADS_STORE } from './constants' const openReqs = {} const databaseCache = {} -const DB_VERSION = 3 +const DB_VERSION = 4 export function getDatabase (instanceName) { if (!instanceName) { @@ -55,6 +55,9 @@ export function getDatabase (instanceName) { if (e.oldVersion < 3) { tx.objectStore(NOTIFICATIONS_STORE).createIndex('statusId', 'statusId') } + if (e.oldVersion < 4) { + db.createObjectStore(THREADS_STORE, {keyPath: 'id'}) + } } req.onsuccess = () => resolve(req.result) }) diff --git a/routes/_database/timelines.js b/routes/_database/timelines.js index 12bd4766..b2a77872 100644 --- a/routes/_database/timelines.js +++ b/routes/_database/timelines.js @@ -14,7 +14,7 @@ import { STATUSES_STORE, ACCOUNT_ID, REBLOG_ID, - STATUS_ID + STATUS_ID, THREADS_STORE } from './constants' function createTimelineKeyRange (timeline, maxId) { @@ -24,14 +24,6 @@ function createTimelineKeyRange (timeline, maxId) { return IDBKeyRange.bound(start, end, true, true) } -// special case for threads – these are in chronological order rather than reverse -// chronological order, and we fetch everything all at once rather than paginating -function createKeyRangeForStatusThread (timeline) { - let start = timeline + '\u0000' - let end = timeline + '\u0000\uffff' - return IDBKeyRange.bound(start, end, true, true) -} - function cacheStatus (status, instanceName) { setInCache(statusesCache, instanceName, status.id, status) setInCache(accountsCache, instanceName, status.account.id, status.account) @@ -69,18 +61,9 @@ async function getStatusTimeline (instanceName, timeline, maxId, limit) { const db = await getDatabase(instanceName) return dbPromise(db, storeNames, 'readonly', (stores, callback) => { let [ timelineStore, statusesStore, accountsStore ] = stores - // Status threads are a special case - these are in forward chronological order - // and we fetch them all at once instead of paginating. - let isStatusThread = timeline.startsWith('status/') - let getReq = isStatusThread - ? timelineStore.getAll(createKeyRangeForStatusThread(timeline)) - : timelineStore.getAll(createTimelineKeyRange(timeline, maxId), limit) - + let getReq = timelineStore.getAll(createTimelineKeyRange(timeline, maxId), limit) getReq.onsuccess = e => { let timelineResults = e.target.result - if (isStatusThread) { - timelineResults = timelineResults.reverse() - } let res = new Array(timelineResults.length) timelineResults.forEach((timelineResult, i) => { fetchStatus(statusesStore, accountsStore, timelineResult.statusId, status => { @@ -92,10 +75,33 @@ async function getStatusTimeline (instanceName, timeline, maxId, limit) { }) } +async function getStatusThread (instanceName, statusId) { + let storeNames = [THREADS_STORE, STATUSES_STORE, ACCOUNTS_STORE] + const db = await getDatabase(instanceName) + return dbPromise(db, storeNames, 'readonly', (stores, callback) => { + let [ threadsStore, statusesStore, accountsStore ] = stores + threadsStore.get(statusId).onsuccess = e => { + let thread = e.target.result.thread + let res = new Array(thread.length) + thread.forEach((otherStatusId, i) => { + fetchStatus(statusesStore, accountsStore, otherStatusId, status => { + res[i] = status + }) + }) + callback(res) + } + }) +} + export async function getTimeline (instanceName, timeline, maxId = null, limit = 20) { - return timeline === 'notifications' - ? getNotificationTimeline(instanceName, timeline, maxId, limit) - : getStatusTimeline(instanceName, timeline, maxId, limit) + if (timeline === 'notifications') { + return getNotificationTimeline(instanceName, timeline, maxId, limit) + } else if (timeline.startsWith('status/')) { + let statusId = timeline.split('/').slice(-1)[0] + return getStatusThread(instanceName, statusId) + } else { + return getStatusTimeline(instanceName, timeline, maxId, limit) + } } // @@ -177,7 +183,6 @@ function createTimelineId (timeline, id) { } async function insertTimelineNotifications (instanceName, timeline, notifications) { - /* no await */ scheduleCleanup() for (let notification of notifications) { setInCache(notificationsCache, instanceName, notification.id, notification) setInCache(accountsCache, instanceName, notification.account.id, notification.account) @@ -200,7 +205,6 @@ async function insertTimelineNotifications (instanceName, timeline, notification } async function insertTimelineStatuses (instanceName, timeline, statuses) { - /* no await */ scheduleCleanup() for (let status of statuses) { cacheStatus(status, instanceName) } @@ -218,10 +222,34 @@ async function insertTimelineStatuses (instanceName, timeline, statuses) { }) } +async function insertStatusThread (instanceName, statusId, statuses) { + for (let status of statuses) { + cacheStatus(status, instanceName) + } + const db = await getDatabase(instanceName) + let storeNames = [THREADS_STORE, STATUSES_STORE, ACCOUNTS_STORE] + await dbPromise(db, storeNames, 'readwrite', (stores) => { + let [ threadsStore, statusesStore, accountsStore ] = stores + threadsStore.put({ + id: statusId, + thread: statuses.map(_ => _.id) + }) + for (let status of statuses) { + storeStatus(statusesStore, accountsStore, status) + } + }) +} + export async function insertTimelineItems (instanceName, timeline, timelineItems) { - return timeline === 'notifications' - ? insertTimelineNotifications(instanceName, timeline, timelineItems) - : insertTimelineStatuses(instanceName, timeline, timelineItems) + /* no await */ scheduleCleanup() + if (timeline === 'notifications') { + return insertTimelineNotifications(instanceName, timeline, timelineItems) + } else if (timeline.startsWith('status/')) { + let statusId = timeline.split('/').slice(-1)[0] + return insertStatusThread(instanceName, statusId, timelineItems) + } else { + return insertTimelineStatuses(instanceName, timeline, timelineItems) + } } //