fix: large streaming gaps by refetching (#1362)

fixes #1356

src/routes/_actions/stream/fillStreamingGap.js

@@ -1,15 +1,23 @@
-// TODO: should probably just keep fetching timeline items in the gap, not stop at 40
 import { addStatusesOrNotifications } from '../addStatusOrNotification'
 import { getTimeline } from '../../_api/timelines'
 
-const TIMELINE_GAP_BATCH_SIZE = 40
+const TIMELINE_GAP_BATCH_SIZE = 20 // Mastodon timeline API maximum limit
+const MAX_NUM_REQUESTS = 15 // to avoid getting caught in an infinite loop somehow
 
 // fill in the "streaming gap" – i.e. fetch the most recent items so that there isn't
 // a big gap in the timeline if you haven't looked at it in awhile
 export async function fillStreamingGap (instanceName, accessToken, timelineName, firstTimelineItemId) {
-  let newTimelineItems = await getTimeline(instanceName, accessToken,
-    timelineName, null, firstTimelineItemId, TIMELINE_GAP_BATCH_SIZE)
-  if (newTimelineItems.length) {
-    addStatusesOrNotifications(instanceName, timelineName, newTimelineItems)
-  }
+  let maxId = null
+  let numRequests = 0
+  let newTimelineItems
+
+  do {
+    numRequests++
+    newTimelineItems = await getTimeline(instanceName, accessToken,
+      timelineName, maxId, firstTimelineItemId, TIMELINE_GAP_BATCH_SIZE)
+    if (newTimelineItems.length) {
+      addStatusesOrNotifications(instanceName, timelineName, newTimelineItems)
+      maxId = newTimelineItems[newTimelineItems.length - 1].id
+    }
+  } while (numRequests < MAX_NUM_REQUESTS && newTimelineItems.length === TIMELINE_GAP_BATCH_SIZE)
 }

tests/spec/107-streaming-gap.js

@@ -2,7 +2,14 @@ import { loginAsFoobar } from '../roles'
 import {
   forceOffline,
   forceOnline,
-  getNthStatus, homeNavButton, localTimelineNavButton, notificationBadge, notificationsNavButton, sleep
+  getNthStatus,
+  homeNavButton,
+  localTimelineNavButton,
+  notificationBadge,
+  notificationsNavButton,
+  sleep,
+  times,
+  validateTimeline
 } from '../utils'
 import {
   postAs
@@ -82,3 +89,26 @@ test('fills timeline gap while away from notifications timeline - badge updates'
     .expect(notificationBadge.exists).notOk()
     .expect(getNthStatus(1).innerText).contains('sneaky mention!', { timeout })
 })
+
+test('fills a large gap while away from home timeline', async t => {
+  await loginAsFoobar(t)
+  await t
+    .expect(getNthStatus(1).exists).ok({ timeout })
+    .hover(getNthStatus(1))
+  await postAs('admin', 'hello hello world')
+  await t.expect(getNthStatus(1).innerText).contains('hello hello world', { timeout })
+  await forceOffline()
+  await sleep(1000)
+  const GAP_SIZE = 60
+  for (let i = 0; i < GAP_SIZE; i++) {
+    await postAs('admin', `posted while offline #${i + 1}`)
+  }
+  await sleep(1000)
+  await forceOnline()
+  await t.expect(getNthStatus(1).innerText).contains(`posted while offline #${GAP_SIZE}`, {
+    timeout: timeout * 2
+  })
+  const expected = times(GAP_SIZE, i => ({ content: `posted while offline #${GAP_SIZE - i}` }))
+    .concat([{ content: 'hello hello world' }])
+  await validateTimeline(t, expected)
+})