fix(extension): continue X imports across empty pages

apps/browser-extension/utils/twitter-import.test.ts

@@ -0,0 +1,249 @@
+import { afterEach, describe, expect, mock, test } from "bun:test"
+
+mock.module("#imports", () => ({
+	storage: {
+		defineItem(key: string) {
+			return {
+				async getValue() {
+					switch (key) {
+						case "local:bearer-token":
+							return "sm_test"
+						case "session:twitter-cookie":
+							return "cookie"
+						case "session:twitter-csrf":
+							return "csrf"
+						case "session:twitter-auth-token":
+							return "auth"
+						default:
+							return null
+					}
+				},
+				async setValue() {},
+			}
+		},
+	},
+}))
+
+const { TwitterImporter } = await import("./twitter-import")
+
+const originalFetch = globalThis.fetch
+
+afterEach(() => {
+	globalThis.fetch = originalFetch
+})
+
+type TimelineEntry = {
+	entryId: string
+	sortIndex: string
+	content: Record<string, unknown>
+}
+
+function timelinePage(entries: TimelineEntry[]) {
+	return {
+		data: {
+			bookmark_timeline_v2: {
+				timeline: {
+					instructions: [{ type: "TimelineAddEntries", entries }],
+				},
+			},
+		},
+	}
+}
+
+function cursorEntry(value: string): TimelineEntry {
+	return {
+		entryId: `cursor-bottom-${value}`,
+		sortIndex: "0",
+		content: { value },
+	}
+}
+
+function tombstoneEntry(id: string): TimelineEntry {
+	return {
+		entryId: `tweet-${id}`,
+		sortIndex: "0",
+		content: {
+			itemContent: {
+				tweet_results: { result: { __typename: "TweetTombstone" } },
+			},
+		},
+	}
+}
+
+function tweetEntry(id: string): TimelineEntry {
+	return {
+		entryId: `tweet-${id}`,
+		sortIndex: "0",
+		content: {
+			itemContent: {
+				tweet_results: {
+					result: {
+						__typename: "Tweet",
+						legacy: {
+							favorite_count: 1,
+							created_at: "Mon Jan 01 00:00:00 +0000 2024",
+							id_str: id,
+							full_text: `Tweet ${id}`,
+						},
+						core: {
+							user_results: {
+								result: {
+									legacy: {
+										id_str: "author-1",
+										name: "Author",
+										profile_image_url_https: "",
+										screen_name: "author",
+										verified: false,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+function jsonResponse(body: unknown, status = 200) {
+	return new Response(JSON.stringify(body), {
+		status,
+		headers: { "Content-Type": "application/json" },
+	})
+}
+
+function installFetch(responses: Response[]) {
+	const twitterRequests: string[] = []
+	const savedIds: string[][] = []
+
+	globalThis.fetch = mock(async (input, init) => {
+		const url = String(input)
+
+		if (url.startsWith("https://x.com/")) {
+			twitterRequests.push(url)
+			const response = responses.shift()
+			if (!response) {
+				throw new Error(`Unexpected Twitter request: ${url}`)
+			}
+			return response
+		}
+
+		if (url.endsWith("/v3/documents/batch")) {
+			const body = JSON.parse(String(init?.body)) as {
+				documents: Array<{ customId: string }>
+			}
+			savedIds.push(body.documents.map((document) => document.customId))
+			return jsonResponse({ success: true })
+		}
+
+		throw new Error(`Unexpected request: ${url}`)
+	}) as unknown as typeof fetch
+
+	return { savedIds, twitterRequests }
+}
+
+function requestCursors(requests: string[]) {
+	return requests.map((request) => {
+		const variables = new URL(request).searchParams.get("variables")
+		return variables
+			? (JSON.parse(variables) as { cursor?: string }).cursor
+			: undefined
+	})
+}
+
+function createCallbacks() {
+	const completions: number[] = []
+	const errors: Error[] = []
+	const progress: string[] = []
+
+	return {
+		completions,
+		config: {
+			onComplete: async (total: number) => {
+				completions.push(total)
+			},
+			onError: async (error: Error) => {
+				errors.push(error)
+			},
+			onProgress: async (message: string) => {
+				progress.push(message)
+			},
+		},
+		errors,
+		progress,
+	}
+}
+
+describe("TwitterImporter pagination", () => {
+	test("continues past an untransformable page when a new cursor is present", async () => {
+		const { savedIds, twitterRequests } = installFetch([
+			jsonResponse(
+				timelinePage([tombstoneEntry("gone"), cursorEntry("cursor-a")]),
+			),
+			jsonResponse(timelinePage([tweetEntry("later-tweet")])),
+		])
+		const { completions, config, errors } = createCallbacks()
+		const waits: number[] = []
+		const importer = new TwitterImporter(config, async (milliseconds) => {
+			waits.push(milliseconds)
+		})
+
+		await importer.startImport()
+
+		expect(errors).toEqual([])
+		expect(completions).toEqual([1])
+		expect(requestCursors(twitterRequests)).toEqual([undefined, "cursor-a"])
+		expect(savedIds).toEqual([["later-tweet"]])
+		expect(waits).toEqual([1000])
+	})
+
+	test("reports a cyclic cursor chain instead of claiming completion", async () => {
+		const { twitterRequests } = installFetch([
+			jsonResponse(timelinePage([cursorEntry("cursor-a")])),
+			jsonResponse(timelinePage([cursorEntry("cursor-b")])),
+			jsonResponse(timelinePage([cursorEntry("cursor-a")])),
+		])
+		const { completions, config, errors } = createCallbacks()
+		const importer = new TwitterImporter(config, async () => {})
+
+		await importer.startImport()
+
+		expect(completions).toEqual([])
+		expect(errors.map((error) => error.message)).toEqual([
+			"X returned a repeated pagination cursor. Import stopped after 0 tweets to avoid a loop.",
+		])
+		expect(requestCursors(twitterRequests)).toEqual([
+			undefined,
+			"cursor-a",
+			"cursor-b",
+		])
+	})
+
+	test("retries a rate-limited cursor without treating it as a cycle", async () => {
+		const { savedIds, twitterRequests } = installFetch([
+			jsonResponse(timelinePage([cursorEntry("cursor-a")])),
+			jsonResponse({ error: "rate limited" }, 429),
+			jsonResponse(timelinePage([tweetEntry("after-retry")])),
+		])
+		const { completions, config, errors, progress } = createCallbacks()
+		const waits: number[] = []
+		const importer = new TwitterImporter(config, async (milliseconds) => {
+			waits.push(milliseconds)
+		})
+
+		await importer.startImport()
+
+		expect(errors).toEqual([])
+		expect(completions).toEqual([1])
+		expect(requestCursors(twitterRequests)).toEqual([
+			undefined,
+			"cursor-a",
+			"cursor-a",
+		])
+		expect(savedIds).toEqual([["after-retry"]])
+		expect(waits).toEqual([1000, 60000])
+		expect(progress).toContain(
+			"Rate limit reached. Waiting for 60 seconds before retrying...",
+		)
+	})
+})

apps/browser-extension/utils/twitter-import.ts

@@ -6,6 +6,7 @@ import { saveAllTweets } from "./api"
 import type { MemoryPayload } from "./types"
 import { createTwitterAPIHeaders } from "./twitter-auth"
 import { getTwitterTokens } from "./storage"
+import { getNextUnseenCursor } from "./twitter-pagination"
 import {
 	BOOKMARKS_URL,
 	BOOKMARK_COLLECTION_URL,
@@ -20,6 +21,11 @@ export type ImportProgressCallback = (message: string) => Promise<void>
 
 export type ImportCompleteCallback = (totalImported: number) => Promise<void>
 
+export type TwitterImportWait = (milliseconds: number) => Promise<void>
+
+const waitFor: TwitterImportWait = (milliseconds) =>
+	new Promise((resolve) => setTimeout(resolve, milliseconds))
+
 export interface TwitterImportConfig {
 	isFolderImport?: boolean
 	bookmarkCollectionId?: string
@@ -39,6 +45,8 @@ export interface TwitterImportConfig {
 class RateLimiter {
 	private waitTime = 60000 // Start with 1 minute
 
+	constructor(private readonly wait: TwitterImportWait = waitFor) {}
+
 	async handleRateLimit(onProgress: ImportProgressCallback): Promise<void> {
 		const waitTimeInSeconds = this.waitTime / 1000
 
@@ -46,7 +54,7 @@ class RateLimiter {
 			`Rate limit reached. Waiting for ${waitTimeInSeconds} seconds before retrying...`,
 		)
 
-		await new Promise((resolve) => setTimeout(resolve, this.waitTime))
+		await this.wait(this.waitTime)
 		this.waitTime *= 2 // Exponential backoff
 	}
 
@@ -60,9 +68,14 @@ class RateLimiter {
  */
 export class TwitterImporter {
 	private importInProgress = false
-	private rateLimiter = new RateLimiter()
+	private rateLimiter: RateLimiter
 
-	constructor(private config: TwitterImportConfig) {}
+	constructor(
+		private config: TwitterImportConfig,
+		private readonly wait: TwitterImportWait = waitFor,
+	) {
+		this.rateLimiter = new RateLimiter(wait)
+	}
 
 	/**
 	 * Starts the import process for all Twitter bookmarks
@@ -95,6 +108,7 @@ export class TwitterImporter {
 		cursor = "",
 		totalImported = 0,
 		uniqueGroupId = "twitter_bookmarks",
+		seenCursors = new Set<string>(),
 	): Promise<void> {
 		try {
 			// Use a local variable to track imported count
@@ -134,7 +148,12 @@ export class TwitterImporter {
 
 				if (response.status === 429) {
 					await this.rateLimiter.handleRateLimit(this.config.onProgress)
-					return this.batchImportAll(cursor, totalImported, uniqueGroupId)
+					return this.batchImportAll(
+						cursor,
+						totalImported,
+						uniqueGroupId,
+						seenCursors,
+					)
 				}
 				throw new Error(
 					`Failed to fetch data: ${response.status} - ${errorText}`,
@@ -193,10 +212,26 @@ export class TwitterImporter {
 				data.data?.bookmark_collection_timeline?.timeline?.instructions ||
 				[]
 			const nextCursor = extractNextCursor(instructions)
+			const cursorToFetch = getNextUnseenCursor(nextCursor, seenCursors)
 
-			if (nextCursor && tweets.length > 0) {
-				await new Promise((resolve) => setTimeout(resolve, 1000)) // Rate limiting
-				await this.batchImportAll(nextCursor, importedCount, uniqueGroupId)
+			if (nextCursor && !cursorToFetch) {
+				await this.config.onError(
+					new Error(
+						`X returned a repeated pagination cursor. Import stopped after ${importedCount} tweets to avoid a loop.`,
+					),
+				)
+				return
+			}
+
+			if (cursorToFetch) {
+				seenCursors.add(cursorToFetch)
+				await this.wait(1000) // Rate limiting
+				await this.batchImportAll(
+					cursorToFetch,
+					importedCount,
+					uniqueGroupId,
+					seenCursors,
+				)
 			} else {
 				await this.config.onComplete(importedCount)
 			}

apps/browser-extension/utils/twitter-pagination.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from "bun:test"
+import { getNextUnseenCursor } from "./twitter-pagination"
+
+describe("getNextUnseenCursor", () => {
+	it("continues to a new cursor without requiring tweets on the current page", () => {
+		expect(getNextUnseenCursor("cursor-2", new Set(["cursor-1"]))).toBe(
+			"cursor-2",
+		)
+	})
+
+	it("stops when there is no next cursor", () => {
+		expect(getNextUnseenCursor(null, new Set())).toBeNull()
+		expect(getNextUnseenCursor("", new Set())).toBeNull()
+	})
+
+	it("stops repeated and cyclic cursors", () => {
+		const seenCursors = new Set(["cursor-1", "cursor-2"])
+
+		expect(getNextUnseenCursor("cursor-2", seenCursors)).toBeNull()
+		expect(getNextUnseenCursor("cursor-1", seenCursors)).toBeNull()
+	})
+})

apps/browser-extension/utils/twitter-pagination.ts

@@ -0,0 +1,10 @@
+export function getNextUnseenCursor(
+	nextCursor: string | null,
+	seenCursors: ReadonlySet<string>,
+): string | null {
+	if (!nextCursor || seenCursors.has(nextCursor)) {
+		return null
+	}
+
+	return nextCursor
+}