From 0daa00b9dac9fe4c527d4eac4a40f334078be69c Mon Sep 17 00:00:00 2001 From: shamAnimates <145093437+shamAnimates@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:27:31 +0530 Subject: [PATCH] fix(extension): continue X imports across empty pages --- .../utils/twitter-import.test.ts | 249 ++++++++++++++++++ .../browser-extension/utils/twitter-import.ts | 49 +++- .../utils/twitter-pagination.test.ts | 22 ++ .../utils/twitter-pagination.ts | 10 + 4 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 apps/browser-extension/utils/twitter-import.test.ts create mode 100644 apps/browser-extension/utils/twitter-pagination.test.ts create mode 100644 apps/browser-extension/utils/twitter-pagination.ts diff --git a/apps/browser-extension/utils/twitter-import.test.ts b/apps/browser-extension/utils/twitter-import.test.ts new file mode 100644 index 00000000..7d60ebaa --- /dev/null +++ b/apps/browser-extension/utils/twitter-import.test.ts @@ -0,0 +1,249 @@ +import { afterEach, describe, expect, mock, test } from "bun:test" + +mock.module("#imports", () => ({ + storage: { + defineItem(key: string) { + return { + async getValue() { + switch (key) { + case "local:bearer-token": + return "sm_test" + case "session:twitter-cookie": + return "cookie" + case "session:twitter-csrf": + return "csrf" + case "session:twitter-auth-token": + return "auth" + default: + return null + } + }, + async setValue() {}, + } + }, + }, +})) + +const { TwitterImporter } = await import("./twitter-import") + +const originalFetch = globalThis.fetch + +afterEach(() => { + globalThis.fetch = originalFetch +}) + +type TimelineEntry = { + entryId: string + sortIndex: string + content: Record +} + +function timelinePage(entries: TimelineEntry[]) { + return { + data: { + bookmark_timeline_v2: { + timeline: { + instructions: [{ type: "TimelineAddEntries", entries }], + }, + }, + }, + } +} + +function cursorEntry(value: string): TimelineEntry { + return { + entryId: `cursor-bottom-${value}`, + sortIndex: "0", + content: { value }, + } +} + +function tombstoneEntry(id: string): TimelineEntry { + return { + entryId: `tweet-${id}`, + sortIndex: "0", + content: { + itemContent: { + tweet_results: { result: { __typename: "TweetTombstone" } }, + }, + }, + } +} + +function tweetEntry(id: string): TimelineEntry { + return { + entryId: `tweet-${id}`, + sortIndex: "0", + content: { + itemContent: { + tweet_results: { + result: { + __typename: "Tweet", + legacy: { + favorite_count: 1, + created_at: "Mon Jan 01 00:00:00 +0000 2024", + id_str: id, + full_text: `Tweet ${id}`, + }, + core: { + user_results: { + result: { + legacy: { + id_str: "author-1", + name: "Author", + profile_image_url_https: "", + screen_name: "author", + verified: false, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +function jsonResponse(body: unknown, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }) +} + +function installFetch(responses: Response[]) { + const twitterRequests: string[] = [] + const savedIds: string[][] = [] + + globalThis.fetch = mock(async (input, init) => { + const url = String(input) + + if (url.startsWith("https://x.com/")) { + twitterRequests.push(url) + const response = responses.shift() + if (!response) { + throw new Error(`Unexpected Twitter request: ${url}`) + } + return response + } + + if (url.endsWith("/v3/documents/batch")) { + const body = JSON.parse(String(init?.body)) as { + documents: Array<{ customId: string }> + } + savedIds.push(body.documents.map((document) => document.customId)) + return jsonResponse({ success: true }) + } + + throw new Error(`Unexpected request: ${url}`) + }) as unknown as typeof fetch + + return { savedIds, twitterRequests } +} + +function requestCursors(requests: string[]) { + return requests.map((request) => { + const variables = new URL(request).searchParams.get("variables") + return variables + ? (JSON.parse(variables) as { cursor?: string }).cursor + : undefined + }) +} + +function createCallbacks() { + const completions: number[] = [] + const errors: Error[] = [] + const progress: string[] = [] + + return { + completions, + config: { + onComplete: async (total: number) => { + completions.push(total) + }, + onError: async (error: Error) => { + errors.push(error) + }, + onProgress: async (message: string) => { + progress.push(message) + }, + }, + errors, + progress, + } +} + +describe("TwitterImporter pagination", () => { + test("continues past an untransformable page when a new cursor is present", async () => { + const { savedIds, twitterRequests } = installFetch([ + jsonResponse( + timelinePage([tombstoneEntry("gone"), cursorEntry("cursor-a")]), + ), + jsonResponse(timelinePage([tweetEntry("later-tweet")])), + ]) + const { completions, config, errors } = createCallbacks() + const waits: number[] = [] + const importer = new TwitterImporter(config, async (milliseconds) => { + waits.push(milliseconds) + }) + + await importer.startImport() + + expect(errors).toEqual([]) + expect(completions).toEqual([1]) + expect(requestCursors(twitterRequests)).toEqual([undefined, "cursor-a"]) + expect(savedIds).toEqual([["later-tweet"]]) + expect(waits).toEqual([1000]) + }) + + test("reports a cyclic cursor chain instead of claiming completion", async () => { + const { twitterRequests } = installFetch([ + jsonResponse(timelinePage([cursorEntry("cursor-a")])), + jsonResponse(timelinePage([cursorEntry("cursor-b")])), + jsonResponse(timelinePage([cursorEntry("cursor-a")])), + ]) + const { completions, config, errors } = createCallbacks() + const importer = new TwitterImporter(config, async () => {}) + + await importer.startImport() + + expect(completions).toEqual([]) + expect(errors.map((error) => error.message)).toEqual([ + "X returned a repeated pagination cursor. Import stopped after 0 tweets to avoid a loop.", + ]) + expect(requestCursors(twitterRequests)).toEqual([ + undefined, + "cursor-a", + "cursor-b", + ]) + }) + + test("retries a rate-limited cursor without treating it as a cycle", async () => { + const { savedIds, twitterRequests } = installFetch([ + jsonResponse(timelinePage([cursorEntry("cursor-a")])), + jsonResponse({ error: "rate limited" }, 429), + jsonResponse(timelinePage([tweetEntry("after-retry")])), + ]) + const { completions, config, errors, progress } = createCallbacks() + const waits: number[] = [] + const importer = new TwitterImporter(config, async (milliseconds) => { + waits.push(milliseconds) + }) + + await importer.startImport() + + expect(errors).toEqual([]) + expect(completions).toEqual([1]) + expect(requestCursors(twitterRequests)).toEqual([ + undefined, + "cursor-a", + "cursor-a", + ]) + expect(savedIds).toEqual([["after-retry"]]) + expect(waits).toEqual([1000, 60000]) + expect(progress).toContain( + "Rate limit reached. Waiting for 60 seconds before retrying...", + ) + }) +}) diff --git a/apps/browser-extension/utils/twitter-import.ts b/apps/browser-extension/utils/twitter-import.ts index a9ab243a..837b6375 100644 --- a/apps/browser-extension/utils/twitter-import.ts +++ b/apps/browser-extension/utils/twitter-import.ts @@ -6,6 +6,7 @@ import { saveAllTweets } from "./api" import type { MemoryPayload } from "./types" import { createTwitterAPIHeaders } from "./twitter-auth" import { getTwitterTokens } from "./storage" +import { getNextUnseenCursor } from "./twitter-pagination" import { BOOKMARKS_URL, BOOKMARK_COLLECTION_URL, @@ -20,6 +21,11 @@ export type ImportProgressCallback = (message: string) => Promise export type ImportCompleteCallback = (totalImported: number) => Promise +export type TwitterImportWait = (milliseconds: number) => Promise + +const waitFor: TwitterImportWait = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)) + export interface TwitterImportConfig { isFolderImport?: boolean bookmarkCollectionId?: string @@ -39,6 +45,8 @@ export interface TwitterImportConfig { class RateLimiter { private waitTime = 60000 // Start with 1 minute + constructor(private readonly wait: TwitterImportWait = waitFor) {} + async handleRateLimit(onProgress: ImportProgressCallback): Promise { const waitTimeInSeconds = this.waitTime / 1000 @@ -46,7 +54,7 @@ class RateLimiter { `Rate limit reached. Waiting for ${waitTimeInSeconds} seconds before retrying...`, ) - await new Promise((resolve) => setTimeout(resolve, this.waitTime)) + await this.wait(this.waitTime) this.waitTime *= 2 // Exponential backoff } @@ -60,9 +68,14 @@ class RateLimiter { */ export class TwitterImporter { private importInProgress = false - private rateLimiter = new RateLimiter() + private rateLimiter: RateLimiter - constructor(private config: TwitterImportConfig) {} + constructor( + private config: TwitterImportConfig, + private readonly wait: TwitterImportWait = waitFor, + ) { + this.rateLimiter = new RateLimiter(wait) + } /** * Starts the import process for all Twitter bookmarks @@ -95,6 +108,7 @@ export class TwitterImporter { cursor = "", totalImported = 0, uniqueGroupId = "twitter_bookmarks", + seenCursors = new Set(), ): Promise { try { // Use a local variable to track imported count @@ -134,7 +148,12 @@ export class TwitterImporter { if (response.status === 429) { await this.rateLimiter.handleRateLimit(this.config.onProgress) - return this.batchImportAll(cursor, totalImported, uniqueGroupId) + return this.batchImportAll( + cursor, + totalImported, + uniqueGroupId, + seenCursors, + ) } throw new Error( `Failed to fetch data: ${response.status} - ${errorText}`, @@ -193,10 +212,26 @@ export class TwitterImporter { data.data?.bookmark_collection_timeline?.timeline?.instructions || [] const nextCursor = extractNextCursor(instructions) + const cursorToFetch = getNextUnseenCursor(nextCursor, seenCursors) - if (nextCursor && tweets.length > 0) { - await new Promise((resolve) => setTimeout(resolve, 1000)) // Rate limiting - await this.batchImportAll(nextCursor, importedCount, uniqueGroupId) + if (nextCursor && !cursorToFetch) { + await this.config.onError( + new Error( + `X returned a repeated pagination cursor. Import stopped after ${importedCount} tweets to avoid a loop.`, + ), + ) + return + } + + if (cursorToFetch) { + seenCursors.add(cursorToFetch) + await this.wait(1000) // Rate limiting + await this.batchImportAll( + cursorToFetch, + importedCount, + uniqueGroupId, + seenCursors, + ) } else { await this.config.onComplete(importedCount) } diff --git a/apps/browser-extension/utils/twitter-pagination.test.ts b/apps/browser-extension/utils/twitter-pagination.test.ts new file mode 100644 index 00000000..aace2aca --- /dev/null +++ b/apps/browser-extension/utils/twitter-pagination.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "bun:test" +import { getNextUnseenCursor } from "./twitter-pagination" + +describe("getNextUnseenCursor", () => { + it("continues to a new cursor without requiring tweets on the current page", () => { + expect(getNextUnseenCursor("cursor-2", new Set(["cursor-1"]))).toBe( + "cursor-2", + ) + }) + + it("stops when there is no next cursor", () => { + expect(getNextUnseenCursor(null, new Set())).toBeNull() + expect(getNextUnseenCursor("", new Set())).toBeNull() + }) + + it("stops repeated and cyclic cursors", () => { + const seenCursors = new Set(["cursor-1", "cursor-2"]) + + expect(getNextUnseenCursor("cursor-2", seenCursors)).toBeNull() + expect(getNextUnseenCursor("cursor-1", seenCursors)).toBeNull() + }) +}) diff --git a/apps/browser-extension/utils/twitter-pagination.ts b/apps/browser-extension/utils/twitter-pagination.ts new file mode 100644 index 00000000..f1d092c5 --- /dev/null +++ b/apps/browser-extension/utils/twitter-pagination.ts @@ -0,0 +1,10 @@ +export function getNextUnseenCursor( + nextCursor: string | null, + seenCursors: ReadonlySet, +): string | null { + if (!nextCursor || seenCursors.has(nextCursor)) { + return null + } + + return nextCursor +}