/** * Twitter Bookmarks Import Module * Handles the import process for Twitter bookmarks */ import { saveAllTweets } from "./api"; import { createTwitterAPIHeaders, getTwitterTokens } from "./twitter-auth"; import { BOOKMARKS_URL, buildRequestVariables, extractNextCursor, getAllTweets, type TwitterAPIResponse, } from "./twitter-utils"; export type ImportProgressCallback = (message: string) => Promise; export type ImportCompleteCallback = (totalImported: number) => Promise; export interface TwitterImportConfig { onProgress: ImportProgressCallback; onComplete: ImportCompleteCallback; onError: (error: Error) => Promise; } /** * Rate limiting configuration */ class RateLimiter { private waitTime = 60000; // Start with 1 minute async handleRateLimit(onProgress: ImportProgressCallback): Promise { const waitTimeInSeconds = this.waitTime / 1000; await onProgress( `Rate limit reached. Waiting for ${waitTimeInSeconds} seconds before retrying...`, ); await new Promise((resolve) => setTimeout(resolve, this.waitTime)); this.waitTime *= 2; // Exponential backoff } reset(): void { this.waitTime = 60000; } } /** * Main class for handling Twitter bookmarks import */ export class TwitterImporter { private importInProgress = false; private rateLimiter = new RateLimiter(); constructor(private config: TwitterImportConfig) {} /** * Starts the import process for all Twitter bookmarks * @returns Promise that resolves when import is complete */ async startImport(): Promise { if (this.importInProgress) { throw new Error("Import already in progress"); } this.importInProgress = true; const uniqueGroupId = crypto.randomUUID(); try { await this.batchImportAll("", 0, uniqueGroupId); this.rateLimiter.reset(); } catch (error) { await this.config.onError(error as Error); } finally { this.importInProgress = false; } } /** * Recursive function to import all bookmarks with pagination * @param cursor - Pagination cursor for Twitter API * @param totalImported - Number of tweets imported so far */ private async batchImportAll( cursor = "", totalImported = 0, uniqueGroupId = "twitter_bookmarks", ): Promise { try { // Use a local variable to track imported count let importedCount = totalImported; // Get authentication tokens const tokens = await getTwitterTokens(); if (!tokens) { await this.config.onProgress( "Please visit Twitter/X first to capture authentication tokens", ); return; } // Create headers for API request const headers = createTwitterAPIHeaders(tokens); // Build API request with pagination const variables = buildRequestVariables(cursor); const urlWithCursor = cursor ? `${BOOKMARKS_URL}&variables=${encodeURIComponent(JSON.stringify(variables))}` : BOOKMARKS_URL; const response = await fetch(urlWithCursor, { method: "GET", headers, redirect: "follow", }); if (!response.ok) { const errorText = await response.text(); console.error(`Twitter API Error ${response.status}:`, errorText); if (response.status === 429) { await this.rateLimiter.handleRateLimit(this.config.onProgress); return this.batchImportAll(cursor, totalImported, uniqueGroupId); } throw new Error( `Failed to fetch data: ${response.status} - ${errorText}`, ); } const data: TwitterAPIResponse = await response.json(); const tweets = getAllTweets(data); const documents: MemoryPayload[] = []; // Convert tweets to MemoryPayload for (const tweet of tweets) { try { const metadata = { sm_source: "consumer", tweet_id: tweet.id_str, author: tweet.user.screen_name, created_at: tweet.created_at, likes: tweet.favorite_count, retweets: tweet.retweet_count || 0, sm_internal_group_id: uniqueGroupId, }; documents.push({ containerTags: ["sm_project_twitter_bookmarks"], content: `https://x.com/${tweet.user.screen_name}/status/${tweet.id_str}`, metadata, customId: tweet.id_str, }); importedCount++; await this.config.onProgress( `Imported ${importedCount} tweets, so far...`, ); } catch (error) { console.error("Error importing tweet:", error); } } try { if (documents.length > 0) { await saveAllTweets(documents); } console.log("Tweets saved"); console.log("Documents:", documents); } catch (error) { console.error("Error saving tweets batch:", error); await this.config.onError(error as Error); return; } // Handle pagination const instructions = data.data?.bookmark_timeline_v2?.timeline?.instructions; const nextCursor = extractNextCursor(instructions || []); console.log("Next cursor:", nextCursor); console.log("Tweets length:", tweets.length); if (nextCursor && tweets.length > 0) { await new Promise((resolve) => setTimeout(resolve, 1000)); // Rate limiting await this.batchImportAll(nextCursor, importedCount, uniqueGroupId); } else { await this.config.onComplete(importedCount); } } catch (error) { console.error("Batch import error:", error); await this.config.onError(error as Error); } } }