supermemory/apps/extension/src/twitter.ts

import { getBaseURL } from "@/lib/utils";
import { features } from "./twitter.constants";

const DEBUG = true;

const log = (message: string, ...args: any[]) => {
  if (DEBUG) {
    console.log("[twitter]", message, ...args);
  }
};

const logError = (message: string, error?: any) => {
  // Always log errors
  console.error(message, error);
};

export const waitForRequiredData = () => {
  return new Promise((resolve) => {
    const checkData = () => {
      chrome.storage.local.get(
        ["bookmarksApiId", "cookie", "csrf", "auth"],
        (result) => {
          if (
            result.bookmarksApiId &&
            result.cookie &&
            result.csrf &&
            result.auth
          ) {
            log("Got all data needed to fetch bookmarks");
            resolve(true);
          } else {
            setTimeout(checkData, 100); // Check again after 100ms
          }
        }
      );
    };
    checkData();
  });
};

export const handleExportXBookmarks = async () => {
  log("Received export request from popup");

  // First, create the Twitter tab
  const tab = await chrome.tabs.create({
    url: "https://x.com/i/bookmarks/all",
  });

  // Wait for the tab to finish loading
  await new Promise((resolve) => {
    const checkTab = () => {
      if (tab.id) {
        chrome.tabs.get(tab.id, (updatedTab) => {
          if (updatedTab.status === "complete") {
            resolve(true);
          } else {
            setTimeout(checkTab, 100);
          }
        });
      }
    };
    checkTab();
  });

  void chrome.storage.local.set({ attemptingImportCurrently: true });

  chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => {
    const currentTab = tabs[0];
    if (currentTab.id) {
      try {
        await chrome.scripting.executeScript({
          target: { tabId: currentTab.id },
          files: ["scripts/content.js"],
        });
      } catch (error) {
        console.error("Error injecting content script:", error);
      }
    }
  });

  // Wait for required data and start the bookmark export
  await waitForRequiredData();
  await sendProgressUpdate(0, 200);

  await getBookmarks();

  return { status: 200 };
};

const getNextCursor = (entries: any[]) => {
  const cursorEntry = entries.find((entry) =>
    entry.entryId.startsWith("cursor-bottom-")
  );
  return cursorEntry ? cursorEntry.content.value : null;
};

type MessageType = "IMPORT_PROGRESS_UPDATE" | "IMPORT_COMPLETE";

const sendMessageToTwitterTabs = async (
  message: {
    type: MessageType;
    payload: any;
  },
  retries = 3
) => {
  try {
    log(`Sending message of type ${message.type}:`, message.payload);

    const tabs = await chrome.tabs.query({
      url: ["*://*.twitter.com/*", "*://*.x.com/*"],
    });

    if (tabs.length === 0) {
      if (retries > 0) {
        // Wait and retry if no matching tab found
        await new Promise((resolve) => setTimeout(resolve, 1000));
        return sendMessageToTwitterTabs(message, retries - 1);
      }
      throw new Error("No Twitter tab found");
    }

    const messagePromises = tabs.map((tab) => {
      if (!tab.id) return Promise.resolve();
      return chrome.tabs.sendMessage(tab.id, message).catch((error) => {
        logError(`Failed to send message to tab ${tab.id}:`, error);
      });
    });

    await Promise.all(messagePromises);
    log(`Message sent to ${tabs.length} tabs`);
  } catch (error) {
    logError(`Error sending message of type ${message.type}:`, error);
  }
};

const sendProgressUpdate = (progress: number, status: number) => {
  return sendMessageToTwitterTabs({
    type: "IMPORT_PROGRESS_UPDATE",
    payload: { progress, status },
  });
};

const sendImportComplete = () => {
  return sendMessageToTwitterTabs({
    type: "IMPORT_COMPLETE",
    payload: { success: true },
  });
};

const getBookmarks = async (cursor = "", totalImported = 0, allTweets = []) => {
  try {
    const getSessionData = (): Promise<{
      cookie: string;
      csrf: string;
      auth: string;
      bookmarksApiId: string;
    }> => {
      return new Promise((resolve) => {
        const checkData = () => {
          chrome.storage.local.get(
            ["cookie", "csrf", "auth", "bookmarksApiId"],
            (result) => {
              if (
                result.cookie &&
                result.csrf &&
                result.auth &&
                result.bookmarksApiId
              ) {
                resolve({
                  cookie: result.cookie,
                  csrf: result.csrf,
                  auth: result.auth,
                  bookmarksApiId: result.bookmarksApiId,
                });
              } else {
                setTimeout(checkData, 3000); // Check again after 3 seconds
              }
            }
          );
        };
        checkData();
      });
    };

    const sessionResult = await getSessionData();
    const baseURL = await getBaseURL();

    const headers = new Headers();
    headers.append("Cookie", sessionResult.cookie);
    headers.append("X-Csrf-token", sessionResult.csrf);
    headers.append("Authorization", sessionResult.auth);

    const variables = {
      count: 100,
      cursor: cursor,
      includePromotedContent: false,
    };
    const API_URL = `https://x.com/i/api/graphql/${
      sessionResult.bookmarksApiId
    }/Bookmarks?features=${encodeURIComponent(
      JSON.stringify(features)
    )}&variables=${encodeURIComponent(JSON.stringify(variables))}` as const;

    const response = await fetch(API_URL, {
      method: "GET",
      headers: headers,
      redirect: "follow",
    });

    if (response.status === 429) {
      log("Rate limited, waiting 60 seconds before retrying...");
      await sendProgressUpdate(totalImported, 429);
      await new Promise((resolve) => setTimeout(resolve, 60000));
      return getBookmarks(cursor, totalImported, allTweets);
    }

    let data = (await response.json()) as any;

    // Handle 400 error with missing features
    if (response.status === 400 && data.errors?.[0]?.message) {
      const errorMsg = data.errors[0].message;
      const missingFeatures = errorMsg.match(
        /following features cannot be null: (.*?)$/
      )?.[1];

      if (missingFeatures) {
        const featuresList = missingFeatures.split(", ");
        const updatedFeatures = { ...features };

        featuresList.forEach((feature: string) => {
          updatedFeatures[feature as keyof typeof updatedFeatures] = true;
        });

        // Retry with updated features
        const newUrl = `https://x.com/i/api/graphql/${
          sessionResult.bookmarksApiId
        }/Bookmarks?features=${encodeURIComponent(
          JSON.stringify(updatedFeatures)
        )}&variables=${encodeURIComponent(JSON.stringify(variables))}`;

        const retryResponse = await fetch(newUrl, {
          method: "GET",
          headers: headers,
          redirect: "follow",
        });

        const retryData = await retryResponse.json();
        data = retryData;
      }
    }

    const entries =
      data.data?.bookmark_timeline_v2?.timeline?.instructions?.[0]?.entries ||
      [];

    const tweetEntries = entries.filter((entry: { entryId: string }) =>
      entry.entryId.startsWith("tweet-")
    );

    // Process tweets and create URLs
    const tweetUrls = tweetEntries
      .map((tweet: any) => {
        const tweetId = tweet?.entryId?.split("-")[1];
        const username =
          tweet?.content?.itemContent?.tweet_results?.result?.core?.user_results
            ?.result?.legacy?.screen_name;

        if (!tweetId) return null;
        return `https://x.com/${username || "supermemoryai"}/status/${tweetId}`;
      })
      .filter(Boolean);

    // Send progress update before processing batch
    await sendProgressUpdate(totalImported, 102); // 102 = Processing batch

    // Send all tweets in parallel
    const addRequests = tweetUrls.map((tweetUrl: string) =>
      fetch(`${baseURL}/backend/v1/add`, {
        method: "POST",
        headers: {
          "Content-Type": "application/json",
        },
        body: JSON.stringify({
          content: tweetUrl,
        }),
      })
    );

    const results = await Promise.all(addRequests);

    // Track different response statuses
    const statusCounts = results.reduce((acc: { [key: number]: number }, r) => {
      const status = r.status;
      acc[status] = (acc[status] || 0) + 1;
      return acc;
    }, {});

    // Log status counts
    Object.entries(statusCounts).forEach(([status, count]) => {
      log(`Status ${status}: ${count} tweets`);
    });

    // If we have any 409s, send that in the progress update
    if (statusCounts[409]) {
      await sendProgressUpdate(totalImported, 409);
    }

    // Consider request successful even if some tweets already exist (409)
    const failedRequests = results.filter(
      (r) => !r.ok && r.status !== 409
    ).length;
    if (failedRequests > 0) {
      await sendProgressUpdate(totalImported, 500); // 500 = Some requests failed
    }

    allTweets = allTweets.concat(tweetEntries);
    const newTweetsCount = tweetEntries.length;
    totalImported += newTweetsCount;

    // Update progress based on total tweets found so far
    const estimatedTotalTweets = Math.max(totalImported, 200); // Adjust estimate as we find more
    await sendProgressUpdate(
      totalImported,
      failedRequests > 0 ? 500 : statusCounts[409] ? 409 : 200
    );

    log(
      `Processing batch: ${newTweetsCount} new tweets, total: ${totalImported}`
    );
    const nextCursor = getNextCursor(entries);

    if (nextCursor && newTweetsCount > 0) {
      // Add delay between batches to avoid rate limiting
      await new Promise((resolve) => setTimeout(resolve, 1500));
      return getBookmarks(nextCursor, totalImported, allTweets);
    } else {
      // Final success update
      await sendImportComplete();

      chrome.storage.local.set({ bookmarks: allTweets }, () => {
        log("Bookmarks stored in local storage");
      });
      const timestamp = new Date().toISOString();
      chrome.storage.local.get(["successful_exports"], (result) => {
        const successful_exports = result.successful_exports || [];
        successful_exports.push({
          timestamp,
          tweetCount: allTweets.length,
        });
        chrome.storage.local.set({ successful_exports }, () => {
          log(`Export completed: ${allTweets.length} tweets`);
        });
      });
      void chrome.storage.local.set({ attemptingImportCurrently: false });

      // After successful import completion:
      window.postMessage(
        { type: "TWITTER_IMPORT_COMPLETE" },
        window.location.origin
      );
    }
  } catch (error) {
    logError("Error fetching bookmarks:", error);
    void chrome.storage.local.set({ attemptingImportCurrently: false });
  }
};

export const setupTwitterHeaderListener = () => {
  const extraOptions =
    chrome.runtime.getManifest().manifest_version === 2
      ? ["blocking", "requestHeaders"]
      : ["requestHeaders"];

  // Add extraHeaders only if we're on a Chromium-based browser
  if (!navigator.userAgent.includes("Firefox")) {
    extraOptions.push("extraHeaders");
  }

  chrome.webRequest.onBeforeSendHeaders.addListener(
    (details) => {
      try {
        // Validate input
        if (!details || !details.url) {
          console.error("Invalid details object received");
          return;
        }

        // Check if URL is from Twitter/X
        if (
          !(
            details.url.includes("x.com") || details.url.includes("twitter.com")
          )
        ) {
          return;
        }

        // Ensure requestHeaders exists
        if (!details.requestHeaders) {
          console.error("No request headers found");
          return;
        }

        // Check if stuff is already stored
        chrome.storage.local.get(
          ["bookmarksApiId", "cookie", "csrf", "auth"],
          (result) => {
            try {
              // Check if the URL matches the pattern for bookmarks API
              const bookmarksUrlPattern =
                /https:\/\/x\.com\/i\/api\/graphql\/([^/]+)\/Bookmarks\?/;
              const match = details.url.match(bookmarksUrlPattern);

              if (match?.[1] && !result.bookmarksApiId) {
                const bookmarksApiId = match[1];
                chrome.storage.local.set({ bookmarksApiId }, () => {
                  if (chrome.runtime.lastError) {
                    console.error(
                      "Error storing bookmarksApiId:",
                      chrome.runtime.lastError
                    );
                    return;
                  }
                  console.log(`Stored bookmarksApiId: ${bookmarksApiId}`);
                });
              }

              // Extract headers with error handling
              const getHeaderValue = (name: string): string => {
                const header = details.requestHeaders?.find(
                  (h) => h.name.toLowerCase() === name.toLowerCase()
                );
                return header?.value || "";
              };

              const auth = getHeaderValue("authorization");
              const cookie = getHeaderValue("cookie");
              const csrf = getHeaderValue("x-csrf-token");

              // Only update storage if values have changed
              if (
                result.cookie !== cookie ||
                result.csrf !== csrf ||
                result.auth !== auth
              ) {
                chrome.storage.local.set({ cookie, csrf, auth }, () => {
                  if (chrome.runtime.lastError) {
                    console.error(
                      "Error updating credentials:",
                      chrome.runtime.lastError
                    );
                    return;
                  }
                  console.log(
                    "Updated cookie, csrf, and auth in local storage"
                  );
                });
              }
            } catch (err) {
              console.error("Error processing request headers:", err);
            }
          }
        );
      } catch (err) {
        console.error("Top level error in onBeforeSendHeaders listener:", err);
      }
    },
    { urls: ["*://x.com/*", "*://twitter.com/*"] },
    extraOptions
  );
};