mirror of
https://github.com/supermemoryai/supermemory.git
synced 2026-09-11 22:51:05 +00:00
check for ok instead of fuckall
This commit is contained in:
Dhravya Shah
parent
4d9775e66d
commit
30d5fb4733
3 changed files with 151 additions and 129 deletions
|
|
@ -14,14 +14,13 @@ const tweetToMd = (tweet: Tweet) => {
|
|||
const BOOKMARKS_URL = `https://x.com/i/api/graphql/xLjCVTqYWz8CGSprLU349w/Bookmarks?features=${encodeURIComponent(JSON.stringify(features))}`;
|
||||
|
||||
const BACKEND_URL = "https://supermemory.ai";
|
||||
|
||||
// This is to prevent going over the rate limit
|
||||
const INITIAL_WAIT_TIME = 60000; // 1 minute in milliseconds
|
||||
let waitTime = INITIAL_WAIT_TIME;
|
||||
let lastTwitterFetch = 0;
|
||||
|
||||
const batchImportAll = async (cursor = "", totalImported = 0) => {
|
||||
chrome.storage.session.get(["cookie", "csrf", "auth"], (result) => {
|
||||
chrome.storage.session.get(["cookie", "csrf", "auth"], async (result) => {
|
||||
if (!result.cookie || !result.csrf || !result.auth) {
|
||||
console.log("cookie, csrf, or auth is missing");
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -46,126 +45,155 @@ const batchImportAll = async (cursor = "", totalImported = 0) => {
|
|||
? `${BOOKMARKS_URL}&variables=${encodeURIComponent(JSON.stringify(variables))}`
|
||||
: BOOKMARKS_URL;
|
||||
|
||||
fetch(urlWithCursor, requestOptions)
|
||||
.then((response) => response.json())
|
||||
.then((data) => {
|
||||
const tweets = getAllTweets(data);
|
||||
let importedCount = 0;
|
||||
let batchImportedCount = 0;
|
||||
|
||||
for (const tweet of tweets) {
|
||||
console.log(tweet);
|
||||
|
||||
const tweetMd = tweetToMd(tweet);
|
||||
(async () => {
|
||||
chrome.storage.local.get(["jwt"], ({ jwt }) => {
|
||||
if (!jwt) {
|
||||
console.error("No JWT found");
|
||||
return;
|
||||
}
|
||||
fetch(`${BACKEND_URL}/api/store`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${jwt}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
pageContent: tweetMd,
|
||||
url: `https://twitter.com/supermemoryai/status/${tweet.id_str}`,
|
||||
title: `Tweet by ${tweet.user.name}`,
|
||||
description: tweet.text.slice(0, 200),
|
||||
type: "tweet",
|
||||
}),
|
||||
}).then(async (ers) => {
|
||||
console.log(ers.status);
|
||||
importedCount++;
|
||||
totalImported++;
|
||||
console.log(totalImported);
|
||||
chrome.tabs.query(
|
||||
{ active: true, currentWindow: true },
|
||||
async function (tabs) {
|
||||
if (tabs.length > 0) {
|
||||
let currentTabId = tabs[0].id;
|
||||
|
||||
if (!currentTabId) {
|
||||
return;
|
||||
}
|
||||
|
||||
await chrome.tabs.sendMessage(currentTabId, {
|
||||
type: "import-update",
|
||||
importedCount: totalImported,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
})();
|
||||
try {
|
||||
const response = await fetch(urlWithCursor, requestOptions);
|
||||
if (!response.ok) {
|
||||
if (response.status === 429) {
|
||||
console.error("Rate limit exceeded");
|
||||
await handleRateLimit();
|
||||
return batchImportAll(cursor, totalImported); // Retry after waiting
|
||||
} else {
|
||||
console.error("Error fetching bookmarks", response.status);
|
||||
throw new Error("Failed to fetch data");
|
||||
}
|
||||
}
|
||||
|
||||
console.log("tweets", tweets);
|
||||
console.log("data", data);
|
||||
const data = await response.json();
|
||||
const tweets = getAllTweets(data);
|
||||
|
||||
const instructions =
|
||||
data.data?.bookmark_timeline_v2?.timeline?.instructions;
|
||||
const lastInstruction = instructions?.[0].entries.pop();
|
||||
for (const tweet of tweets) {
|
||||
console.log(tweet);
|
||||
|
||||
if (lastInstruction?.entryId.startsWith("cursor-bottom-")) {
|
||||
let nextCursor = lastInstruction?.content?.value;
|
||||
const tweetMd = tweetToMd(tweet);
|
||||
try {
|
||||
await importTweet(tweetMd, tweet);
|
||||
batchImportedCount++;
|
||||
totalImported++;
|
||||
console.log("Total imported:", totalImported);
|
||||
await updateImportProgress(totalImported);
|
||||
} catch (error) {
|
||||
console.error("Error importing tweet:", error);
|
||||
await handleSupermemoryError();
|
||||
return; // Stop the process
|
||||
}
|
||||
}
|
||||
|
||||
if (!nextCursor) {
|
||||
for (let i = instructions.length - 1; i >= 0; i--) {
|
||||
if (instructions[i].entryId.startsWith("cursor-bottom-")) {
|
||||
nextCursor = instructions[i].content.value;
|
||||
break;
|
||||
}
|
||||
console.log("tweets", tweets);
|
||||
console.log("data", data);
|
||||
|
||||
const instructions =
|
||||
data.data?.bookmark_timeline_v2?.timeline?.instructions;
|
||||
const lastInstruction = instructions?.[0].entries.pop();
|
||||
|
||||
if (lastInstruction?.entryId.startsWith("cursor-bottom-")) {
|
||||
let nextCursor = lastInstruction?.content?.value;
|
||||
|
||||
if (!nextCursor) {
|
||||
for (let i = instructions.length - 1; i >= 0; i--) {
|
||||
if (instructions[i].entryId.startsWith("cursor-bottom-")) {
|
||||
nextCursor = instructions[i].content.value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (nextCursor) {
|
||||
batchImportAll(nextCursor, totalImported); // Recursively call with new cursor
|
||||
} else {
|
||||
console.log("All bookmarks imported");
|
||||
|
||||
chrome.tabs.query(
|
||||
{ active: true, currentWindow: true },
|
||||
async function (tabs) {
|
||||
if (tabs.length > 0) {
|
||||
let currentTabId = tabs[0].id;
|
||||
|
||||
if (!currentTabId) {
|
||||
return;
|
||||
}
|
||||
|
||||
await chrome.runtime.sendMessage({
|
||||
type: "import-done",
|
||||
importedCount: totalImported,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.log("All bookmarks imported");
|
||||
// Send a "done" message to the content script
|
||||
chrome.tabs.query(
|
||||
{ active: true, currentWindow: true },
|
||||
async function (tabs) {
|
||||
if (tabs.length > 0) {
|
||||
let currentTabId = tabs[0].id;
|
||||
|
||||
if (!currentTabId) {
|
||||
return;
|
||||
}
|
||||
|
||||
await chrome.runtime.sendMessage({
|
||||
type: "import-done",
|
||||
importedCount: totalImported,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (nextCursor) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000)); // 1 second delay between batches
|
||||
batchImportAll(nextCursor, totalImported); // Recursively call with new cursor
|
||||
} else {
|
||||
await sendImportDoneMessage(totalImported);
|
||||
}
|
||||
} else {
|
||||
await sendImportDoneMessage(totalImported);
|
||||
}
|
||||
|
||||
// Reset wait time after successful import
|
||||
waitTime = INITIAL_WAIT_TIME;
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
await handleRateLimit();
|
||||
batchImportAll(cursor, totalImported); // Retry after waiting
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const handleRateLimit = async () => {
|
||||
const waitTimeInSeconds = waitTime / 1000;
|
||||
console.log(`Waiting for ${waitTimeInSeconds} seconds due to rate limit...`);
|
||||
await sendMessageToCurrentTab(
|
||||
`Rate limit reached. Waiting for ${waitTimeInSeconds} seconds before retrying...`,
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
||||
waitTime *= 2; // Double the wait time for next potential rate limit
|
||||
};
|
||||
|
||||
const handleSupermemoryError = async () => {
|
||||
const errorMessage =
|
||||
"ALERT: Supermemory is unable to save tweets right now. Please contact support at feedback@supermemory.ai";
|
||||
console.error(errorMessage);
|
||||
await sendMessageToCurrentTab(errorMessage);
|
||||
};
|
||||
|
||||
const sendMessageToCurrentTab = async (message: string) => {
|
||||
const tabs = await chrome.tabs.query({ active: true, currentWindow: true });
|
||||
if (tabs.length > 0 && tabs[0].id) {
|
||||
await chrome.tabs.sendMessage(tabs[0].id, {
|
||||
type: "import-update",
|
||||
importedMessage: message,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const updateImportProgress = async (totalImported: number) => {
|
||||
await sendMessageToCurrentTab(`Imported ${totalImported} tweets`);
|
||||
};
|
||||
|
||||
const importTweet = async (tweetMd: string, tweet: Tweet) => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
chrome.storage.local.get(["jwt"], ({ jwt }) => {
|
||||
if (!jwt) {
|
||||
console.error("No JWT found");
|
||||
reject(new Error("No JWT found"));
|
||||
return;
|
||||
}
|
||||
fetch(`${BACKEND_URL}/api/store`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${jwt}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
pageContent: tweetMd,
|
||||
url: `https://twitter.com/supermemoryai/status/${tweet.id_str}`,
|
||||
title: `Tweet by ${tweet.user.name}`,
|
||||
description: tweet.text.slice(0, 200),
|
||||
type: "tweet",
|
||||
}),
|
||||
})
|
||||
.catch((error) => console.error(error));
|
||||
.then(async (response) => {
|
||||
if (!response.ok) {
|
||||
console.log(
|
||||
"Supermemory error",
|
||||
response.status,
|
||||
await response.text(),
|
||||
);
|
||||
reject(new Error("Failed to save tweet to Supermemory"));
|
||||
} else {
|
||||
console.log("Tweet saved successfully");
|
||||
resolve();
|
||||
}
|
||||
})
|
||||
.catch(reject);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const sendImportDoneMessage = async (totalImported: number) => {
|
||||
console.log("All bookmarks imported");
|
||||
await chrome.runtime.sendMessage({
|
||||
type: "import-done",
|
||||
importedCount: totalImported,
|
||||
});
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ export default function ContentApp({
|
|||
|
||||
const [webNote, setWebNote] = useState<string>("");
|
||||
|
||||
const [importedCount, setImportedCount] = useState(0);
|
||||
const [importedMessage, setImportedMessage] = useState("Importing...");
|
||||
const [isImporting, setIsImporting] = useState(false);
|
||||
const [importDone, setImportDone] = useState(false);
|
||||
|
||||
|
|
@ -117,7 +117,7 @@ export default function ContentApp({
|
|||
chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
|
||||
if (request.type === "import-update") {
|
||||
setIsImporting(true);
|
||||
setImportedCount(request.importedCount);
|
||||
setImportedMessage(request.importedMessage);
|
||||
}
|
||||
|
||||
if (request.type === "import-done") {
|
||||
|
|
@ -232,10 +232,7 @@ export default function ContentApp({
|
|||
action: (
|
||||
<button
|
||||
onClick={() => {
|
||||
window.open(
|
||||
`https://supermemory.ai`,
|
||||
"_blank",
|
||||
);
|
||||
window.open(`https://supermemory.ai`, "_blank");
|
||||
}}
|
||||
>
|
||||
View
|
||||
|
|
@ -417,7 +414,7 @@ export default function ContentApp({
|
|||
|
||||
{isImporting && (
|
||||
<div className="flex items-center gap-2">
|
||||
<p>Imported {importedCount} bookmarks</p>
|
||||
<p>{importedMessage}</p>
|
||||
<svg
|
||||
className="animate-spin w-6 h-6"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
|
|
|
|||
|
|
@ -131,20 +131,17 @@ export const createMemoryFromAPI = async (input: {
|
|||
}
|
||||
|
||||
try {
|
||||
const response = await vectorSaveResponse.json();
|
||||
|
||||
const expectedResponse = z.object({ status: z.literal("ok") });
|
||||
|
||||
const parsedResponse = expectedResponse.safeParse(response);
|
||||
|
||||
if (!parsedResponse.success) {
|
||||
if (!vectorSaveResponse.ok) {
|
||||
const resp = await vectorSaveResponse.text();
|
||||
return {
|
||||
success: false,
|
||||
data: 0,
|
||||
error: `Failed to save to vector store. Backend returned error: ${parsedResponse.error.message}`,
|
||||
error: `Failed to save to vector store. Backend returned error: ${resp}`,
|
||||
};
|
||||
}
|
||||
|
||||
await vectorSaveResponse.json();
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: 1,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue