added logic for importing all tweets

apps/cf-ai-backend/src/env.d.ts

@@ -4,7 +4,7 @@ interface Env {
 	SECURITY_KEY: string;
 	OPENAI_API_KEY: string;
 	GOOGLE_AI_API_KEY: string;
-	MY_QUEUE: Queue<TweetData>;
+	MY_QUEUE: Queue<TweetData[]>;
 	KV: KVNamespace;
 }
 
@@ -14,4 +14,5 @@ interface TweetData {
 	authorName: string;
 	handle: string;
 	time: string;
+	saveToUser: string;
 }

apps/cf-ai-backend/src/index.ts

@@ -4,6 +4,7 @@ import { CloudflareVectorizeStore } from '@langchain/cloudflare';
 import { OpenAIEmbeddings } from './OpenAIEmbedder';
 import { GoogleGenerativeAI } from '@google/generative-ai';
 import routeMap from './routes';
+import { queue } from './routes/queue';
 
 function isAuthorized(request: Request, env: Env): boolean {
 	return request.headers.get('X-Custom-Auth-Key') === env.SECURITY_KEY;
@@ -45,4 +46,5 @@ export default {
 		}
 		return await handler(request, store, embeddings, model, env, ctx);
 	},
+	queue,
 };

apps/cf-ai-backend/src/routes.ts

@@ -3,6 +3,7 @@ import * as apiAdd from './routes/add';
 import * as apiQuery from './routes/query';
 import * as apiAsk from './routes/ask';
 import * as apiChat from './routes/chat';
+import * as apiBatchUploadTweets from './routes/batchUploadTweets';
 import { OpenAIEmbeddings } from './OpenAIEmbedder';
 import { GenerativeModel } from '@google/generative-ai';
 import { Request } from '@cloudflare/workers-types';
@@ -26,6 +27,8 @@ routeMap.set('/ask', apiAsk);
 
 routeMap.set('/chat', apiChat);
 
+routeMap.set('/batchUploadTweets', apiBatchUploadTweets);
+
 // Add more route mappings as needed
 // routeMap.set('/api/otherRoute', { ... });
 

apps/cf-ai-backend/src/routes/add.ts

@@ -2,6 +2,7 @@ import { Request } from '@cloudflare/workers-types';
 import { type CloudflareVectorizeStore } from '@langchain/cloudflare';
 import { OpenAIEmbeddings } from '../OpenAIEmbedder';
 import { GenerativeModel } from '@google/generative-ai';
+import { seededRandom } from '../util';
 
 export async function POST(request: Request, store: CloudflareVectorizeStore, _: OpenAIEmbeddings, m: GenerativeModel, env: Env) {
 	const body = (await request.json()) as {
@@ -20,15 +21,6 @@ export async function POST(request: Request, store: CloudflareVectorizeStore, _:
 
 	const ourID = `${body.url}-${body.user}`;
 
-	// WHY? Because this helps us to prevent duplicate entries for the same URL and user
-	function seededRandom(seed: string) {
-		let x = [...seed].reduce((acc, cur) => acc + cur.charCodeAt(0), 0);
-		return () => {
-			x = (x * 9301 + 49297) % 233280;
-			return x / 233280;
-		};
-	}
-
 	const random = seededRandom(ourID);
 	const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15);
 

apps/cf-ai-backend/src/routes/batchUploadTweets.ts

@@ -0,0 +1,38 @@
+import { Request } from '@cloudflare/workers-types';
+import { type CloudflareVectorizeStore } from '@langchain/cloudflare';
+import { OpenAIEmbeddings } from '../OpenAIEmbedder';
+import { GenerativeModel } from '@google/generative-ai';
+
+export async function POST(request: Request, store: CloudflareVectorizeStore, _: OpenAIEmbeddings, m: GenerativeModel, env: Env) {
+	const body = (await request.json()) as TweetData[] | undefined;
+
+	if (!body) {
+		return new Response(JSON.stringify({ message: 'Body is missing' }), { status: 400 });
+	}
+
+	const bytes = new TextEncoder().encode(JSON.stringify(body)).length;
+
+	if (bytes < 128000) {
+		await env.MY_QUEUE.send(body);
+	} else {
+		let bytesTillNow = 0;
+		let batches: TweetData[] = [];
+
+		const getByteLength = (data: string) => new TextEncoder().encode(data).length;
+
+		for (let i = 0; i < body.length; i++) {
+			const byteLength = getByteLength(JSON.stringify(body[i]));
+
+			if (bytesTillNow + byteLength < 100000) {
+				bytesTillNow += byteLength;
+				batches.push(body[i]);
+			} else {
+				await env.MY_QUEUE.send(batches);
+				batches = [body[i]];
+				bytesTillNow = byteLength;
+			}
+		}
+	}
+
+	return new Response(JSON.stringify({ message: 'Document Added' }), { status: 200 });
+}

apps/cf-ai-backend/src/routes/queue.ts

@@ -0,0 +1,89 @@
+import { CloudflareVectorizeStore } from '@langchain/cloudflare';
+import { OpenAIEmbeddings } from '../OpenAIEmbedder';
+import { seededRandom } from '../util';
+
+export const queue = async (batch: MessageBatch, env: Env): Promise<void> => {
+	const messages = batch.messages[0].body as TweetData[];
+
+	const token = messages[0].saveToUser;
+
+	if (!token) {
+		return;
+	}
+
+	const limits = (await fetch('https://supermemory.dhr.wtf/api/getCount', {
+		headers: {
+			Authorization: `Bearer ${token}`,
+		},
+	}).then((res) => res.json())) as { limit: number; tweetsCount: number; user: string };
+
+	if (messages.length > limits.limit - limits.tweetsCount) {
+		messages.splice(limits.limit - limits.tweetsCount);
+	}
+
+	if (messages.length === 0) {
+		return;
+	}
+
+	const embeddings = new OpenAIEmbeddings({
+		apiKey: env.OPENAI_API_KEY,
+		modelName: 'text-embedding-3-small',
+	});
+
+	const store = new CloudflareVectorizeStore(embeddings, {
+		index: env.VECTORIZE_INDEX,
+	});
+
+	const collectedDocsUUIDs: {
+		document: {
+			pageContent: string;
+			metadata: { title: string; description: string; space: string; url: string; user: string };
+			id: string;
+		};
+	}[] = [];
+
+	messages.forEach(async (message) => {
+		const ourID = `${message.postUrl}-${limits.user}`;
+
+		const random = seededRandom(ourID);
+		const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15);
+
+		await env.KV.put(uuid, ourID);
+		const pageContent = `This is a tweet from ${message.authorName}, it was posted on ${message.time}. The tweet reads: ${message.tweetText}`;
+
+		collectedDocsUUIDs.push({
+			document: {
+				pageContent,
+				metadata: {
+					title: 'Twitter Bookmark',
+					description: '',
+					space: 'Bookmarked Tweets',
+					url: message.postUrl,
+					user: limits.user,
+				},
+				id: uuid,
+			},
+		});
+	});
+
+	await store.addDocuments(
+		collectedDocsUUIDs.map(({ document }) => document),
+		{
+			ids: collectedDocsUUIDs.map(({ document }) => document.id),
+		},
+	);
+
+	const res = await fetch('https://supermemory.dhr.wtf/api/addTweetsToDb', {
+		method: 'POST',
+		headers: {
+			Authorization: `Bearer ${token}`,
+		},
+		body: JSON.stringify(messages),
+	});
+
+	if (res.status !== 200) {
+		console.error('Error adding tweets to db');
+	}
+
+	console.log(`consumed from our queue: ${messages}`);
+};

apps/cf-ai-backend/src/util.ts

@@ -0,0 +1,7 @@
+export function seededRandom(seed: string) {
+	let x = [...seed].reduce((acc, cur) => acc + cur.charCodeAt(0), 0);
+	return () => {
+		x = (x * 9301 + 49297) % 233280;
+		return x / 233280;
+	};
+}

apps/cf-ai-backend/wrangler.toml

@@ -13,6 +13,9 @@ binding = "AI"
  queue = "batch-vector-queue"
  binding = "MY_QUEUE"
 
+ [[queues.consumers]]
+ queue = "batch-vector-queue"
+
 [[kv_namespaces]]
 binding = "KV"
 id = "37a90353da63401e84e20e71165531d0"

apps/extension/src/SideBar.tsx

@@ -35,7 +35,7 @@ function sendUrlToAPI() {
   }
 }
 
-function SideBar() {
+function SideBar({ jwt }: { jwt: string }) {
   // TODO: Implement getting bookmarks from Twitter API directly
   // chrome.runtime.onMessage.addListener(function (request) {
   //   if (request.action === 'showProgressIndicator') {
@@ -62,6 +62,7 @@ function SideBar() {
     authorName: string;
     handle: string;
     time: string;
+    saveToUser: string;
   }
 
   const fetchBookmarks = () => {
@@ -136,6 +137,7 @@ function SideBar() {
               tweetText,
               time: time ?? "",
               postUrl,
+              saveToUser: jwt,
             });
 
             setLog([...log, `Scraped tweet: ${tweets.length}`]);
@@ -162,7 +164,7 @@ function SideBar() {
       setIsImportingTweets(false);
       const jsonData = JSON.stringify(tweetsArray); // Convert the array to JSON
 
-      // TODO: SEND jsonData to server
+      // TODO: send jsonData to the API
       console.log(jsonData);
     }
   };

apps/extension/src/content.tsx

@@ -38,8 +38,8 @@ const jwt = chrome.storage.local.get("jwt").then((data) => {
   return data.jwt;
 }) as Promise<string>;
 
-jwt.then(() => {
+jwt.then((token) => {
   ReactDOM.createRoot(
     document.getElementById("anycontext-app-container")!,
-  ).render(<SideBar />);
+  ).render(<SideBar jwt={token} />);
 });

apps/web/public/twitter.svg

@@ -0,0 +1 @@
+<svg viewBox="0 0 256 209" width="256" height="209" xmlns="http://www.w3.org/2000/svg" preserveAspectRatio="xMidYMid"><path d="M256 25.45c-9.42 4.177-19.542 7-30.166 8.27 10.845-6.5 19.172-16.793 23.093-29.057a105.183 105.183 0 0 1-33.351 12.745C205.995 7.201 192.346.822 177.239.822c-29.006 0-52.523 23.516-52.523 52.52 0 4.117.465 8.125 1.36 11.97-43.65-2.191-82.35-23.1-108.255-54.876-4.52 7.757-7.11 16.78-7.11 26.404 0 18.222 9.273 34.297 23.365 43.716a52.312 52.312 0 0 1-23.79-6.57c-.003.22-.003.44-.003.661 0 25.447 18.104 46.675 42.13 51.5a52.592 52.592 0 0 1-23.718.9c6.683 20.866 26.08 36.05 49.062 36.475-17.975 14.086-40.622 22.483-65.228 22.483-4.24 0-8.42-.249-12.529-.734 23.243 14.902 50.85 23.597 80.51 23.597 96.607 0 149.434-80.031 149.434-149.435 0-2.278-.05-4.543-.152-6.795A106.748 106.748 0 0 0 256 25.45" fill="#55acee"/></svg>

apps/web/public/web.svg

@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6">
+  <path stroke-linecap="round" stroke-linejoin="round" d="M12 21a9.004 9.004 0 0 0 8.716-6.747M12 21a9.004 9.004 0 0 1-8.716-6.747M12 21c2.485 0 4.5-4.03 4.5-9S14.485 3 12 3m0 18c-2.485 0-4.5-4.03-4.5-9S9.515 3 12 3m0 0a8.997 8.997 0 0 1 7.843 4.582M12 3a8.997 8.997 0 0 0-7.843 4.582m15.686 0A11.953 11.953 0 0 1 12 10.5c-2.998 0-5.74-1.1-7.843-2.918m15.686 0A8.959 8.959 0 0 1 21 12c0 .778-.099 1.533-.284 2.253m0 0A17.919 17.919 0 0 1 12 16.5c-3.162 0-6.133-.815-8.716-2.247m0 0A9.015 9.015 0 0 1 3 12c0-1.605.42-3.113 1.157-4.418" />
+</svg>  

apps/web/src/app/api/addTweetsToDb/route.ts

@@ -0,0 +1,91 @@
+import { db } from "@/server/db";
+import { eq } from "drizzle-orm";
+import { sessions, storedContent, users } from "@/server/db/schema";
+import { type NextRequest, NextResponse } from "next/server";
+
+export const runtime = "edge";
+
+interface TweetData {
+  tweetText: string;
+  postUrl: string;
+  authorName: string;
+  handle: string;
+  time: string;
+  saveToUser: string;
+}
+
+export async function POST(req: NextRequest) {
+  const token =
+    req.cookies.get("next-auth.session-token")?.value ??
+    req.cookies.get("__Secure-authjs.session-token")?.value ??
+    req.cookies.get("authjs.session-token")?.value ??
+    req.headers.get("Authorization")?.replace("Bearer ", "");
+
+  if (!token) {
+    return new Response(
+      JSON.stringify({ message: "Invalid Key, session not found." }),
+      { status: 404 },
+    );
+  }
+
+  const sessionData = await db
+    .select()
+    .from(sessions)
+    .where(eq(sessions.sessionToken, token!));
+
+  if (!sessionData || sessionData.length === 0) {
+    return new Response(
+      JSON.stringify({ message: "Invalid Key, session not found." }),
+      { status: 404 },
+    );
+  }
+
+  const user = await db
+    .select()
+    .from(users)
+    .where(eq(users.id, sessionData[0].userId))
+    .limit(1);
+
+  if (!user || user.length === 0) {
+    return NextResponse.json(
+      { message: "Invalid Key, session not found." },
+      { status: 404 },
+    );
+  }
+
+  const session = { session: sessionData[0], user: user[0] };
+
+  const data = (await req.json()) as TweetData[];
+
+  for (const tweet of data) {
+    const { id } = (
+      await db
+        .insert(storedContent)
+        .values({
+          content: tweet.tweetText,
+          title: "Twitter Bookmark",
+          description: "",
+          url: tweet.postUrl,
+          baseUrl: "https://twitter.com",
+          image: "https://supermemory.dhr.wtf/twitter.svg",
+          savedAt: new Date(),
+          user: session.user.id,
+          type: "twitter-bookmark",
+        })
+        .returning({ id: storedContent.id })
+    )[0];
+
+    if (!id) {
+      return NextResponse.json(
+        {
+          message: "Error",
+          error:
+            "Something went wrong when inserting the tweet to storedContent",
+        },
+        { status: 500 },
+      );
+    }
+  }
+
+  return NextResponse.json({ message: "OK", data: "Success" }, { status: 200 });
+}

apps/web/src/app/api/getCount/route.ts

@@ -0,0 +1,66 @@
+import { db } from "@/server/db";
+import { and, eq, sql } from "drizzle-orm";
+import { sessions, storedContent, users } from "@/server/db/schema";
+import { type NextRequest, NextResponse } from "next/server";
+
+export const runtime = "edge";
+
+export async function GET(req: NextRequest) {
+  const token =
+    req.cookies.get("next-auth.session-token")?.value ??
+    req.cookies.get("__Secure-authjs.session-token")?.value ??
+    req.cookies.get("authjs.session-token")?.value ??
+    req.headers.get("Authorization")?.replace("Bearer ", "");
+
+  if (!token) {
+    return new Response(
+      JSON.stringify({ message: "Invalid Key, session not found." }),
+      { status: 404 },
+    );
+  }
+
+  const sessionData = await db
+    .select()
+    .from(sessions)
+    .where(eq(sessions.sessionToken, token!));
+
+  if (!sessionData || sessionData.length === 0) {
+    return new Response(
+      JSON.stringify({ message: "Invalid Key, session not found." }),
+      { status: 404 },
+    );
+  }
+
+  const user = await db
+    .select()
+    .from(users)
+    .where(eq(users.id, sessionData[0].userId))
+    .limit(1);
+
+  if (!user || user.length === 0) {
+    return NextResponse.json(
+      { message: "Invalid Key, session not found." },
+      { status: 404 },
+    );
+  }
+
+  const session = { session: sessionData[0], user: user[0] };
+
+  const count = await db
+    .select({
+      count: sql<number>`count(*)`.mapWith(Number),
+    })
+    .from(storedContent)
+    .where(
+      and(
+        eq(storedContent.user, session.user.id),
+        eq(storedContent.type, "twitter-bookmark"),
+      ),
+    );
+
+  return NextResponse.json({
+    tweetsCount: count[0].count,
+    limit: 1000,
+    user: session.user.email,
+  });
+}

apps/web/src/server/helpers.ts

@@ -17,17 +17,19 @@ export async function getMetaData(url: string) {
     ? descriptionMatch[1]
     : "Description not found";
 
-  // Extract Open Graph image
-  const imageMatch = html.match(
-    /<meta property="og:image" content="(.*?)"\s*\/?>/,
+  // Extract favicon
+  const faviconMatch = html.match(
+    /<link rel="(?:icon|shortcut icon)" href="(.*?)"\s*\/?>/,
   );
-  const image = imageMatch ? imageMatch[1] : "Image not found";
+  const favicon = faviconMatch
+    ? faviconMatch[1]
+    : "https://supermemory.dhr.wtf/web.svg";
 
   // Prepare the metadata object
   const metadata = {
     title,
     description,
-    image,
+    image: favicon,
     baseUrl,
   };
   return metadata;