merge

apps/cf-ai-backend/src/env.d.ts

@@ -4,7 +4,7 @@ interface Env {
 	SECURITY_KEY: string;
 	OPENAI_API_KEY: string;
 	GOOGLE_AI_API_KEY: string;
-	MY_QUEUE: Queue<TweetData>;
+	MY_QUEUE: Queue<TweetData[]>;
 	KV: KVNamespace;
 }
 
@@ -14,4 +14,5 @@ interface TweetData {
 	authorName: string;
 	handle: string;
 	time: string;
+	saveToUser: string;
 }

apps/cf-ai-backend/src/index.ts

@@ -4,6 +4,7 @@ import { CloudflareVectorizeStore } from '@langchain/cloudflare';
 import { OpenAIEmbeddings } from './OpenAIEmbedder';
 import { GoogleGenerativeAI } from '@google/generative-ai';
 import routeMap from './routes';
+import { queue } from './routes/queue';
 
 function isAuthorized(request: Request, env: Env): boolean {
 	return request.headers.get('X-Custom-Auth-Key') === env.SECURITY_KEY;
@@ -45,4 +46,5 @@ export default {
 		}
 		return await handler(request, store, embeddings, model, env, ctx);
 	},
+	queue,
 };

apps/cf-ai-backend/src/routes.ts

@@ -3,6 +3,7 @@ import * as apiAdd from './routes/add';
 import * as apiQuery from './routes/query';
 import * as apiAsk from './routes/ask';
 import * as apiChat from './routes/chat';
+import * as apiBatchUploadTweets from './routes/batchUploadTweets';
 import { OpenAIEmbeddings } from './OpenAIEmbedder';
 import { GenerativeModel } from '@google/generative-ai';
 import { Request } from '@cloudflare/workers-types';
@@ -26,6 +27,8 @@ routeMap.set('/ask', apiAsk);
 
 routeMap.set('/chat', apiChat);
 
+routeMap.set('/batchUploadTweets', apiBatchUploadTweets);
+
 // Add more route mappings as needed
 // routeMap.set('/api/otherRoute', { ... });
 

apps/cf-ai-backend/src/routes/add.ts

@@ -2,6 +2,7 @@ import { Request } from '@cloudflare/workers-types';
 import { type CloudflareVectorizeStore } from '@langchain/cloudflare';
 import { OpenAIEmbeddings } from '../OpenAIEmbedder';
 import { GenerativeModel } from '@google/generative-ai';
+import { seededRandom } from '../util';
 
 export async function POST(request: Request, store: CloudflareVectorizeStore, _: OpenAIEmbeddings, m: GenerativeModel, env: Env) {
 	const body = (await request.json()) as {
@@ -20,15 +21,6 @@ export async function POST(request: Request, store: CloudflareVectorizeStore, _:
 
 	const ourID = `${body.url}-${body.user}`;
 
-	// WHY? Because this helps us to prevent duplicate entries for the same URL and user
-	function seededRandom(seed: string) {
-		let x = [...seed].reduce((acc, cur) => acc + cur.charCodeAt(0), 0);
-		return () => {
-			x = (x * 9301 + 49297) % 233280;
-			return x / 233280;
-		};
-	}
-
 	const random = seededRandom(ourID);
 	const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15);
 

apps/cf-ai-backend/src/routes/batchUploadTweets.ts

@@ -0,0 +1,38 @@
+import { Request } from '@cloudflare/workers-types';
+import { type CloudflareVectorizeStore } from '@langchain/cloudflare';
+import { OpenAIEmbeddings } from '../OpenAIEmbedder';
+import { GenerativeModel } from '@google/generative-ai';
+
+export async function POST(request: Request, store: CloudflareVectorizeStore, _: OpenAIEmbeddings, m: GenerativeModel, env: Env) {
+	const body = (await request.json()) as TweetData[] | undefined;
+
+	if (!body) {
+		return new Response(JSON.stringify({ message: 'Body is missing' }), { status: 400 });
+	}
+
+	const bytes = new TextEncoder().encode(JSON.stringify(body)).length;
+
+	if (bytes < 128000) {
+		await env.MY_QUEUE.send(body);
+	} else {
+		let bytesTillNow = 0;
+		let batches: TweetData[] = [];
+
+		const getByteLength = (data: string) => new TextEncoder().encode(data).length;
+
+		for (let i = 0; i < body.length; i++) {
+			const byteLength = getByteLength(JSON.stringify(body[i]));
+
+			if (bytesTillNow + byteLength < 100000) {
+				bytesTillNow += byteLength;
+				batches.push(body[i]);
+			} else {
+				await env.MY_QUEUE.send(batches);
+				batches = [body[i]];
+				bytesTillNow = byteLength;
+			}
+		}
+	}
+
+	return new Response(JSON.stringify({ message: 'Document Added' }), { status: 200 });
+}

apps/cf-ai-backend/src/routes/queue.ts

@@ -0,0 +1,96 @@
+import { CloudflareVectorizeStore } from '@langchain/cloudflare';
+import { OpenAIEmbeddings } from '../OpenAIEmbedder';
+import { seededRandom } from '../util';
+
+export const queue = async (batch: MessageBatch, env: Env): Promise<void> => {
+	const messages = batch.messages[0].body as TweetData[];
+
+	const token = messages[0].saveToUser;
+
+	if (!token) {
+		return;
+	}
+
+	const limits = (await fetch('https://supermemory.dhr.wtf/api/getCount', {
+		headers: {
+			Authorization: `Bearer ${token}`,
+		},
+	}).then((res) => res.json())) as { limit: number; tweetsCount: number; user: string };
+
+	if (messages.length > limits.limit - limits.tweetsCount) {
+		messages.splice(limits.limit - limits.tweetsCount);
+	}
+
+	if (messages.length === 0) {
+		return;
+	}
+
+	const embeddings = new OpenAIEmbeddings({
+		apiKey: env.OPENAI_API_KEY,
+		modelName: 'text-embedding-3-small',
+	});
+
+	const store = new CloudflareVectorizeStore(embeddings, {
+		index: env.VECTORIZE_INDEX,
+	});
+
+	const collectedDocsUUIDs: {
+		document: {
+			pageContent: string;
+			metadata: { title: string; description: string; space: string; url: string; user: string };
+			id: string;
+		};
+	}[] = [];
+
+	messages.forEach(async (message) => {
+		const ourID = `${message.postUrl}-${limits.user}`;
+
+		const random = seededRandom(ourID);
+		const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15);
+
+		await env.KV.put(uuid, ourID);
+		const pageContent = `This is a tweet from ${message.authorName}, it was posted on ${message.time}. The tweet reads: ${message.tweetText}`;
+
+		collectedDocsUUIDs.push({
+			document: {
+				pageContent,
+				metadata: {
+					title: 'Twitter Bookmark',
+					description: '',
+					space: 'Bookmarked Tweets',
+					url: message.postUrl,
+					user: limits.user,
+				},
+				id: uuid,
+			},
+		});
+	});
+
+	console.log(collectedDocsUUIDs);
+
+	await store.addDocuments(
+		collectedDocsUUIDs.map(({ document }) => document),
+		{
+			ids: collectedDocsUUIDs.map(({ document }) => document.id),
+		},
+	);
+
+	console.log(token);
+
+	const res = await fetch('https://supermemory.dhr.wtf/api/addTweetsToDb', {
+		method: 'POST',
+		headers: {
+			Authorization: `Bearer ${token}`,
+		},
+		body: JSON.stringify(messages),
+	});
+
+	console.log(res.status, res.statusText);
+
+	if (res.status !== 200) {
+		console.log(await res.json());
+		console.error('Error adding tweets to db');
+	}
+
+	console.log(`consumed from our queue: ${messages}`);
+};

apps/cf-ai-backend/src/util.ts

@@ -0,0 +1,7 @@
+export function seededRandom(seed: string) {
+	let x = [...seed].reduce((acc, cur) => acc + cur.charCodeAt(0), 0);
+	return () => {
+		x = (x * 9301 + 49297) % 233280;
+		return x / 233280;
+	};
+}

apps/cf-ai-backend/wrangler.toml

@@ -13,6 +13,9 @@ binding = "AI"
  queue = "batch-vector-queue"
  binding = "MY_QUEUE"
 
+ [[queues.consumers]]
+ queue = "batch-vector-queue"
+
 [[kv_namespaces]]
 binding = "KV"
 id = "37a90353da63401e84e20e71165531d0"

apps/extension/src/App.tsx

@@ -44,7 +44,7 @@ function App() {
     getUserData();
   }, []);
 
-  // TODO: Implement getting bookmarks from API directly
+  // TODO: Implement getting bookmarks from Twitter API directly
   // const [status, setStatus] = useState('');
   // const [bookmarks, setBookmarks] = useState<TweetData[]>([]);
 
@@ -149,7 +149,7 @@ function App() {
   );
 }
 
-// TODO: Implement getting bookmarks from API directly
+// TODO: Implement getting bookmarks from Twitter API directly
 // async function fetchAllBookmarks(
 //   authorizationHeader: string,
 //   csrfToken: string,

apps/extension/src/SideBar.tsx

@@ -32,12 +32,12 @@ function sendUrlToAPI(spaces: number[]) {
   } else {
     // const content = Entire page content, but cleaned up for the LLM. No ads, no scripts, no styles, just the text. if article, just the importnat info abou tit.
     const content = document.documentElement.innerText;
-    chrome.runtime.sendMessage({ type: "urlChange", content, url });
+    chrome.runtime.sendMessage({ type: "urlChange", content, url, spaces });
   }
 }
 
-function SideBar() {
-  // TODO: Implement getting bookmarks from API directly
+function SideBar({ jwt }: { jwt: string }) {
+  // TODO: Implement getting bookmarks from Twitter API directly
   // chrome.runtime.onMessage.addListener(function (request) {
   //   if (request.action === 'showProgressIndicator') {
   //     // TODO: SHOW PROGRESS INDICATOR
@@ -56,12 +56,25 @@ function SideBar() {
 	const [spaces, setSpaces] = useState<Space[]>();
   const [selectedSpaces, setSelectedSpaces] = useState<number[]>([]);
 
+  const [isImportingTweets, setIsImportingTweets] = useState(false);
+
+  const [log, setLog] = useState<string[]>([]);
+
   interface TweetData {
     tweetText: string;
     postUrl: string;
     authorName: string;
     handle: string;
     time: string;
+    saveToUser: string;
+  }
+
+  function sendBookmarkedTweetsToAPI(tweets: TweetData[], token: string) {
+    chrome.runtime.sendMessage({
+      type: "sendBookmarkedTweets",
+      jwt: token,
+      tweets,
+    });
   }
 
 	const fetchSpaces = async () => {
@@ -76,6 +89,9 @@ function SideBar() {
   const fetchBookmarks = () => {
     const tweets: TweetData[] = []; // Initialize an empty array to hold all tweet elements
 
+    setIsImportingTweets(true);
+    console.log("Importing tweets");
+
     const scrollInterval = 1000;
     const scrollStep = 5000; // Pixels to scroll on each step
 
@@ -88,10 +104,12 @@ function SideBar() {
       if (currentTweetCount === previousTweetCount) {
         unchangedCount++;
         if (unchangedCount >= 2) {
-          // Stop if the count has not changed 5 times
-          console.log("Scraping complete");
-          console.log("Total tweets scraped: ", tweets.length);
-          console.log("Downloading tweets as JSON...");
+          setLog([
+            ...log,
+            "Scraping complete",
+            `Total tweets scraped: ${tweets.length}`,
+            "Downloading tweets as JSON...",
+          ]);
           clearInterval(scrollToEndIntervalID); // Stop scrolling
           observer.disconnect(); // Stop observing DOM changes
           downloadTweetsAsJson(tweets); // Download the tweets list as a JSON file
@@ -140,8 +158,10 @@ function SideBar() {
               tweetText,
               time: time ?? "",
               postUrl,
+              saveToUser: jwt,
             });
-            console.log("Tweets capturados: ", tweets.length);
+
+            setLog([...log, `Scraped tweet: ${tweets.length}`]);
           }
         });
     }
@@ -162,15 +182,43 @@ function SideBar() {
     observer.observe(document.body, { childList: true, subtree: true });
 
     function downloadTweetsAsJson(tweetsArray: TweetData[]) {
-      const jsonData = JSON.stringify(tweetsArray); // Convert the array to JSON
-
-      // TODO: SEND jsonData to server
-      console.log(jsonData);
+      setLog([...log, "Saving the tweets to our database..."]);
+      sendBookmarkedTweetsToAPI(tweetsArray, jwt);
+      setIsImportingTweets(false);
     }
   };
 
   return (
     <>
+      {isImportingTweets && (
+        <div className="anycontext-overlay anycontext-fixed anycontext-font-sans anycontext-inset-0 anycontext-bg-black anycontext-bg-opacity-50">
+          <div className="anycontext-flex anycontext-items-center anycontext-justify-center anycontext-h-screen">
+            <div className="anycontext-flex anycontext-flex-col anycontext-items-center">
+              <svg
+                xmlns="http://www.w3.org/2000/svg"
+                fill="none"
+                viewBox="0 0 24 24"
+                strokeWidth={2}
+                stroke="currentColor"
+                className="anycontext-w-10 anycontext-h-10 anycontext-animate-spin"
+              >
+                <path
+                  strokeLinecap="round"
+                  strokeLinejoin="round"
+                  d="M17.593 3.322c1.1.128 1.907 1.077 1.907 2.185V21L12 17.25 4.5 21V5.507c0-1.108.806-2.057 1.907-2.185a48.507 48.507 0 0 1 11.186 0Z"
+                />
+              </svg>
+              <p className="anycontext-mt-2">Importing your tweets...</p>
+              <div className="anycontext-mt-2">
+                {log.map((message, index) => (
+                  <p key={index}>{message}</p>
+                ))}
+              </div>
+            </div>
+          </div>
+        </div>
+      )}
+
       <TooltipProvider>
         <div className="anycontext-flex anycontext-group anycontext-flex-col anycontext-gap-2 anycontext-fixed anycontext-bottom-12 anycontext-right-0 anycontext-z-[99999] anycontext-font-sans">
           {window.location.href.includes("twitter.com") ||
@@ -227,9 +275,6 @@ function SideBar() {
               >
                 <DialogTrigger asChild>
                   <button
-                    onClick={() => {
-											return;
-                    }}
                     disabled={savedWebsites.includes(window.location.href)}
                     className="anycontext-open-button disabled:anycontext-opacity-30 anycontext-bg-transparent
 											anycontext-border-none anycontext-m-0 anycontext-p-0"

apps/extension/src/background.ts

@@ -6,7 +6,16 @@ const backendUrl =
     ? "http://localhost:3000"
     : "https://supermemory.dhr.wtf";
 
-// TODO: Implement getting bookmarks from API directly
+interface TweetData {
+  tweetText: string;
+  postUrl: string;
+  authorName: string;
+  handle: string;
+  time: string;
+  saveToUser: string;
+}
+
+// TODO: Implement getting bookmarks from Twitter API directly
 // let authorizationHeader: string | null = null;
 // let csrfToken: string | null = null;
 // let cookies: string | null = null;
@@ -130,7 +139,7 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
       return true;
     })();
   }
-  // TODO: Implement getting bookmarks from API directly
+  // TODO: Implement getting bookmarks from Twitter API directly
   // else if (request.action === 'getAuthData') {
   //   sendResponse({
   //     authorizationHeader: authorizationHeader,
@@ -138,4 +147,22 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
   //     cookies: cookies
   //   });
   // }
+  else if (request.type === "sendBookmarkedTweets") {
+    const jwt = request.jwt;
+    const tweets = request.tweets as TweetData[];
+
+    (async () => {
+      await fetch(`${backendUrl}/api/vectorizeTweets`, {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${jwt}`,
+        },
+        body: JSON.stringify(tweets),
+      }).then(async (response) => {
+        return response.json();
+      });
+    })();
+
+    return true;
+  }
 });

apps/extension/src/content.tsx

@@ -38,8 +38,8 @@ const jwt = chrome.storage.local.get("jwt").then((data) => {
   return data.jwt;
 }) as Promise<string>;
 
-jwt.then(() => {
+jwt.then((token) => {
   ReactDOM.createRoot(
     document.getElementById("anycontext-app-container")!,
-  ).render(<SideBar />);
+  ).render(<SideBar jwt={token} />);
 });

apps/extension/src/util.ts

@@ -9,5 +9,5 @@ export const getEnv = () => {
   // })
 
   // return null
-  return "development";
+  return "production";
 };

apps/web/public/twitter.svg

@@ -0,0 +1 @@
+<svg viewBox="0 0 256 209" width="256" height="209" xmlns="http://www.w3.org/2000/svg" preserveAspectRatio="xMidYMid"><path d="M256 25.45c-9.42 4.177-19.542 7-30.166 8.27 10.845-6.5 19.172-16.793 23.093-29.057a105.183 105.183 0 0 1-33.351 12.745C205.995 7.201 192.346.822 177.239.822c-29.006 0-52.523 23.516-52.523 52.52 0 4.117.465 8.125 1.36 11.97-43.65-2.191-82.35-23.1-108.255-54.876-4.52 7.757-7.11 16.78-7.11 26.404 0 18.222 9.273 34.297 23.365 43.716a52.312 52.312 0 0 1-23.79-6.57c-.003.22-.003.44-.003.661 0 25.447 18.104 46.675 42.13 51.5a52.592 52.592 0 0 1-23.718.9c6.683 20.866 26.08 36.05 49.062 36.475-17.975 14.086-40.622 22.483-65.228 22.483-4.24 0-8.42-.249-12.529-.734 23.243 14.902 50.85 23.597 80.51 23.597 96.607 0 149.434-80.031 149.434-149.435 0-2.278-.05-4.543-.152-6.795A106.748 106.748 0 0 0 256 25.45" fill="#55acee"/></svg>

apps/web/public/web.svg

@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6">
+  <path stroke-linecap="round" stroke-linejoin="round" d="M12 21a9.004 9.004 0 0 0 8.716-6.747M12 21a9.004 9.004 0 0 1-8.716-6.747M12 21c2.485 0 4.5-4.03 4.5-9S14.485 3 12 3m0 18c-2.485 0-4.5-4.03-4.5-9S9.515 3 12 3m0 0a8.997 8.997 0 0 1 7.843 4.582M12 3a8.997 8.997 0 0 0-7.843 4.582m15.686 0A11.953 11.953 0 0 1 12 10.5c-2.998 0-5.74-1.1-7.843-2.918m15.686 0A8.959 8.959 0 0 1 21 12c0 .778-.099 1.533-.284 2.253m0 0A17.919 17.919 0 0 1 12 16.5c-3.162 0-6.133-.815-8.716-2.247m0 0A9.015 9.015 0 0 1 3 12c0-1.605.42-3.113 1.157-4.418" />
+</svg>  

apps/web/src/app/api/addTweetsToDb/route.ts

@@ -0,0 +1,91 @@
+import { db } from "@/server/db";
+import { eq } from "drizzle-orm";
+import { sessions, storedContent, users } from "@/server/db/schema";
+import { type NextRequest, NextResponse } from "next/server";
+
+export const runtime = "edge";
+
+interface TweetData {
+  tweetText: string;
+  postUrl: string;
+  authorName: string;
+  handle: string;
+  time: string;
+  saveToUser: string;
+}
+
+export async function POST(req: NextRequest) {
+  const token =
+    req.cookies.get("next-auth.session-token")?.value ??
+    req.cookies.get("__Secure-authjs.session-token")?.value ??
+    req.cookies.get("authjs.session-token")?.value ??
+    req.headers.get("Authorization")?.replace("Bearer ", "");
+
+  if (!token) {
+    return new Response(
+      JSON.stringify({ message: "Invalid Key, TOKEN not found." }),
+      { status: 404 },
+    );
+  }
+
+  const sessionData = await db
+    .select()
+    .from(sessions)
+    .where(eq(sessions.sessionToken, token!));
+
+  if (!sessionData || sessionData.length === 0) {
+    return new Response(
+      JSON.stringify({ message: "Invalid Key, session not found." }),
+      { status: 404 },
+    );
+  }
+
+  const user = await db
+    .select()
+    .from(users)
+    .where(eq(users.id, sessionData[0].userId))
+    .limit(1);
+
+  if (!user || user.length === 0) {
+    return NextResponse.json(
+      { message: "Invalid Key, session not found." },
+      { status: 404 },
+    );
+  }
+
+  const session = { session: sessionData[0], user: user[0] };
+
+  const data = (await req.json()) as TweetData[];
+
+  for (const tweet of data) {
+    const { id } = (
+      await db
+        .insert(storedContent)
+        .values({
+          content: tweet.tweetText,
+          title: "Twitter Bookmark",
+          description: "",
+          url: tweet.postUrl,
+          baseUrl: "https://twitter.com",
+          image: "https://supermemory.dhr.wtf/twitter.svg",
+          savedAt: new Date(),
+          user: session.user.id,
+          type: "twitter-bookmark",
+        })
+        .returning({ id: storedContent.id })
+    )[0];
+
+    if (!id) {
+      return NextResponse.json(
+        {
+          message: "Error",
+          error:
+            "Something went wrong when inserting the tweet to storedContent",
+        },
+        { status: 500 },
+      );
+    }
+  }
+
+  return NextResponse.json({ message: "OK", data: "Success" }, { status: 200 });
+}

apps/web/src/app/api/getCount/route.ts

@@ -0,0 +1,66 @@
+import { db } from "@/server/db";
+import { and, eq, sql } from "drizzle-orm";
+import { sessions, storedContent, users } from "@/server/db/schema";
+import { type NextRequest, NextResponse } from "next/server";
+
+export const runtime = "edge";
+
+export async function GET(req: NextRequest) {
+  const token =
+    req.cookies.get("next-auth.session-token")?.value ??
+    req.cookies.get("__Secure-authjs.session-token")?.value ??
+    req.cookies.get("authjs.session-token")?.value ??
+    req.headers.get("Authorization")?.replace("Bearer ", "");
+
+  if (!token) {
+    return new Response(
+      JSON.stringify({ message: "Invalid Key, session not found." }),
+      { status: 404 },
+    );
+  }
+
+  const sessionData = await db
+    .select()
+    .from(sessions)
+    .where(eq(sessions.sessionToken, token!));
+
+  if (!sessionData || sessionData.length === 0) {
+    return new Response(
+      JSON.stringify({ message: "Invalid Key, session not found." }),
+      { status: 404 },
+    );
+  }
+
+  const user = await db
+    .select()
+    .from(users)
+    .where(eq(users.id, sessionData[0].userId))
+    .limit(1);
+
+  if (!user || user.length === 0) {
+    return NextResponse.json(
+      { message: "Invalid Key, session not found." },
+      { status: 404 },
+    );
+  }
+
+  const session = { session: sessionData[0], user: user[0] };
+
+  const count = await db
+    .select({
+      count: sql<number>`count(*)`.mapWith(Number),
+    })
+    .from(storedContent)
+    .where(
+      and(
+        eq(storedContent.user, session.user.id),
+        eq(storedContent.type, "twitter-bookmark"),
+      ),
+    );
+
+  return NextResponse.json({
+    tweetsCount: count[0].count,
+    limit: 1000,
+    user: session.user.email,
+  });
+}

apps/web/src/app/api/spaces/route.ts

@@ -3,10 +3,15 @@ import { sessions, space, users } from "@/server/db/schema";
 import { eq } from "drizzle-orm";
 import { NextRequest, NextResponse } from "next/server";
 
+<<<<<<< HEAD
 export const runtime = "edge"
 
 export async function GET(req: NextRequest) {
+=======
+export const runtime = "edge";
+>>>>>>> 7648bdaa8cbe42a90f05865f8c555c9a3911af9b
 
+export async function GET(req: NextRequest) {
   const token =
     req.cookies.get("next-auth.session-token")?.value ??
     req.cookies.get("__Secure-authjs.session-token")?.value ??
@@ -56,8 +61,9 @@ export async function GET(req: NextRequest) {
     );
   }
 
-	const user = userData[0]
+  const user = userData[0];
 
+<<<<<<< HEAD
 	const spaces = await db
 		.select()
 		.from(space)
@@ -71,5 +77,19 @@ export async function GET(req: NextRequest) {
 		message: "OK",
 		data: spaces
 	}, { status: 200 })
+=======
+  const spaces = await db
+    .select()
+    .from(space)
+    .where(eq(space.user, user.id))
+    .all();
+>>>>>>> 7648bdaa8cbe42a90f05865f8c555c9a3911af9b
 
+  return NextResponse.json(
+    {
+      message: "OK",
+      data: spaces,
+    },
+    { status: 200 },
+  );
 }

apps/web/src/app/api/vectorizeTweets/route.ts

@@ -0,0 +1,63 @@
+import { db } from "@/server/db";
+import { eq } from "drizzle-orm";
+import { sessions, storedContent, users } from "@/server/db/schema";
+import { type NextRequest, NextResponse } from "next/server";
+import { env } from "@/env";
+
+export const runtime = "edge";
+
+interface TweetData {
+  tweetText: string;
+  postUrl: string;
+  authorName: string;
+  handle: string;
+  time: string;
+  saveToUser: string;
+}
+
+export async function POST(req: NextRequest) {
+  const token =
+    req.cookies.get("next-auth.session-token")?.value ??
+    req.cookies.get("__Secure-authjs.session-token")?.value ??
+    req.cookies.get("authjs.session-token")?.value ??
+    req.headers.get("Authorization")?.replace("Bearer ", "");
+
+  if (!token) {
+    return new Response(
+      JSON.stringify({ message: "Invalid Key, session not found." }),
+      { status: 404 },
+    );
+  }
+
+  const sessionData = await db
+    .select()
+    .from(sessions)
+    .where(eq(sessions.sessionToken, token!));
+
+  if (!sessionData || sessionData.length === 0) {
+    return new Response(
+      JSON.stringify({ message: "Invalid Key, session not found." }),
+      { status: 404 },
+    );
+  }
+
+  const body = (await req.json()) as TweetData[];
+
+  console.log(body);
+
+  const resp = await fetch(
+    `https://cf-ai-backend.dhravya.workers.dev/batchUploadTweets`,
+    {
+      headers: {
+        "X-Custom-Auth-Key": env.BACKEND_SECURITY_KEY,
+      },
+      method: "POST",
+      body: JSON.stringify(body),
+    },
+  );
+
+  return new Response(await resp.text(), {
+    status: resp.status,
+    headers: resp.headers,
+  });
+}

apps/web/src/server/helpers.ts

@@ -17,17 +17,19 @@ export async function getMetaData(url: string) {
     ? descriptionMatch[1]
     : "Description not found";
 
-  // Extract Open Graph image
-  const imageMatch = html.match(
-    /<meta property="og:image" content="(.*?)"\s*\/?>/,
+  // Extract favicon
+  const faviconMatch = html.match(
+    /<link rel="(?:icon|shortcut icon)" href="(.*?)"\s*\/?>/,
   );
-  const image = imageMatch ? imageMatch[1] : "Image not found";
+  const favicon = faviconMatch
+    ? faviconMatch[1]
+    : "https://supermemory.dhr.wtf/web.svg";
 
   // Prepare the metadata object
   const metadata = {
     title,
     description,
-    image,
+    image: favicon,
     baseUrl,
   };
   return metadata;