change embedding model

.gitignore

@@ -1,6 +1,7 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 bun.lockb
 
+packages/scripts/*
 # Dependencies
 node_modules/
 /.pnp

apps/backend/drizzle/0015_perpetual_mauler.sql

@@ -0,0 +1 @@
+ALTER TABLE "chunks" ALTER COLUMN "embeddings" SET DATA TYPE vector(768);

apps/backend/drizzle/meta/_journal.json

@@ -106,6 +106,13 @@
       "when": 1736852938881,
       "tag": "0014_mighty_the_captain",
       "breakpoints": true
+    },
+    {
+      "idx": 15,
+      "version": "7",
+      "when": 1737920848112,
+      "tag": "0015_perpetual_mauler",
+      "breakpoints": true
     }
   ]
 }

apps/backend/package.json

@@ -5,6 +5,7 @@
     "deploy": "bunx wrangler deploy --minify",
     "generate-migration": "dotenv -- npx drizzle-kit generate",
     "migrate:local": "bun run ./scripts/migrate.ts",
+    "migrate:prod": "NODE_ENV=production bun run ./scripts/migrate.ts",
     "tail": "bunx wrangler tail"
   },
   "dependencies": {

apps/backend/scripts/migrate.ts

@@ -6,11 +6,12 @@ import postgres from "postgres";
 
 config();
 
-if (!process.env.DATABASE_URL) {
-	throw new Error("DATABASE_URL is not set");
-}
+const isProd = process.env.NODE_ENV === "production";
+const connectionString = isProd ? process.env.PROD_DATABASE_URL : process.env.DATABASE_URL;
 
-const connectionString = process.env.DATABASE_URL!;
+if (!connectionString) {
+	throw new Error(`${isProd ? "PROD_DATABASE_URL" : "DATABASE_URL"} is not set`);
+}
 
 console.log("Connecting to:", connectionString.replace(/:[^:@]+@/, ":****@")); // Log sanitized connection string
 

apps/backend/src/routes/actions.ts

@@ -54,8 +54,6 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
 
       const { messages, threadId } = await c.req.valid("json");
 
-      // TODO: add rate limiting
-
       const unfilteredCoreMessages = convertToCoreMessages(
         (messages as Message[])
           .filter((m) => m.content.length > 0)
@@ -67,125 +65,66 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
                 ? `<context>${JSON.stringify(m.annotations)}</context>`
                 : ""),
             experimental_attachments:
-              m.experimental_attachments &&
-              m.experimental_attachments.length > 0
+              m.experimental_attachments?.length &&
+              m.experimental_attachments?.length > 0
                 ? m.experimental_attachments
                 : (m.data as { files: [] })?.files,
           }))
       );
 
-      // make sure that there is no empty messages. if there is, remove it.
       const coreMessages = unfilteredCoreMessages.filter(
         (message) => message.content.length > 0
       );
-      // .map(async (c) => {
-      //   if (
-      //     Array.isArray(c.content) &&
-      //     c.content.some((c) => c.type !== "text")
-      //   ) {
-      //     // convert attachments (IMAGE and files) to base64 by fetching them
-      //     const attachments = c.content.filter((c) => c.type !== "text");
-      //     const base64Attachments = await Promise.all(
-      //       attachments.map(async (a) => {
-      //         const type = (a as ImagePart | FilePart).type;
-      //         if (type === "image") {
-      //           const response = await fetch((a as ImagePart).image.toString());
-      //           return response.arrayBuffer();
-      //         } else if (type === "file") {
-      //           const response = await fetch((a as FilePart).data.toString());
-      //           return response.arrayBuffer();
-      //         }
-      //       })
-      //     );
-      //   }
-      // });
-
-      console.log("Core messages", JSON.stringify(coreMessages, null, 2));
-
-      let threadUuid = threadId;
 
+      const db = database(c.env.HYPERDRIVE.connectionString);
       const { initLogger, wrapAISDKModel } = await import("braintrust");
 
+      // Initialize clients and loggers
       const logger = initLogger({
         projectName: "supermemory",
         apiKey: c.env.BRAINTRUST_API_KEY,
       });
 
-      // const gemini = createOpenAI({
-      //   apiKey: c.env.GEMINI_API_KEY,
-      //   baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/",
-      // });
-      const openaiClient = openai(c.env);
+      const googleClient = wrapAISDKModel(openai(c.env).chat("gpt-4o"));
 
-      const googleClient = wrapAISDKModel(
-        // google(c.env.GEMINI_API_KEY).chat("gemini-exp-1206")
-        openai(c.env).chat("gpt-4o")
-      );
-
-      // Create new thread if none exists
-      if (!threadUuid) {
-        const uuid = randomId();
-        const newThread = await database(c.env.HYPERDRIVE.connectionString)
-          .insert(chatThreads)
-          .values({
-            firstMessage: messages[0].content,
-            userId: user.id,
-            uuid: uuid,
-            messages: coreMessages,
-          })
-          .returning();
-
-        threadUuid = newThread[0].uuid;
-      }
-      const openAi = openai(c.env);
-
-
-
-      let lastUserMessage = coreMessages
-        .reverse()
-        .find((i) => i.role === "user");
-
-      // get the text of lastUserMEssage
+      // Get last user message and generate embedding in parallel with thread creation
+      let lastUserMessage = coreMessages.findLast((i) => i.role === "user");
       const queryText =
         typeof lastUserMessage?.content === "string"
           ? lastUserMessage.content
           : lastUserMessage?.content.map((c) => (c as TextPart).text).join("");
 
-      console.log("querytext", queryText);
-
-      if (!queryText ||queryText.length === 0) {
-        return c.json({ error: "Failed to generate embedding for query" }, 500);
+      if (!queryText || queryText.length === 0) {
+        return c.json({ error: "Empty query" }, 400);
       }
 
-      const embedStart = performance.now();
-      const { data: embedding } = await c.env.AI.run("@cf/baai/bge-base-en-v1.5", {
-        text: queryText,
-      });
-      const embedEnd = performance.now();
-      console.log(`Embedding generation took ${embedEnd - embedStart}ms`);
+      // Run embedding generation and thread creation in parallel
+      const [{ data: embedding }, thread] = await Promise.all([
+        c.env.AI.run("@cf/baai/bge-base-en-v1.5", { text: queryText }),
+        !threadId
+          ? db
+              .insert(chatThreads)
+              .values({
+                firstMessage: messages[0].content,
+                userId: user.id,
+                uuid: randomId(),
+                messages: coreMessages,
+              })
+              .returning()
+          : null,
+      ]);
+
+      const threadUuid = threadId || thread?.[0].uuid;
 
       if (!embedding) {
-        return c.json({ error: "Failed to generate embedding for query" }, 500);
+        return c.json({ error: "Failed to generate embedding" }, 500);
       }
 
-      // Perform semantic search using cosine similarity
-      // Log the query text to debug what we're searching for
-      console.log("Searching for:", queryText);
-      console.log("user id", user.id);
+      // Perform semantic search
+      const similarity = sql<number>`1 - (${cosineDistance(chunk.embeddings, embedding[0])})`;
 
-      const similarity = sql<number>`1 - (${cosineDistance(
-        chunk.embeddings,
-        embedding[0]
-      )})`;
-
-      // Find similar chunks using cosine similarity
-      // Join with documents table to get chunks only from documents the user has access to
-      // First get all results to normalize
-      // Get top 20 results first to avoid processing entire dataset
-      const dbQueryStart = performance.now();
-      const topResults = await database(c.env.HYPERDRIVE.connectionString)
+      const finalResults = await db
         .select({
-          similarity,
           id: documents.id,
           content: documents.content,
           type: documents.type,
@@ -200,62 +139,10 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
         .from(chunk)
         .innerJoin(documents, eq(chunk.documentId, documents.id))
         .where(and(eq(documents.userId, user.id), sql`${similarity} > 0.4`))
-        .orderBy(desc(similarity));
+        .orderBy(desc(similarity))
+        .limit(5);
 
-      // Get unique documents with their highest similarity chunks
-      const uniqueDocuments = Object.values(
-        topResults.reduce(
-          (acc, curr) => {
-            if (
-              !acc[curr.id] ||
-              acc[curr.id].content === curr.content ||
-              acc[curr.id].url === curr.url
-            ) {
-              acc[curr.id] = curr;
-            }
-            return acc;
-          },
-          {} as Record<number, (typeof topResults)[0]>
-        )
-      ).slice(0, 5);
-
-      const dbQueryEnd = performance.now();
-      console.log(`Database query took ${dbQueryEnd - dbQueryStart}ms`);
-
-      // Calculate min/max once for the subset
-      const processingStart = performance.now();
-      const minSimilarity = Math.min(
-        ...uniqueDocuments.map((r) => r.similarity)
-      );
-      const maxSimilarity = Math.max(
-        ...uniqueDocuments.map((r) => r.similarity)
-      );
-      const range = maxSimilarity - minSimilarity;
-
-      // Normalize the results
-      const normalizedResults = uniqueDocuments.map((result) => ({
-        ...result,
-        normalizedSimilarity:
-          range === 0 ? 1 : (result.similarity - minSimilarity) / range,
-      }));
-
-      // Get either all results above 0.6 threshold, or at least top 3 results
-      const results = normalizedResults
-        .sort((a, b) => b.normalizedSimilarity - a.normalizedSimilarity)
-        .slice(
-          0,
-          Math.max(
-            3,
-            normalizedResults.filter((r) => r.normalizedSimilarity > 0.6).length
-          )
-        );
-
-      const processingEnd = performance.now();
-      console.log(
-        `Results processing took ${processingEnd - processingStart}ms`
-      );
-
-      const cleanDocumentsForContext = results.map((d) => ({
+      const cleanDocumentsForContext = finalResults.map((d) => ({
         title: d.title,
         description: d.description,
         url: d.url,
@@ -263,8 +150,6 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
         content: d.content,
       }));
 
-      // Update lastUserMessage with search results
-      const messageUpdateStart = performance.now();
       if (lastUserMessage) {
         lastUserMessage.content =
           typeof lastUserMessage.content === "string"
@@ -277,26 +162,30 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
                   text: `<context>${JSON.stringify(cleanDocumentsForContext)}</context>`,
                 },
               ];
-      }
-
-      // edit the last coreusermessage in the array
-      if (lastUserMessage) {
         coreMessages[coreMessages.length - 1] = lastUserMessage;
       }
-      const messageUpdateEnd = performance.now();
-      console.log(
-        `Message update took ${messageUpdateEnd - messageUpdateStart}ms`
-      );
 
       try {
-        const streamStart = performance.now();
+        const data = new StreamData();
+        data.appendMessageAnnotation(
+          finalResults.map((r) => ({
+            id: r.id,
+            content: r.content,
+            type: r.type,
+            url: r.url,
+            title: r.title,
+            description: r.description,
+            ogImage: r.ogImage,
+            userId: r.userId,
+            createdAt: r.createdAt.toISOString(),
+            updatedAt: r.updatedAt?.toISOString() || null,
+          }))
+        );
+
         const result = await streamText({
           model: googleClient,
           experimental_providerMetadata: {
-            metadata: {
-              userId: user.id,
-              chatThreadId: threadUuid,
-            },
+            metadata: { userId: user.id, chatThreadId: threadUuid ?? "" },
           },
           experimental_transform: smoothStream(),
           messages: [
@@ -323,12 +212,11 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
           ],
           async onFinish(completion) {
             try {
-              // remove context from lastUserMessage
               if (lastUserMessage) {
                 lastUserMessage.content =
                   typeof lastUserMessage.content === "string"
                     ? lastUserMessage.content.replace(
-                        /<context>([\s\S]*?)<\/context>/g,
+                        /<context>[\s\S]*?<\/context>/g,
                         ""
                       )
                     : lastUserMessage.content.filter(
@@ -338,60 +226,34 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
                             part.text.startsWith("<context>")
                           )
                       );
-
                 coreMessages[coreMessages.length - 1] = lastUserMessage;
               }
 
-              console.log("results", results);
-
               const newMessages = [
                 ...coreMessages,
                 {
                   role: "assistant",
                   content:
                     completion.text +
-                    `<context>[${JSON.stringify(results)}]</context>`,
+                    `<context>[${JSON.stringify(finalResults)}]</context>`,
                 },
               ];
-              await data.close();
 
               if (threadUuid) {
-                await database(c.env.HYPERDRIVE.connectionString)
+                await db
                   .update(chatThreads)
                   .set({ messages: newMessages })
                   .where(eq(chatThreads.uuid, threadUuid));
               }
             } catch (error) {
               console.error("Failed to update thread:", error);
-              // Continue execution - the message was delivered even if saving failed
             }
           },
         });
 
-        const streamEnd = performance.now();
-        console.log(`Stream response took ${streamEnd - streamStart}ms`);
-
-        const data = new StreamData();
-
-        const context = results.map((r) => ({
-          similarity: r.similarity,
-          id: r.id,
-          content: r.content,
-          type: r.type,
-          url: r.url,
-          title: r.title,
-          description: r.description,
-          ogImage: r.ogImage,
-          userId: r.userId,
-          createdAt: r.createdAt.toISOString(),
-          updatedAt: r.updatedAt?.toISOString() || null,
-        }));
-        // Full context objects in the data
-        data.appendMessageAnnotation(context);
-
         return result.toDataStreamResponse({
           headers: {
-            "Supermemory-Thread-Uuid": threadUuid,
+            "Supermemory-Thread-Uuid": threadUuid ?? "",
             "Content-Type": "text/x-unknown",
             "content-encoding": "identity",
             "transfer-encoding": "chunked",
@@ -408,7 +270,6 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
           );
         }
 
-        // Handle database connection errors
         if ((error as AISDKError).cause === "ECONNREFUSED") {
           return c.json({ error: "Database connection failed" }, 503);
         }

apps/backend/src/utils/fetchers.ts

@@ -35,11 +35,6 @@ export const fetchContent = async (
       tweetUrl.search = ""; // Remove all search params
       const tweetId = tweetUrl.pathname.split("/").pop();
 
-      const unrolledTweetContent = await step.do(
-        "get unrolled tweet content",
-        async () => await unrollTweets(tweetUrl.toString())
-      );
-
       const rawBaseTweetContent = await step.do(
         "extract tweet content",
         async () => {
@@ -72,8 +67,10 @@ export const fetchContent = async (
         };
         raw: string;
       };
-
-      if (!unrolledTweetContent || isErr(unrolledTweetContent)) {
+      const unrolledTweetContent = {
+        value: [rawBaseTweetContent],
+      };
+      if (true) {
         console.error("Can't get thread, reverting back to single tweet");
         tweetContent = {
           text: rawBaseTweetContent.text,

apps/web/app/components/Reminders.tsx

@@ -3,7 +3,6 @@ import Markdown from "react-markdown";
 
 import { useNavigate } from "@remix-run/react";
 
-import { client } from "../lib/utils/api";
 import image from "./gradients/gradient1.png";
 import { AddMemoryModal } from "./memories/AddMemory";
 import { Button } from "./ui/button";

apps/web/app/root.tsx

@@ -29,6 +29,7 @@ import "@fontsource/geist-sans/900.css";
 import { QueryClient, QueryClientProvider, useQuery } from "@tanstack/react-query";
 import { Toaster } from "sonner";
 import posthog from "posthog-js";
+import { PostHogProvider, usePostHog} from 'posthog-js/react'
 
 const queryClient = new QueryClient();
 
@@ -130,10 +131,10 @@ export const loader = async ({ request, context }: LoaderFunctionArgs) => {
 	});
 };
 
-// Use React.memo to memoize the App component
 const App = React.memo(function App() {
 	const data = useLoaderData<typeof loader>();
 	const [theme] = useTheme();
+	const posthog = usePostHog();
 
 	useEffect(() => {
 		if (data.user) {
@@ -147,16 +148,16 @@ const App = React.memo(function App() {
 				lastName: data.user.lastName,
 			});
 		}
-	}, []);
+	}, [data.user]);
 
 	return (
-		<html lang="en" data-theme={theme ?? "light"} className={theme ?? ""}>
+		<html lang="en" data-theme={theme ?? "light"}>
 			<head>
 				<Meta />
 				<Links />
 				<NonFlashOfWrongThemeEls ssrTheme={Boolean(data.theme)} />
 			</head>
-			<body>
+			<body className={theme ?? ""}>
 				<Outlet />
 				<ScrollRestoration />
 				<Scripts />
@@ -174,23 +175,17 @@ const App = React.memo(function App() {
 });
 
 export default function AppWithProviders() {	
-	const data = useLoaderData<typeof loader>();
-
-	const specifiedTheme = useMemo(() => data.theme, [data.theme]);
-
-	const MemoizedApp = React.memo(App);
-	const MemoizedThemeProvider = useMemo(
-		() => <ThemeProvider specifiedTheme={specifiedTheme}>
-			<MemoizedApp />
-		</ThemeProvider>,
-		[specifiedTheme]
-	);
+	const data = useLoaderData<typeof loader>()
 
 	return (
-		<KeyboardProvider>
-			<QueryClientProvider client={queryClient}>
-				{MemoizedThemeProvider}
-			</QueryClientProvider>
-		</KeyboardProvider>
+		<PostHogProvider client={posthog}>
+			<KeyboardProvider>
+				<QueryClientProvider client={queryClient}>
+					<ThemeProvider specifiedTheme={data.theme}>
+						<App />
+					</ThemeProvider>
+				</QueryClientProvider>
+			</KeyboardProvider>
+		</PostHogProvider>
 	);
 }

apps/web/app/routes/pitch.index.tsx

@@ -0,0 +1,93 @@
+import React, { useEffect } from "react";
+
+import posthog from "posthog-js";
+import { Logo } from "~/components/icons/Logo";
+import { Theme, useTheme } from "~/lib/theme-provider";
+
+function PitchPage1() {
+	return (
+		<div className="h-screen w-screen flex flex-col justify-center px-4 sm:px-8">
+			<Logo className="w-[min(25vw,16rem)] h-[min(25vw,16rem)] mb-4" />
+			<div className="w-full max-w-[90vw]">
+				<h1 className="text-[min(10vw,12rem)] font-bold tracking-tight leading-none whitespace-nowrap">
+					<span className="inline">super</span>
+					<span className="inline">memory</span>
+				</h1>
+                <p className="text-[min(3vw,3rem)] font-medium tracking-tight leading-none mt-8">
+                    The second brain platform for everyone. <br />
+                    <span className="text-[min(1.5vw,1rem)] text-gray-500">dhravya shah draft</span>
+                </p>
+			</div>
+		</div>
+	);
+}
+
+function PitchPage2() {
+	return (
+		<div className="h-screen w-screen flex flex-col justify-center px-4 sm:px-8">
+			<h2 className="text-[min(5vw,4rem)] font-bold tracking-tight leading-none mb-12">current problems</h2>
+			<div className="grid grid-cols-4 gap-4 max-w-7xl mx-auto w-full h-[60vh] relative">
+				<div className="absolute -right-24 top-1/2 -translate-y-1/2 w-48">
+					<img 
+						src="https://www.harleytherapy.co.uk/counselling/wp-content/uploads/4624465693_115ce5fa02-400x300.jpg"
+						alt="Messy desk with papers"
+						className="rounded-lg shadow-lg"
+					/>
+					<div className="font-handwritten text-lg text-blue-600 -rotate-12 mt-2 ml-4">
+						current knowledgebase
+						<svg className="w-12 h-12 -mt-2 ml-2 transform rotate-45" viewBox="0 0 24 24" fill="none" stroke="currentColor">
+							<path d="M5 12h14M12 5l7 7-7 7" strokeWidth={2} strokeLinecap="round" strokeLinejoin="round"/>
+						</svg>
+					</div>
+				</div>
+
+				<div className="col-span-2 row-span-2 border rounded-xl p-8 hover:bg-gray-50 transition-colors transform hover:scale-105 hover:rotate-1">
+					<h3 className="font-semibold text-3xl mb-4">Knowledge Management</h3>
+					<p className="text-gray-600 text-lg">Information overload is real. Notes scattered everywhere, bookmarks lost in endless folders, important details buried in email threads. We're building this for Chakshu who's drowning in digital chaos.</p>
+				</div>
+				<div className="border rounded-xl p-6 hover:bg-gray-50 transition-colors transform hover:-rotate-2">
+					<h3 className="font-semibold text-xl mb-2">Trust & Privacy</h3>
+					<p className="text-gray-600 text-sm">Developer friends like Kshunya and Siddharth need their data private and secure</p>
+				</div>
+				<div className="border rounded-xl p-6 hover:bg-gray-50 transition-colors transform hover:rotate-2">
+					<h3 className="font-semibold text-xl mb-2">Enterprise Search</h3>
+					<p className="text-gray-600 text-sm">Cloudflare's internal knowledge is a maze of confusion</p>
+				</div>
+				<div className="col-span-2 border rounded-xl p-6 hover:bg-gray-50 transition-colors transform hover:-rotate-1">
+					<h3 className="font-semibold text-2xl mb-3">Digital Chaos</h3>
+					<p className="text-gray-600">Notes in Notion, bookmarks in Chrome, knowledge in Slack, wisdom in emails... it's everywhere and nowhere</p>
+				</div>
+				<div className="border rounded-xl p-6 hover:bg-gray-50 transition-colors transform hover:rotate-1">
+					<h3 className="font-semibold text-xl mb-2">Information Anxiety</h3>
+					<p className="text-gray-600 text-sm">Brent spends hours searching through old emails</p>
+				</div>
+				<div className="border rounded-xl p-6 hover:bg-gray-50 transition-colors transform hover:-rotate-1">
+					<h3 className="font-semibold text-xl mb-2">Developer Cost</h3>
+					<p className="text-gray-600 text-sm">Memory APIs are a costly maze of complexity</p>
+				</div>
+			</div>
+		</div>
+	);
+}
+
+function Pitch() {
+	const [theme, setTheme] = useTheme();
+	useEffect(() => {
+		posthog.capture("pitch_viewed");
+		setTheme(Theme.LIGHT);
+	}, []);
+
+	return (
+		<div className="snap-y snap-mandatory h-screen w-screen overflow-y-auto">
+			<div className="snap-start">
+				<PitchPage1 />
+			</div>
+            <div className="snap-start">
+				<PitchPage2 />
+			</div>
+			{/* Add more pages here with snap-start class */}
+		</div>
+	);
+}
+
+export default Pitch;

packages/db/schema.ts

@@ -215,7 +215,7 @@ export const chunk = pgTable(
       .notNull(),
     textContent: text("text_content"),
     orderInDocument: integer("order_in_document").notNull(),
-    embeddings: vector("embeddings", { dimensions: 1536 }),
+    embeddings: vector("embeddings", { dimensions: 768 }),
     metadata: jsonb("metadata").$type<Metadata>(),
     createdAt: timestamp("created_at", { withTimezone: true })
       .notNull()