diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 9c1f459d..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Build and Deploy Changes - -on: - push: - branches: [main] - paths: - - "apps/web/**" - - "apps/extension/**" - - "apps/cf-ai-backend/**" - pull_request: - branches: [main] - paths: - - "apps/web/**" - - "apps/extension/**" - - "apps/cf-ai-backend/**" - -jobs: - build-extension: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/buildextension - - build-app: - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Setup Bun - uses: oven-sh/setup-bun@v1 - with: - bun-version: latest - - - name: Install packages - run: bun i - shell: bash - - - name: Build app - run: bun run pages:build - working-directory: apps/web - shell: bash - env: - GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} - GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} - NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }} - DATABASE_URL: ${{ secrets.DATABASE_URL }} - NEXTAUTH_URL: ${{ secrets.NEXTAUTH_URL }} - BACKEND_SECURITY_KEY: ${{ secrets.BACKEND_SECURITY_KEY }} - - - name: Publish to Cloudflare Pages - uses: cloudflare/pages-action@v1 - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - with: - apiToken: ${{ secrets.CF_API_TOKEN }} - accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - projectName: ${{ secrets.CLOUDFLARE_PROJECT_NAME }} - directory: apps/web/.vercel/output/static - branch: main - - # deploy-cf-worker: - # runs-on: ubuntu-latest - # steps: - # - name: Checkout repo - # uses: actions/checkout@v3 - - # - name: Deploy to Cloudflare Workers - # uses: cloudflare/wrangler-action@1.2.0 - # with: - # apiToken: ${{ secrets.CF_API_TOKEN }} - # workingDirectory: apps/cf-ai-backend - # env: - # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - # SECURITY_KEY: ${{ secrets.BACKEND_SECURITY_KEY }} diff --git a/.gitignore b/.gitignore index d8480f00..c80a841f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ bun.lockb .*.vars .wrangler .million +yarn.lock +package-lock.json # Dependencies node_modules diff --git a/SETUP-GUIDE.md b/SETUP-GUIDE.md index 7d69b545..f51b1cb1 100644 --- a/SETUP-GUIDE.md +++ b/SETUP-GUIDE.md @@ -13,12 +13,13 @@ 3. Create a `.dev.vars` file in `apps/web` with the following content: ```bash -GOOGLE_CLIENT_ID="-" -GOOGLE_CLIENT_SECRET="-" +GOOGLE_CLIENT_ID="-" // required, visit https://developers.google.com/identity/protocols/oauth2 +GOOGLE_CLIENT_SECRET="-" // required NEXTAUTH_SECRET='nextauthsecret' DATABASE_URL='database.sqlite' NEXTAUTH_URL='http://localhost:3000' BACKEND_SECURITY_KEY='veryrandomsecuritykey' +BACKEND_BASE_URL="where your backend is hosted" ``` 4. Setup the database: @@ -28,10 +29,10 @@ First, edit the `wrangler.toml` file in `apps/web` to point the d1 database to y You can create a d1 database by running this command ``` -wrangler d1 create DATABASE_NAME +bunx wrangler d1 create ``` -And then replace these values +And then replace database_name and database_id with the values ``` [[d1_databases]] @@ -43,10 +44,12 @@ database_id = "YOUR_DB_ID" Simply run this command in `apps/web` ``` -wrangler d1 execute dev-d1-anycontext --local --file=db/prepare.sql +bunx wrangler d1 migrations apply ``` -If it runs, you can set up the cloud database as well by removing the `--local` flag. +If it runs, you can set up the cloud database as well by removing the `--local` flag, + +if you just want to contribute to frontend then just run `bun run dev` in the root of the project and done! (you won't be able to try ai stuff), otherwise continue... 5. You need to host your own worker for the `apps/cf-ai-backend` module. diff --git a/apps/browser-rendering b/apps/browser-rendering index b37c9623..4d21045a 160000 --- a/apps/browser-rendering +++ b/apps/browser-rendering @@ -1 +1 @@ -Subproject commit b37c962365a36cf342a31a196f4908f4f1343553 +Subproject commit 4d21045a45fdbf56b7483d3704ae0474ebf044fb diff --git a/apps/cf-ai-backend/README.md b/apps/cf-ai-backend/README.md index 86409f29..91d6b77e 100644 --- a/apps/cf-ai-backend/README.md +++ b/apps/cf-ai-backend/README.md @@ -1,58 +1,50 @@ -# Hono minimal project +baseURL: https://new-cf-ai-backend.dhravya.workers.dev -This is a minimal project with [Hono](https://github.com/honojs/hono/) for Cloudflare Workers. +Authentication: +You must authenticate with a header and `Authorization: bearer token` for each request in `/api/*` routes. -## Features +### Add content: -- Minimal -- TypeScript -- Wrangler to develop and deploy. -- [Jest](https://jestjs.io/ja/) for testing. - -## Usage - -Initialize +POST `/api/add` with ``` -npx create-cloudflare my-app https://github.com/honojs/hono-minimal +body { + pageContent: z.string(), + title: z.string().optional(), + description: z.string().optional(), + space: z.string().optional(), + url: z.string(), + user: z.string(), +} ``` -Install +### Query without user data + +GET `/api/ask` with +query `?query=testing` + +(this is temp but works perfectly, will change soon for chat use cases specifically) + +### Query vectorize and get results in natural language + +POST `/api/chat` with ``` -yarn install +query paramters (?query=...&" { + query: z.string(), + topK: z.number().optional().default(10), + user: z.string(), + spaces: z.string().optional(), + sourcesOnly: z.string().optional().default("false"), + model: z.string().optional().default("gpt-4o"), + } + +body z.object({ + chatHistory: z.array(contentObj).optional(), +}); ``` -Develop +### Delete vectors -``` -yarn dev -``` - -Test - -``` -yarn test -``` - -Deploy - -``` -yarn deploy -``` - -## Examples - -See: - -## For more information - -See: - -## Author - -Yusuke Wada - -## License - -MIT +DELETE `/api/delete` with +query param websiteUrl, user diff --git a/apps/cf-ai-backend/package.json b/apps/cf-ai-backend/package.json index 480f9601..78353e08 100644 --- a/apps/cf-ai-backend/package.json +++ b/apps/cf-ai-backend/package.json @@ -6,7 +6,7 @@ "scripts": { "test": "jest --verbose", "deploy": "wrangler deploy", - "dev": "wrangler dev", + "dev": "wrangler dev --remote --port 8686", "start": "wrangler dev", "unsafe-reset-vector-db": "wrangler vectorize delete supermem-vector && wrangler vectorize create --dimensions=1536 supermem-vector-1 --metric=cosine" }, diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 87495c59..44dba383 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -21,8 +21,6 @@ export async function initQuery( index: c.env.VECTORIZE_INDEX, }); - const DEFAULT_MODEL = "gpt-4o"; - let selectedModel: | ReturnType> | ReturnType> @@ -52,12 +50,6 @@ export async function initQuery( break; } - if (!selectedModel) { - throw new Error( - `Model ${model} not found and default model ${DEFAULT_MODEL} is also not available.`, - ); - } - return { store, model: selectedModel }; } @@ -72,19 +64,60 @@ export async function deleteDocument({ c: Context<{ Bindings: Env }>; store: CloudflareVectorizeStore; }) { - const toBeDeleted = `${url}-${user}`; + const toBeDeleted = `${url}#supermemory-web`; const random = seededRandom(toBeDeleted); const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15); - await c.env.KV.list({ prefix: uuid }).then(async (keys) => { - for (const key of keys.keys) { - await c.env.KV.delete(key.name); - await store.delete({ ids: [key.name] }); + const allIds = await c.env.KV.list({ prefix: uuid }); + + if (allIds.keys.length > 0) { + const savedVectorIds = allIds.keys.map((key) => key.name); + const vectors = await c.env.VECTORIZE_INDEX.getByIds(savedVectorIds); + // We don't actually delete document directly, we just remove the user from the metadata. + // If there's no user left, we can delete the document. + const newVectors = vectors.map((vector) => { + delete vector.metadata[`user-${user}`]; + + // Get count of how many users are left + const userCount = Object.keys(vector.metadata).filter((key) => + key.startsWith("user-"), + ).length; + + // If there's no user left, we can delete the document. + // need to make sure that every chunk is deleted otherwise it would be problematic. + if (userCount === 0) { + store.delete({ ids: savedVectorIds }); + void Promise.all(savedVectorIds.map((id) => c.env.KV.delete(id))); + return null; + } + + return vector; + }); + + // If all vectors are null (deleted), we can delete the KV too. Otherwise, we update (upsert) the vectors. + if (newVectors.every((v) => v === null)) { + await c.env.KV.delete(uuid); + } else { + await c.env.VECTORIZE_INDEX.upsert(newVectors.filter((v) => v !== null)); } - }); + } +} + +function sanitizeKey(key: string): string { + if (!key) throw new Error("Key cannot be empty"); + + // Remove or replace invalid characters + let sanitizedKey = key.replace(/[.$"]/g, "_"); + + // Ensure key does not start with $ + if (sanitizedKey.startsWith("$")) { + sanitizedKey = sanitizedKey.substring(1); + } + + return sanitizedKey; } export async function batchCreateChunksAndEmbeddings({ @@ -98,19 +131,47 @@ export async function batchCreateChunksAndEmbeddings({ chunks: string[]; context: Context<{ Bindings: Env }>; }) { - const ourID = `${body.url}-${body.user}`; - - await deleteDocument({ url: body.url, user: body.user, c: context, store }); - + //! NOTE that we use #supermemory-web to ensure that + //! If a user saves it through the extension, we don't want other users to be able to see it. + // Requests from the extension should ALWAYS have a unique ID with the USERiD in it. + // I cannot stress this enough, important for security. + const ourID = `${body.url}#supermemory-web`; const random = seededRandom(ourID); + const uuid = + random().toString(36).substring(2, 15) + + random().toString(36).substring(2, 15); + + const allIds = await context.env.KV.list({ prefix: uuid }); + + // If some chunks for that content already exist, we'll just update the metadata to include + // the user. + if (allIds.keys.length > 0) { + const savedVectorIds = allIds.keys.map((key) => key.name); + const vectors = await context.env.VECTORIZE_INDEX.getByIds(savedVectorIds); + + // Now, we'll update all vector metadatas with one more userId and all spaceIds + const newVectors = vectors.map((vector) => { + vector.metadata = { + ...vector.metadata, + [`user-${body.user}`]: 1, + + // For each space in body, add the spaceId to the vector metadata + ...(body.spaces ?? [])?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}), + }; + + return vector; + }); + + await context.env.VECTORIZE_INDEX.upsert(newVectors); + return; + } for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; - const uuid = - random().toString(36).substring(2, 15) + - random().toString(36).substring(2, 15) + - "-" + - i; + const chunkId = `${uuid}-${i}`; const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${chunk}`; @@ -121,19 +182,25 @@ export async function batchCreateChunksAndEmbeddings({ metadata: { title: body.title?.slice(0, 50) ?? "", description: body.description ?? "", - space: body.space ?? "", url: body.url, - user: body.user, + type: body.type ?? "page", + content: newPageContent, + + [sanitizeKey(`user-${body.user}`)]: 1, + ...body.spaces?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}), }, }, ], { - ids: [uuid], + ids: [chunkId], }, ); console.log("Docs added: ", docs); - await context.env.KV.put(uuid, ourID); + await context.env.KV.put(chunkId, ourID); } } diff --git a/apps/cf-ai-backend/src/index.test.ts b/apps/cf-ai-backend/src/index.test.ts deleted file mode 100644 index bbf66fb5..00000000 --- a/apps/cf-ai-backend/src/index.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import app from "."; - -// TODO: write more tests -describe("Test the application", () => { - it("Should return 200 response", async () => { - const res = await app.request("http://localhost/"); - expect(res.status).toBe(200); - }), - it("Should return 404 response", async () => { - const res = await app.request("http://localhost/404"); - expect(res.status).toBe(404); - }); -}); diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 19770dec..60188090 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { Hono } from "hono"; -import { CoreMessage, streamText } from "ai"; +import { CoreMessage, generateText, streamText, tool } from "ai"; import { chatObj, Env, vectorObj } from "./types"; import { batchCreateChunksAndEmbeddings, @@ -14,9 +14,18 @@ import { bearerAuth } from "hono/bearer-auth"; import { zValidator } from "@hono/zod-validator"; import chunkText from "./utils/chonker"; import { systemPrompt, template } from "./prompts/prompt1"; +import { swaggerUI } from "@hono/swagger-ui"; +import { createOpenAI } from "@ai-sdk/openai"; const app = new Hono<{ Bindings: Env }>(); +app.get( + "/ui", + swaggerUI({ + url: "/doc", + }), +); + // ------- MIDDLEWARES ------- app.use("*", poweredBy()); app.use("*", timing()); @@ -31,6 +40,17 @@ app.use("/api/", async (c, next) => { }); // ------- MIDDLEWARES END ------- +const fileSchema = z + .instanceof(File) + .refine( + (file) => file.size <= 10 * 1024 * 1024, + "File size should be less than 10MB", + ) // Validate file size + .refine( + (file) => ["image/jpeg", "image/png", "image/gif"].includes(file.type), + "Invalid file type", + ); // Validate file type + app.get("/", (c) => { return c.text("Supermemory backend API is running!"); }); @@ -54,6 +74,82 @@ app.post("/api/add", zValidator("json", vectorObj), async (c) => { return c.json({ status: "ok" }); }); +app.post( + "/api/add-with-image", + zValidator( + "form", + z.object({ + images: z + .array(fileSchema) + .min(1, "At least one image is required") + .optional(), + "images[]": z + .array(fileSchema) + .min(1, "At least one image is required") + .optional(), + text: z.string().optional(), + spaces: z.array(z.string()).optional(), + url: z.string(), + user: z.string(), + }), + (c) => { + console.log(c); + }, + ), + async (c) => { + const body = c.req.valid("form"); + + const { store } = await initQuery(c); + + if (!(body.images || body["images[]"])) { + return c.json({ status: "error", message: "No images found" }, 400); + } + + const imagePromises = (body.images ?? body["images[]"]).map( + async (image) => { + const buffer = await image.arrayBuffer(); + const input = { + image: [...new Uint8Array(buffer)], + prompt: + "What's in this image? caption everything you see in great detail. If it has text, do an OCR and extract all of it.", + max_tokens: 1024, + }; + const response = await c.env.AI.run( + "@cf/llava-hf/llava-1.5-7b-hf", + input, + ); + console.log(response.description); + return response.description; + }, + ); + + const imageDescriptions = await Promise.all(imagePromises); + + await batchCreateChunksAndEmbeddings({ + store, + body: { + url: body.url, + user: body.user, + type: "image", + description: + imageDescriptions.length > 1 + ? `A group of ${imageDescriptions.length} images on ${body.url}` + : imageDescriptions[0], + spaces: body.spaces, + pageContent: imageDescriptions.join("\n"), + title: "Image content from the web", + }, + chunks: [ + imageDescriptions, + ...(body.text ? chunkText(body.text, 1536) : []), + ].flat(), + context: c, + }); + + return c.json({ status: "ok" }); + }, +); + app.get( "/api/ask", zValidator( @@ -74,6 +170,159 @@ app.get( }, ); +// This is a special endpoint for our "chatbot-only" solutions. +// It does both - adding content AND chatting with it. +app.post( + "/api/autoChatOrAdd", + zValidator( + "query", + z.object({ + query: z.string(), + user: z.string(), + }), + ), + zValidator("json", chatObj), + async (c) => { + const { query, user } = c.req.valid("query"); + const { chatHistory } = c.req.valid("json"); + + const { store, model } = await initQuery(c); + + let task: "add" | "chat" = "chat"; + let thingToAdd: "page" | "image" | "text" | undefined = undefined; + let addContent: string | undefined = undefined; + + // This is a "router". this finds out if the user wants to add a document, or chat with the AI to get a response. + const routerQuery = await generateText({ + model: model, + system: `You are Supermemory chatbot. You can either add a document to the supermemory database, or return a chat response. Based on this query, + You must determine what to do. Basically if it feels like a "question", then you should intiate a chat. If it feels like a "command" or feels like something that could be forwarded to the AI, then you should add a document. + You must also extract the "thing" to add and what type of thing it is.`, + prompt: `Question from user: ${query}`, + tools: { + decideTask: tool({ + description: + "Decide if the user wants to add a document or chat with the AI", + parameters: z.object({ + generatedTask: z.enum(["add", "chat"]), + contentToAdd: z.object({ + thing: z.enum(["page", "image", "text"]), + content: z.string(), + }), + }), + execute: async ({ generatedTask, contentToAdd }) => { + task = generatedTask; + thingToAdd = contentToAdd.thing; + addContent = contentToAdd.content; + }, + }), + }, + }); + + if ((task as string) === "add") { + // addString is the plaintext string that the user wants to add to the database + let addString: string = addContent; + + if (thingToAdd === "page") { + // TODO: Sometimes this query hangs, and errors out. we need to do proper error management here. + const response = await fetch("https://md.dhr.wtf/?url=" + addContent, { + headers: { + Authorization: "Bearer " + c.env.SECURITY_KEY, + }, + }); + + addString = await response.text(); + } + + // At this point, we can just go ahead and create the embeddings! + await batchCreateChunksAndEmbeddings({ + store, + body: { + url: addContent, + user, + type: thingToAdd, + pageContent: addString, + title: `${addString.slice(0, 30)}... (Added from chatbot)`, + }, + chunks: chunkText(addString, 1536), + context: c, + }); + + return c.json({ + status: "ok", + response: + "I added the document to your personal second brain! You can now use it to answer questions or chat with me.", + contentAdded: { + type: thingToAdd, + content: addString, + url: + thingToAdd === "page" + ? addContent + : `https://supermemory.ai/note/${Date.now()}`, + }, + }); + } else { + const filter: VectorizeVectorMetadataFilter = { + [`user-${user}`]: 1, + }; + + const queryAsVector = await store.embeddings.embedQuery(query); + + const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { + topK: 5, + filter, + returnMetadata: true, + }); + + const minScore = Math.min(...resp.matches.map(({ score }) => score)); + const maxScore = Math.max(...resp.matches.map(({ score }) => score)); + + // This entire chat part is basically just a dumb down version of the /api/chat endpoint. + const normalizedData = resp.matches.map((data) => ({ + ...data, + normalizedScore: + maxScore !== minScore + ? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98 + : 50, + })); + + const preparedContext = normalizedData.map( + ({ metadata, score, normalizedScore }) => ({ + context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, + score, + normalizedScore, + }), + ); + + const prompt = template({ + contexts: preparedContext, + question: query, + }); + + const initialMessages: CoreMessage[] = [ + { + role: "system", + content: `You are an AI chatbot called "Supermemory.ai". When asked a question by a user, you must take all the context provided to you and give a good, small, but helpful response.`, + }, + { role: "assistant", content: "Hello, how can I help?" }, + ]; + + const userMessage: CoreMessage = { role: "user", content: prompt }; + + const response = await generateText({ + model, + messages: [ + ...initialMessages, + ...((chatHistory || []) as CoreMessage[]), + userMessage, + ], + }); + + return c.json({ status: "ok", response: response.text }); + } + }, +); + /* TODO: Eventually, we should not have to save each user's content in a seperate vector. Lowkey, it makes sense. The user may save their own version of a page - like selected text from twitter.com url. But, it's not scalable *enough*. How can we store the same vectors for the same content, without needing to duplicate for each uer? @@ -85,8 +334,8 @@ app.post( "query", z.object({ query: z.string(), - topK: z.number().optional().default(10), user: z.string(), + topK: z.number().optional().default(10), spaces: z.string().optional(), sourcesOnly: z.string().optional().default("false"), model: z.string().optional().default("gpt-4o"), @@ -97,105 +346,107 @@ app.post( const query = c.req.valid("query"); const body = c.req.valid("json"); - if (body.chatHistory) { - body.chatHistory = body.chatHistory.map((i) => ({ - ...i, - content: i.parts.length > 0 ? i.parts.join(" ") : i.content, - })); + const sourcesOnly = query.sourcesOnly === "true"; + + // Return early for dumb requests + if (sourcesOnly && body.sources) { + return c.json(body.sources); } - const sourcesOnly = query.sourcesOnly === "true"; - const spaces = query.spaces?.split(",") || [undefined]; + const spaces = query.spaces?.split(",") ?? [undefined]; // Get the AI model maker and vector store const { model, store } = await initQuery(c, query.model); - const filter: VectorizeVectorMetadataFilter = { user: query.user }; + if (!body.sources) { + const filter: VectorizeVectorMetadataFilter = { + [`user-${query.user}`]: 1, + }; + console.log("Spaces", spaces); - // Converting the query to a vector so that we can search for similar vectors - const queryAsVector = await store.embeddings.embedQuery(query.query); - const responses: VectorizeMatches = { matches: [], count: 0 }; + // Converting the query to a vector so that we can search for similar vectors + const queryAsVector = await store.embeddings.embedQuery(query.query); + const responses: VectorizeMatches = { matches: [], count: 0 }; - // SLICED to 5 to avoid too many queries - for (const space of spaces.slice(0, 5)) { - if (space !== undefined) { - // it's possible for space list to be [undefined] so we only add space filter conditionally - filter.space = space; + console.log("hello world", spaces); + + // SLICED to 5 to avoid too many queries + for (const space of spaces.slice(0, 5)) { + console.log("space", space); + if (!space && spaces.length > 1) { + // it's possible for space list to be [undefined] so we only add space filter conditionally + filter[`space-${query.user}-${space}`] = 1; + } + + // Because there's no OR operator in the filter, we have to make multiple queries + const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { + topK: query.topK, + filter, + returnMetadata: true, + }); + + // Basically recreating the response object + if (resp.count > 0) { + responses.matches.push(...resp.matches); + responses.count += resp.count; + } } - // Because there's no OR operator in the filter, we have to make multiple queries - const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { - topK: query.topK, - filter, - returnMetadata: true, - }); + const minScore = Math.min(...responses.matches.map(({ score }) => score)); + const maxScore = Math.max(...responses.matches.map(({ score }) => score)); - // Basically recreating the response object - if (resp.count > 0) { - responses.matches.push(...resp.matches); - responses.count += resp.count; + // We are "normalising" the scores - if all of them are on top, we want to make sure that + // we have a way to filter out the noise. + const normalizedData = responses.matches.map((data) => ({ + ...data, + normalizedScore: + maxScore !== minScore + ? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98 + : 50, // If all scores are the same, set them to the middle of the scale + })); + + let highScoreData = normalizedData.filter( + ({ normalizedScore }) => normalizedScore > 50, + ); + + // If the normalsation is not done properly, we have a fallback to just get the + // top 3 scores + if (highScoreData.length === 0) { + highScoreData = normalizedData + .sort((a, b) => b.score - a.score) + .slice(0, 3); + } + + const sortedHighScoreData = highScoreData.sort( + (a, b) => b.normalizedScore - a.normalizedScore, + ); + + body.sources = { + normalizedData, + }; + + // So this is kinda hacky, but the frontend needs to do 2 calls to get sources and chat. + // I think this is fine for now, but we can improve this later. + if (sourcesOnly) { + const idsAsStrings = sortedHighScoreData.map((dataPoint) => + dataPoint.id.toString(), + ); + + const storedContent = await Promise.all( + idsAsStrings.map(async (id) => await c.env.KV.get(id)), + ); + + const metadata = normalizedData.map((datapoint) => datapoint.metadata); + + return c.json({ ids: storedContent, metadata, normalizedData }); } } - const minScore = Math.min(...responses.matches.map(({ score }) => score)); - const maxScore = Math.max(...responses.matches.map(({ score }) => score)); - - // We are "normalising" the scores - if all of them are on top, we want to make sure that - // we have a way to filter out the noise. - const normalizedData = responses.matches.map((data) => ({ - ...data, - normalizedScore: - maxScore !== minScore - ? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98 - : 50, // If all scores are the same, set them to the middle of the scale - })); - - let highScoreData = normalizedData.filter( - ({ normalizedScore }) => normalizedScore > 50, - ); - - // If the normalsation is not done properly, we have a fallback to just get the - // top 3 scores - if (highScoreData.length === 0) { - highScoreData = normalizedData - .sort((a, b) => b.score - a.score) - .slice(0, 3); - } - - const sortedHighScoreData = highScoreData.sort( - (a, b) => b.normalizedScore - a.normalizedScore, - ); - - // So this is kinda hacky, but the frontend needs to do 2 calls to get sources and chat. - // I think this is fine for now, but we can improve this later. - if (sourcesOnly) { - const idsAsStrings = sortedHighScoreData.map((dataPoint) => - dataPoint.id.toString(), - ); - - // We are getting the content ID back, so that the frontend can show the actual sources properly. - // it IS a lot of DB calls, i completely agree. - // TODO: return metadata value here, so that the frontend doesn't have to re-fetch anything. - const storedContent = await Promise.all( - idsAsStrings.map(async (id) => await c.env.KV.get(id)), - ); - - return c.json({ ids: storedContent }); - } - - const vec = responses.matches.map((data) => ({ metadata: data.metadata })); - - const vecWithScores = vec.map((v, i) => ({ - ...v, - score: sortedHighScoreData[i].score, - normalisedScore: sortedHighScoreData[i].normalizedScore, - })); - - const preparedContext = vecWithScores.map( - ({ metadata, score, normalisedScore }) => ({ + const preparedContext = body.sources.normalizedData.map( + ({ metadata, score, normalizedScore }) => ({ context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, score, - normalisedScore, + normalizedScore, }), ); @@ -245,4 +496,28 @@ app.delete( }, ); +// ERROR #1 - this is the api that the editor uses, it is just a scrape off of /api/chat so you may check that out +app.get( + "/api/editorai", + zValidator( + "query", + z.object({ + context: z.string(), + request: z.string(), + }), + ), + async (c) => { + const { context, request } = c.req.valid("query"); + const { model } = await initQuery(c); + + const response = await streamText({ + model, + prompt: `${request}-${context}`, + maxTokens: 224, + }); + + return response.toTextStreamResponse(); + }, +); + export default app; diff --git a/apps/cf-ai-backend/src/prompts/prompt1.ts b/apps/cf-ai-backend/src/prompts/prompt1.ts index aa7694d3..289495b6 100644 --- a/apps/cf-ai-backend/src/prompts/prompt1.ts +++ b/apps/cf-ai-backend/src/prompts/prompt1.ts @@ -6,28 +6,24 @@ To generate your answer: - Carefully analyze the question and identify the key information needed to address it - Locate the specific parts of each context that contain this key information - Compare the relevance scores of the provided contexts -- In the tags, provide a brief justification for which context(s) are more relevant to answering the question based on the scores - Concisely summarize the relevant information from the higher-scoring context(s) in your own words - Provide a direct answer to the question - Use markdown formatting in your answer, including bold, italics, and bullet points as appropriate to improve readability and highlight key points - Give detailed and accurate responses for things like 'write a blog' or long-form questions. - The normalisedScore is a value in which the scores are 'balanced' to give a better representation of the relevance of the context, between 1 and 100, out of the top 10 results - -Provide your justification between tags and your final answer between tags, formatting both in markdown. - +- provide your justification in the end, in a tag If no context is provided, introduce yourself and explain that the user can save content which will allow you to answer questions about that content in the future. Do not provide an answer if no context is provided.`; export const template = ({ contexts, question }) => { // Map over contexts to generate the context and score parts const contextParts = contexts .map( - ({ context, score, normalisedScore }) => ` + ({ context, normalisedScore }) => ` ${context} - score: ${score} normalisedScore: ${normalisedScore} `, ) diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index bea4bf80..dc97777c 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -1,8 +1,9 @@ +import { sourcesZod } from "@repo/shared-types"; import { z } from "zod"; export type Env = { VECTORIZE_INDEX: VectorizeIndex; - AI: Fetcher; + AI: Ai; SECURITY_KEY: string; OPENAI_API_KEY: string; GOOGLE_AI_API_KEY: string; @@ -37,13 +38,15 @@ export const contentObj = z.object({ export const chatObj = z.object({ chatHistory: z.array(contentObj).optional(), + sources: sourcesZod.optional(), }); export const vectorObj = z.object({ pageContent: z.string(), title: z.string().optional(), description: z.string().optional(), - space: z.string().optional(), + spaces: z.array(z.string()).optional(), url: z.string(), user: z.string(), + type: z.string().optional().default("page"), }); diff --git a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts index 3514f579..be5839b1 100644 --- a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts +++ b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; + interface OpenAIEmbeddingsParams { apiKey: string; modelName: string; @@ -32,12 +34,22 @@ export class OpenAIEmbeddings { }), }); - const data = (await response.json()) as { - data: { - embedding: number[]; - }[]; - }; + const data = await response.json(); - return data.data[0].embedding; + const zodTypeExpected = z.object({ + data: z.array( + z.object({ + embedding: z.array(z.number()), + }), + ), + }); + + const json = zodTypeExpected.safeParse(data); + + if (!json.success) { + throw new Error("Invalid response from OpenAI: " + json.error.message); + } + + return json.data.data[0].embedding; } } diff --git a/apps/cf-ai-backend/src/utils/chonker.ts b/apps/cf-ai-backend/src/utils/chonker.ts index 39d4b458..c63020be 100644 --- a/apps/cf-ai-backend/src/utils/chonker.ts +++ b/apps/cf-ai-backend/src/utils/chonker.ts @@ -1,5 +1,8 @@ import nlp from "compromise"; +/** + * Split text into chunks of specified max size with some overlap for continuity. + */ export default function chunkText( text: string, maxChunkSize: number, diff --git a/apps/cf-ai-backend/src/utils/seededRandom.ts b/apps/cf-ai-backend/src/utils/seededRandom.ts index 36a1e4f9..9e315ee8 100644 --- a/apps/cf-ai-backend/src/utils/seededRandom.ts +++ b/apps/cf-ai-backend/src/utils/seededRandom.ts @@ -1,5 +1,9 @@ import { MersenneTwister19937, integer } from "random-js"; +/** + * Hashes a string to a 32-bit integer. + * @param {string} seed - The input string to hash. + */ function hashString(seed: string) { let hash = 0; for (let i = 0; i < seed.length; i++) { @@ -10,6 +14,9 @@ function hashString(seed: string) { return hash; } +/** + * returns a funtion that generates same sequence of random numbers for a given seed between 0 and 1. + */ export function seededRandom(seed: string) { const seedHash = hashString(seed); const engine = MersenneTwister19937.seed(seedHash); diff --git a/apps/cf-ai-backend/tsconfig.json b/apps/cf-ai-backend/tsconfig.json index 2b75d5a0..fcdf6914 100644 --- a/apps/cf-ai-backend/tsconfig.json +++ b/apps/cf-ai-backend/tsconfig.json @@ -1,6 +1,7 @@ { "compilerOptions": { "lib": ["ES2020"], - "types": ["@cloudflare/workers-types"] + "types": ["@cloudflare/workers-types"], + "downlevelIteration": true } } diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index db0ae945..ea93fd63 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -3,9 +3,14 @@ main = "src/index.ts" compatibility_date = "2024-02-23" node_compat = true +# [env.preview] [[vectorize]] binding = "VECTORIZE_INDEX" -index_name = "supermem-vector" +index_name = "supermem-vector-dev" + +# [[vectorize]] +# binding = "VECTORIZE_INDEX" +# index_name = "supermem-vector-prod" [ai] binding = "AI" diff --git a/apps/extension/README.md b/apps/extension/README.md new file mode 100644 index 00000000..d5aceb2c --- /dev/null +++ b/apps/extension/README.md @@ -0,0 +1,37 @@ +# [projectName] + +> This project was bootstrapped using the Extension.js React-TypeScript template. + +## Scripts Available + +In the project directory, you can run: + +### [projectPackageManager] dev + +``` +// Runs the app in the development mode. +// Will open a new browser instance with your extension loaded. +// The page will reload when you make changes. +[projectPackageManager] dev +``` + +### [projectPackageManager] start + +``` +// Runs the app in the production mode. +// Will open a new browser instance with your extension loaded. +// This is how your browser extension will work once published. +[projectPackageManager] start +``` + +### [projectPackageManager] build + +``` +// Builds the app for production. +// Bundles your browser extension in production mode for the target browser. +[projectPackageManager] run build +``` + +## Learn More + +You can learn more in the [Extension.js](https://extension.js.org) documentation. diff --git a/apps/extension/background.ts b/apps/extension/background.ts new file mode 100644 index 00000000..8f3162fc --- /dev/null +++ b/apps/extension/background.ts @@ -0,0 +1,25 @@ +chrome.runtime.onInstalled.addListener(function () { + let context = 'selection'; + let title = "Supermemory - Save Highlight"; + chrome.contextMenus.create({ + title: title, + contexts: ['selection'], + id: context, + }); +}); + +chrome.contextMenus.onClicked.addListener(function (info, tab) { +if (info.menuItemId === 'selection') { + // you can add a link to a cf worker or whatever u want + // fetch("", { + // method: "POST", + // headers: { "Content-Type": "application/json" }, + // body: JSON.stringify({ + // data: info.selectionText, + // }), + // }); + + //so you first save it and then send the reponse to the screen + chrome.tabs.sendMessage(tab?.id || 1, info.selectionText); +} +}); \ No newline at end of file diff --git a/apps/extension/content/ContentApp.tsx b/apps/extension/content/ContentApp.tsx new file mode 100644 index 00000000..b8cb6710 --- /dev/null +++ b/apps/extension/content/ContentApp.tsx @@ -0,0 +1,45 @@ +import React, { useEffect } from "react"; + +export default function ContentApp() { + const [text, setText] = React.useState(""); + const [hover, setHover] = React.useState(false); + + useEffect(() => { + const messageListener = (message: any) => { + setText(message); + setTimeout(() => setText(""), 2000); + }; + chrome.runtime.onMessage.addListener(messageListener); + + document.addEventListener('mousemove', (e)=> { + const percentageX = (e.clientX / window.innerWidth) * 100; + const percentageY = (e.clientY / window.innerHeight) * 100; + + if (percentageX > 75 && percentageY > 75){ + setHover(true) + } else { + setHover(false) + } + }) + return () => { + chrome.runtime.onMessage.removeListener(messageListener); + }; + }, []); + + return ( +
+
+
+
+ +
+

Saved!

+

{text}

+
+
+ ); +} \ No newline at end of file diff --git a/apps/extension/content/base.css b/apps/extension/content/base.css new file mode 100644 index 00000000..bd6213e1 --- /dev/null +++ b/apps/extension/content/base.css @@ -0,0 +1,3 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; \ No newline at end of file diff --git a/apps/extension/content/content.css b/apps/extension/content/content.css new file mode 100644 index 00000000..b8195ee8 --- /dev/null +++ b/apps/extension/content/content.css @@ -0,0 +1,6 @@ +#extension-root { + position: fixed; + bottom: 0; + right: 0; + z-index: 99999; +} diff --git a/apps/extension/content/content.tsx b/apps/extension/content/content.tsx new file mode 100644 index 00000000..65c62d74 --- /dev/null +++ b/apps/extension/content/content.tsx @@ -0,0 +1,15 @@ +import ReactDOM from 'react-dom/client' +import ContentApp from './ContentApp' +import('./base.css') +import('./content.css') + +setTimeout(initial, 4000) + +function initial() { + const rootDiv = document.createElement('div') + rootDiv.id = 'extension-root' + document.body.appendChild(rootDiv) + + const root = ReactDOM.createRoot(rootDiv) + root.render(<>) +} diff --git a/apps/extension/extension-env.d.ts b/apps/extension/extension-env.d.ts new file mode 100644 index 00000000..7cadeab2 --- /dev/null +++ b/apps/extension/extension-env.d.ts @@ -0,0 +1,9 @@ +// Required Extension.js types for TypeScript projects. +// This file is auto-generated and should not be excluded. +// If you need extra types, consider creating a new *.d.ts and +// referencing it in the "include" array in your tsconfig.json file. +// See https://www.typescriptlang.org/tsconfig#include for info. +/// + +// Polyfill types for browser.* APIs. +/// diff --git a/apps/extension/index.html b/apps/extension/index.html new file mode 100644 index 00000000..bb4c07c7 --- /dev/null +++ b/apps/extension/index.html @@ -0,0 +1,11 @@ + + + + + + God Bless Vanilla JavaScript!!! + + +

hello World! Follow @supermemoryai

+ + \ No newline at end of file diff --git a/apps/extension/manifest.json b/apps/extension/manifest.json new file mode 100644 index 00000000..529b0afe --- /dev/null +++ b/apps/extension/manifest.json @@ -0,0 +1,30 @@ + +{ + "name": "Supermemory.ai-Extension", + "description": "Uses the chrome.contextMenus API to customize the context menu.", + "version": "0.1", + "permissions": [ + "contextMenus" + ], + "manifest_version": 3, + "action": { + "default_popup": "index.html" + }, + "background": { + "service_worker": "./background.ts" + }, + "content_scripts": [ + { + "matches": [ + "" + ], + "js": [ + "./content/content.tsx" + ] + } + ], + "icons": { + "16": "public/icon/logo(16).png", + "48": "public/icon/logo(48).png" + } +} \ No newline at end of file diff --git a/apps/extension/package.json b/apps/extension/package.json new file mode 100644 index 00000000..7b06b3da --- /dev/null +++ b/apps/extension/package.json @@ -0,0 +1,20 @@ +{ + "devDependencies": { + "@types/react": "^18.0.9", + "@types/react-dom": "^18.0.5", + "react": "^18.1.0", + "react-dom": "^18.1.0", + "tailwindcss": "^3.4.1", + "typescript": "5.3.3", + "extension": "latest" + }, + "scripts": { + "dev": "extension dev", + "start": "extension start", + "build": "extension build" + }, + "dependencies": {}, + "name": "extension", + "private": true, + "version": "0.0.0" +} \ No newline at end of file diff --git a/apps/extension/public/icon/logo(16).png b/apps/extension/public/icon/logo(16).png new file mode 100644 index 00000000..3c1610b0 Binary files /dev/null and b/apps/extension/public/icon/logo(16).png differ diff --git a/apps/extension/public/icon/logo(48).png b/apps/extension/public/icon/logo(48).png new file mode 100644 index 00000000..de5a6d2e Binary files /dev/null and b/apps/extension/public/icon/logo(48).png differ diff --git a/apps/extension/tailwind.config.js b/apps/extension/tailwind.config.js new file mode 100644 index 00000000..1beb1aca --- /dev/null +++ b/apps/extension/tailwind.config.js @@ -0,0 +1,8 @@ +module.exports = { + content: ['**/*.html', '**/*.tsx'], + theme: { + extend: {} + }, + plugins: [] +} +module.exports = require("@repo/tailwind-config/tailwind.config"); \ No newline at end of file diff --git a/apps/extension/tsconfig.json b/apps/extension/tsconfig.json new file mode 100644 index 00000000..8538580f --- /dev/null +++ b/apps/extension/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "@repo/typescript-config/nextjs.json", + "compilerOptions": { + "allowJs": true, + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": false, + "jsx": "react-jsx", + "lib": ["dom", "dom.iterable", "esnext"], + "moduleResolution": "node", + "module": "esnext", + "resolveJsonModule": true, + "strict": true, + "target": "esnext" + }, + "include": ["./"], + "exclude": ["node_modules", "dist"] +} \ No newline at end of file diff --git a/apps/web/app/(auth)/auth-buttons.tsx b/apps/web/app/(auth)/auth-buttons.tsx index 0e99213e..5b0ad06e 100644 --- a/apps/web/app/(auth)/auth-buttons.tsx +++ b/apps/web/app/(auth)/auth-buttons.tsx @@ -2,7 +2,7 @@ import { Button } from "@repo/ui/shadcn/button"; import React from "react"; -import { signIn } from "../helpers/server/auth"; +import { signIn } from "../../server/auth"; function SignIn() { return ( diff --git a/apps/web/app/(canvas)/canvas/[id]/page.tsx b/apps/web/app/(canvas)/canvas/[id]/page.tsx new file mode 100644 index 00000000..6efb6cf4 --- /dev/null +++ b/apps/web/app/(canvas)/canvas/[id]/page.tsx @@ -0,0 +1,129 @@ +"use client"; + +import { Canvas } from "@repo/ui/components/canvas/components/canvas"; +import React, { useState } from "react"; +import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels"; +import { SettingsIcon, DragIcon } from "@repo/ui/icons"; +import DraggableComponentsContainer from "@repo/ui/components/canvas/components/draggableComponent"; +import { AutocompleteIcon, blockIcon } from "@repo/ui/icons"; +import Image from "next/image"; +import { Switch } from "@repo/ui/shadcn/switch"; +import { Label } from "@repo/ui/shadcn/label"; +import { useRouter } from "next/router"; + +function page() { + const [fullScreen, setFullScreen] = useState(false); + const [visible, setVisible] = useState(true); + + const router = useRouter(); + router.push("/home"); + + return ( +
+
+ { + l[0]! < 20 ? setVisible(false) : setVisible(true); + }} + className={` ${fullScreen ? "w-[calc(100vw-2rem)]" : "w-screen"} transition-all`} + direction="horizontal" + > + { + setTimeout(() => setFullScreen(false), 50); + }} + onCollapse={() => { + setTimeout(() => setFullScreen(true), 50); + }} + defaultSize={30} + collapsible={true} + > +
+
+ Change Filters + setting-icon +
+ {visible ? ( + + ) : ( +

+ Need more space to show! +

+ )} +
+ + +
+ drag-icon +
+ + +
+ +
+ + +
+
+ ); +} + +function SidePanel() { + const [value, setValue] = useState(""); + const [dragAsText, setDragAsText] = useState(false); + return ( + <> +
+ { + setValue(e.target.value); + }} + value={value} + // rows={1} + className="w-full resize-none rounded-xl bg-[#151515] px-3 py-4 text-xl text-[#989EA4] outline-none focus:outline-none sm:max-h-52" + /> +
+
+ setDragAsText(e)} + id="drag-text-mode" + /> + +
+ + + ); +} + +export default page; + +const content = [ + { + content: + "Regional growth patterns diverge, with strong performance in the United States and several emerging markets, contrasted by weaker prospects in many advanced economies, particularly in Europe (World Economic Forum) (OECD). The rapid adoption of artificial intelligence (AI) is expected to drive productivity growth, especially in advanced economies, potentially mitigating labor shortages and boosting income levels in emerging markets (World Economic Forum) (OECD). However, ongoing geopolitical tensions and economic fragmentation are likely to maintain a level of uncertainty and volatility in the global economy (World Economic Forum.", + icon: AutocompleteIcon, + iconAlt: "Autocomplete", + extraInfo: + "Page Url: https://chatgpt.com/c/762cd44e-1752-495b-967a-aa3c23c6024a", + }, + { + content: + "As of mid-2024, the global economy is experiencing modest growth with significant regional disparities. Global GDP growth is projected to be around 3.1% in 2024, rising slightly to 3.2% in 2025. This performance, although below the pre-pandemic average, reflects resilience despite various economic pressures, including tight monetary conditions and geopolitical tensions (IMF)(OECD) Inflation is moderating faster than expected, with global headline inflation projected to fall to 5.8% in 2024 and 4.4% in 2025, contributing to improving real incomes and positive trade growth (IMF) (OECD)", + icon: blockIcon, + iconAlt: "Autocomplete", + extraInfo: + "Page Url: https://www.cnbc.com/2024/05/23/nvidia-keeps-hitting-records-can-investors-still-buy-the-stock.html?&qsearchterm=nvidia", + }, +]; diff --git a/apps/web/app/(canvas)/canvas/layout.tsx b/apps/web/app/(canvas)/canvas/layout.tsx new file mode 100644 index 00000000..9bc3b6d7 --- /dev/null +++ b/apps/web/app/(canvas)/canvas/layout.tsx @@ -0,0 +1,13 @@ +import "../canvasStyles.css"; + +export default function RootLayout({ + children, +}: { + children: React.ReactNode; +}) { + return ( +
+
{children}
+
+ ); +} diff --git a/apps/web/app/(canvas)/canvas/page.tsx b/apps/web/app/(canvas)/canvas/page.tsx new file mode 100644 index 00000000..8b5252af --- /dev/null +++ b/apps/web/app/(canvas)/canvas/page.tsx @@ -0,0 +1,9 @@ +import { redirect } from "next/navigation"; +import React from "react"; + +function page() { + redirect("/signin"); + return
page
; +} + +export default page; diff --git a/apps/web/app/(canvas)/canvasStyles.css b/apps/web/app/(canvas)/canvasStyles.css new file mode 100644 index 00000000..04da2054 --- /dev/null +++ b/apps/web/app/(canvas)/canvasStyles.css @@ -0,0 +1,28 @@ +.tl-background { + background: #1F2428 !important; +} + +.tlui-style-panel.tlui-style-panel__wrapper, .tlui-navigation-panel::before ,.tlui-menu-zone, .tlui-toolbar__tools, .tlui-popover__content, .tlui-menu, .tlui-button__help, .tlui-help-menu, .tlui-dialog__content { + background: #2C3439 !important; + border-top: #2C3439 !important; + border-right: #2C3439 !important; + border-bottom: #2C3439 !important; + border-left: #2C3439 !important; +} + +.tlui-navigation-panel::before { + border-top: #2C3439 !important; + border-right: #2C3439 !important; +} + +.tlui-minimap { + background: #2C3439 !important; +} + +.tlui-minimap__canvas { + background: #1F2428 !important; +} + +.tlui-dialog__overlay { + position: fixed; +} \ No newline at end of file diff --git a/apps/web/app/(dash)/actions.ts b/apps/web/app/(dash)/actions.ts deleted file mode 100644 index 70c2a567..00000000 --- a/apps/web/app/(dash)/actions.ts +++ /dev/null @@ -1,48 +0,0 @@ -"use server"; - -import { cookies, headers } from "next/headers"; -import { db } from "../helpers/server/db"; -import { sessions, users, space } from "../helpers/server/db/schema"; -import { eq } from "drizzle-orm"; -import { redirect } from "next/navigation"; - -export async function ensureAuth() { - const token = - cookies().get("next-auth.session-token")?.value ?? - cookies().get("__Secure-authjs.session-token")?.value ?? - cookies().get("authjs.session-token")?.value ?? - headers().get("Authorization")?.replace("Bearer ", ""); - - if (!token) { - return undefined; - } - - const sessionData = await db - .select() - .from(sessions) - .innerJoin(users, eq(users.id, sessions.userId)) - .where(eq(sessions.sessionToken, token)); - - if (!sessionData || sessionData.length < 0) { - return undefined; - } - - return { - user: sessionData[0]!.user, - session: sessionData[0]!, - }; -} - -export async function getSpaces() { - const data = await ensureAuth(); - if (!data) { - redirect("/signin"); - } - - const sp = await db - .select() - .from(space) - .where(eq(space.user, data.user.email)); - - return sp; -} diff --git a/apps/web/app/(dash)/chat/CodeBlock.tsx b/apps/web/app/(dash)/chat/CodeBlock.tsx new file mode 100644 index 00000000..0bb6a19d --- /dev/null +++ b/apps/web/app/(dash)/chat/CodeBlock.tsx @@ -0,0 +1,90 @@ +import React, { useRef, useState } from "react"; + +const CodeBlock = ({ + lang, + codeChildren, +}: { + lang: string; + codeChildren: React.ReactNode & React.ReactNode[]; +}) => { + const codeRef = useRef(null); + + return ( +
+ +
+ + {codeChildren} + +
+
+ ); +}; + +const CodeBar = React.memo( + ({ + lang, + codeRef, + }: { + lang: string; + codeRef: React.RefObject; + }) => { + const [isCopied, setIsCopied] = useState(false); + return ( +
+ {lang} + +
+ ); + }, +); +export default CodeBlock; diff --git a/apps/web/app/(dash)/chat/[chatid]/page.tsx b/apps/web/app/(dash)/chat/[chatid]/page.tsx new file mode 100644 index 00000000..e37ae07e --- /dev/null +++ b/apps/web/app/(dash)/chat/[chatid]/page.tsx @@ -0,0 +1,38 @@ +import { getFullChatThread } from "@/app/actions/fetchers"; +import { chatSearchParamsCache } from "@/lib/searchParams"; +import ChatWindow from "../chatWindow"; + +async function Page({ + params, + searchParams, +}: { + params: { chatid: string }; + searchParams: Record; +}) { + const { firstTime, q, spaces } = chatSearchParamsCache.parse(searchParams); + + let chat: Awaited>; + + try { + chat = await getFullChatThread(params.chatid); + } catch (e) { + const error = e as Error; + return
This page errored out: {error.message}
; + } + + if (!chat.success || !chat.data) { + console.error(chat.error); + return
Chat not found. Check the console for more details.
; + } + + return ( + 0 ? chat.data : undefined} + threadId={params.chatid} + /> + ); +} + +export default Page; diff --git a/apps/web/app/(dash)/chat/actions.ts b/apps/web/app/(dash)/chat/actions.ts deleted file mode 100644 index 908fe79e..00000000 --- a/apps/web/app/(dash)/chat/actions.ts +++ /dev/null @@ -1 +0,0 @@ -"use server"; diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index 43c337ee..9a18cfe7 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -1,51 +1,438 @@ "use client"; import { AnimatePresence } from "framer-motion"; -import React, { useEffect, useState } from "react"; +import React, { useEffect, useRef, useState } from "react"; import QueryInput from "../home/queryinput"; import { cn } from "@repo/ui/lib/utils"; import { motion } from "framer-motion"; import { useRouter } from "next/navigation"; +import { ChatHistory, sourcesZod } from "@repo/shared-types"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@repo/ui/shadcn/accordion"; +import Markdown from "react-markdown"; +import remarkGfm from "remark-gfm"; +import remarkMath from "remark-math"; +import rehypeKatex from "rehype-katex"; +import rehypeHighlight from "rehype-highlight"; +import { code, p } from "./markdownRenderHelpers"; +import { codeLanguageSubset } from "@/lib/constants"; +import { toast } from "sonner"; +import Link from "next/link"; +import { createChatObject } from "@/app/actions/doers"; +import { ClipboardIcon } from "@heroicons/react/24/outline"; +import { SendIcon } from "lucide-react"; -function ChatWindow({ q }: { q: string }) { - const [layout, setLayout] = useState<"chat" | "initial">("initial"); +function ChatWindow({ + q, + spaces, + initialChat = [ + { + question: q, + answer: { + parts: [], + sources: [], + }, + }, + ], + threadId, +}: { + q: string; + spaces: { id: string; name: string }[]; + initialChat?: ChatHistory[]; + threadId: string; +}) { + const [layout, setLayout] = useState<"chat" | "initial">( + initialChat.length > 1 ? "chat" : "initial", + ); + const [chatHistory, setChatHistory] = useState(initialChat); + + const removeJustificationFromText = (text: string) => { + // remove everything after the first "" word + const justificationLine = text.indexOf(""); + if (justificationLine !== -1) { + // Add that justification to the last chat message + const lastChatMessage = chatHistory[chatHistory.length - 1]; + if (lastChatMessage) { + lastChatMessage.answer.justification = text.slice(justificationLine); + } + return text.slice(0, justificationLine); + } + return text; + }; const router = useRouter(); + const getAnswer = async (query: string, spaces: string[]) => { + const sourcesFetch = await fetch( + `/api/chat?q=${query}&spaces=${spaces}&sourcesOnly=true&threadId=${threadId}`, + { + method: "POST", + body: JSON.stringify({ chatHistory }), + }, + ); + + // TODO: handle this properly + const sources = await sourcesFetch.json(); + + const sourcesParsed = sourcesZod.safeParse(sources); + + if (!sourcesParsed.success) { + console.error(sourcesParsed.error); + toast.error("Something went wrong while getting the sources"); + return; + } + window.scrollTo({ + top: document.documentElement.scrollHeight, + behavior: "smooth", + }); + + const updateChatHistoryAndFetch = async () => { + // Step 1: Update chat history with the assistant's response + await new Promise((resolve) => { + setChatHistory((prevChatHistory) => { + const newChatHistory = [...prevChatHistory]; + const lastAnswer = newChatHistory[newChatHistory.length - 1]; + if (!lastAnswer) { + resolve(undefined); + return prevChatHistory; + } + + const filteredSourceUrls = new Set( + sourcesParsed.data.metadata.map((source) => source.url), + ); + const uniqueSources = sourcesParsed.data.metadata.filter((source) => { + if (filteredSourceUrls.has(source.url)) { + filteredSourceUrls.delete(source.url); + return true; + } + return false; + }); + + lastAnswer.answer.sources = uniqueSources.map((source) => ({ + title: source.title ?? "Untitled", + type: source.type ?? "page", + source: source.url ?? "https://supermemory.ai", + content: source.description ?? "No content available", + numChunks: sourcesParsed.data.metadata.filter( + (f) => f.url === source.url, + ).length, + })); + + resolve(newChatHistory); + return newChatHistory; + }); + }); + + // Step 2: Fetch data from the API + const resp = await fetch( + `/api/chat?q=${query}&spaces=${spaces}&threadId=${threadId}`, + { + method: "POST", + body: JSON.stringify({ chatHistory, sources: sourcesParsed.data }), + }, + ); + + // Step 3: Read the response stream and update the chat history + const reader = resp.body?.getReader(); + let done = false; + while (!done && reader) { + const { value, done: d } = await reader.read(); + if (d) { + setChatHistory((prevChatHistory) => { + createChatObject(threadId, prevChatHistory); + return prevChatHistory; + }); + } + done = d; + + const txt = new TextDecoder().decode(value); + setChatHistory((prevChatHistory) => { + const newChatHistory = [...prevChatHistory]; + const lastAnswer = newChatHistory[newChatHistory.length - 1]; + if (!lastAnswer) return prevChatHistory; + + window.scrollTo({ + top: document.documentElement.scrollHeight, + behavior: "smooth", + }); + + lastAnswer.answer.parts.push({ text: txt }); + return newChatHistory; + }); + } + }; + + updateChatHistoryAndFetch(); + }; + useEffect(() => { - if (q !== "") { - setTimeout(() => { - setLayout("chat"); - }, 300); + if (q.trim().length > 0 || chatHistory.length > 0) { + setLayout("chat"); + const lastChat = chatHistory.length > 0 ? chatHistory.length - 1 : 0; + const startGenerating = chatHistory[lastChat]?.answer.parts[0]?.text + ? false + : true; + if (startGenerating) { + getAnswer( + q, + spaces.map((s) => `${s}`), + ); + } } else { router.push("/home"); } - }, [q]); + }, []); + return ( -
+
{layout === "initial" ? (
- + {}} + initialQuery={q} + initialSpaces={[]} + disabled + />
) : (
-

- {q} -

+
+ {chatHistory.map((chat, idx) => ( +
+
+

+ {chat.question} +

+ +
+ {/* Related memories */} +
0 || chat.answer.parts.length === 0 ? "flex" : "hidden"}`} + > + + + + Related Memories + + {/* TODO: fade out content on the right side, the fade goes away when the user scrolls */} + + {/* Loading state */} + {chat.answer.sources.length > 0 || + (chat.answer.parts.length === 0 && ( + <> + {[1, 2, 3, 4].map((_, idx) => ( +
+
+
+
+ ))} + + ))} + {chat.answer.sources.map((source, idx) => ( + +
+ {source.type} + + {source.numChunks > 1 && ( + {source.numChunks} chunks + )} +
+
+ {source.title} +
+
+ {source.content.length > 100 + ? source.content.slice(0, 100) + "..." + : source.content} +
+ + ))} + + + +
+ + {/* Summary */} +
+
Summary
+
+ {/* Loading state */} + {(chat.answer.parts.length === 0 || + chat.answer.parts.join("").length === 0) && ( +
+
+
+
+
+
+ )} + + + {removeJustificationFromText( + chat.answer.parts + .map((part) => part.text) + .join(""), + )} + + +
+ {/* TODO: speak response */} + {/* */} + {/* copy response */} + + +
+
+
+ {/* Justification */} + {chat.answer.justification && + chat.answer.justification.length && ( +
0 ? "flex" : "hidden"}`} + > + + + + Justification + + + {chat.answer.justification.length > 0 + ? chat.answer.justification + .replaceAll("", "") + .replaceAll("", "") + : "No justification provided."} + + + +
+ )} +
+
+
+ ))} +
+ +
+ { + setChatHistory((prevChatHistory) => { + return [ + ...prevChatHistory, + { + question: q, + answer: { + parts: [], + sources: [], + }, + }, + ]; + }); + await getAnswer( + q, + spaces.map((s) => `${s.id}`), + ); + }} + /> +
)} diff --git a/apps/web/app/(dash)/chat/markdownRenderHelpers.tsx b/apps/web/app/(dash)/chat/markdownRenderHelpers.tsx new file mode 100644 index 00000000..747d4fca --- /dev/null +++ b/apps/web/app/(dash)/chat/markdownRenderHelpers.tsx @@ -0,0 +1,25 @@ +import { DetailedHTMLProps, HTMLAttributes, memo } from "react"; +import { ExtraProps } from "react-markdown"; +import CodeBlock from "./CodeBlock"; + +export const code = memo((props: JSX.IntrinsicElements["code"]) => { + const { className, children } = props; + const match = /language-(\w+)/.exec(className || ""); + const lang = match && match[1]; + + return ; +}); + +export const p = memo( + ( + props?: Omit< + DetailedHTMLProps< + HTMLAttributes, + HTMLParagraphElement + >, + "ref" + >, + ) => { + return

{props?.children}

; + }, +); diff --git a/apps/web/app/(dash)/chat/page.tsx b/apps/web/app/(dash)/chat/page.tsx deleted file mode 100644 index 9e28fda7..00000000 --- a/apps/web/app/(dash)/chat/page.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import ChatWindow from "./chatWindow"; -import { chatSearchParamsCache } from "../../helpers/lib/searchParams"; - -function Page({ - searchParams, -}: { - searchParams: Record; -}) { - const { firstTime, q, spaces } = chatSearchParamsCache.parse(searchParams); - - console.log(spaces); - - return ; -} - -export default Page; diff --git a/apps/web/app/(dash)/dynamicisland.tsx b/apps/web/app/(dash)/dynamicisland.tsx new file mode 100644 index 00000000..8b1b4633 --- /dev/null +++ b/apps/web/app/(dash)/dynamicisland.tsx @@ -0,0 +1,315 @@ +"use client"; + +import { AddIcon } from "@repo/ui/icons"; +import Image from "next/image"; + +import { AnimatePresence, useMotionValueEvent, useScroll } from "framer-motion"; +import { useActionState, useEffect, useRef, useState } from "react"; +import { motion } from "framer-motion"; +import { Label } from "@repo/ui/shadcn/label"; +import { Input } from "@repo/ui/shadcn/input"; +import { Textarea } from "@repo/ui/shadcn/textarea"; +import { createMemory, createSpace } from "../actions/doers"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@repo/ui/shadcn/select"; +import { Space } from "../actions/types"; +import { getSpaces } from "../actions/fetchers"; +import { toast } from "sonner"; +import { useFormStatus } from "react-dom"; + +export function DynamicIsland() { + const { scrollYProgress } = useScroll(); + const [visible, setVisible] = useState(true); + + useMotionValueEvent(scrollYProgress, "change", (current) => { + if (typeof current === "number") { + let direction = current! - scrollYProgress.getPrevious()!; + + if (direction < 0 || direction === 1) { + setVisible(true); + } else { + setVisible(false); + } + } + }); + + return ( +
+ + + + + +
+ ); +} + +export default DynamicIsland; + +function DynamicIslandContent() { + const [show, setshow] = useState(true); + function cancelfn() { + setshow(true); + } + + const lastBtn = useRef(); + + useEffect(() => { + document.addEventListener("keydown", (e) => { + if (e.key === "Escape") { + setshow(true); + } + + if (e.key === "a" && lastBtn.current === "Alt") { + setshow(false); + } + lastBtn.current = e.key; + }); + }, []); + return ( + <> + {show ? ( + + ) : ( + + )} + + ); +} + +const fakeitems = ["page", "spaces"]; + +function ToolBar({ cancelfn }: { cancelfn: () => void }) { + const [spaces, setSpaces] = useState([]); + + const [index, setIndex] = useState(0); + + useEffect(() => { + (async () => { + let spaces = await getSpaces(); + + if (!spaces.success || !spaces.data) { + toast.warning("Unable to get spaces", { + richColors: true, + }); + setSpaces([]); + return; + } + setSpaces(spaces.data); + })(); + }, []); + + return ( + + +
+ setIndex(i)} + /> +
+ {index === 1 ? ( + + ) : ( + + )} + + + ); +} + +export const HoverEffect = ({ + items, + index, + indexFn, +}: { + items: string[]; + index: number; + indexFn: (i: number) => void; +}) => { + return ( +
+ {items.map((item, idx) => ( + + ))} +
+ ); +}; + +function SpaceForm({ cancelfn }: { cancelfn: () => void }) { + return ( +
+
+ + +
+
+ {/* + pull from store + */} + {/*
+ cancel +
*/} + +
+
+ ); +} + +function PageForm({ + cancelfn, + spaces, +}: { + cancelfn: () => void; + spaces: Space[]; +}) { + const [loading, setLoading] = useState(false); + + const { pending } = useFormStatus(); + return ( +
{ + const content = e.get("content")?.toString(); + const space = e.get("space")?.toString(); + + toast.info("Creating memory..."); + + if (!content) { + toast.error("Content is required"); + return; + } + cancelfn(); + const cont = await createMemory({ + content: content, + spaces: space ? [space] : undefined, + }); + + if (cont.success) { + toast.success("Memory created"); + } else { + toast.error("Memory creation failed"); + } + }} + className="bg-secondary border border-muted-foreground px-4 py-3 rounded-2xl mt-2 flex flex-col gap-3 w-[100vw] md:w-[400px]" + > +
+ + +
+
+ +