diff --git a/SETUP-GUIDE.md b/SETUP-GUIDE.md index 7d69b545..f51b1cb1 100644 --- a/SETUP-GUIDE.md +++ b/SETUP-GUIDE.md @@ -13,12 +13,13 @@ 3. Create a `.dev.vars` file in `apps/web` with the following content: ```bash -GOOGLE_CLIENT_ID="-" -GOOGLE_CLIENT_SECRET="-" +GOOGLE_CLIENT_ID="-" // required, visit https://developers.google.com/identity/protocols/oauth2 +GOOGLE_CLIENT_SECRET="-" // required NEXTAUTH_SECRET='nextauthsecret' DATABASE_URL='database.sqlite' NEXTAUTH_URL='http://localhost:3000' BACKEND_SECURITY_KEY='veryrandomsecuritykey' +BACKEND_BASE_URL="where your backend is hosted" ``` 4. Setup the database: @@ -28,10 +29,10 @@ First, edit the `wrangler.toml` file in `apps/web` to point the d1 database to y You can create a d1 database by running this command ``` -wrangler d1 create DATABASE_NAME +bunx wrangler d1 create ``` -And then replace these values +And then replace database_name and database_id with the values ``` [[d1_databases]] @@ -43,10 +44,12 @@ database_id = "YOUR_DB_ID" Simply run this command in `apps/web` ``` -wrangler d1 execute dev-d1-anycontext --local --file=db/prepare.sql +bunx wrangler d1 migrations apply ``` -If it runs, you can set up the cloud database as well by removing the `--local` flag. +If it runs, you can set up the cloud database as well by removing the `--local` flag, + +if you just want to contribute to frontend then just run `bun run dev` in the root of the project and done! (you won't be able to try ai stuff), otherwise continue... 5. You need to host your own worker for the `apps/cf-ai-backend` module. diff --git a/apps/browser-rendering b/apps/browser-rendering index b37c9623..4d21045a 160000 --- a/apps/browser-rendering +++ b/apps/browser-rendering @@ -1 +1 @@ -Subproject commit b37c962365a36cf342a31a196f4908f4f1343553 +Subproject commit 4d21045a45fdbf56b7483d3704ae0474ebf044fb diff --git a/apps/cf-ai-backend/package.json b/apps/cf-ai-backend/package.json index 480f9601..78353e08 100644 --- a/apps/cf-ai-backend/package.json +++ b/apps/cf-ai-backend/package.json @@ -6,7 +6,7 @@ "scripts": { "test": "jest --verbose", "deploy": "wrangler deploy", - "dev": "wrangler dev", + "dev": "wrangler dev --remote --port 8686", "start": "wrangler dev", "unsafe-reset-vector-db": "wrangler vectorize delete supermem-vector && wrangler vectorize create --dimensions=1536 supermem-vector-1 --metric=cosine" }, diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 87495c59..cef781be 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -21,8 +21,6 @@ export async function initQuery( index: c.env.VECTORIZE_INDEX, }); - const DEFAULT_MODEL = "gpt-4o"; - let selectedModel: | ReturnType> | ReturnType> @@ -52,12 +50,6 @@ export async function initQuery( break; } - if (!selectedModel) { - throw new Error( - `Model ${model} not found and default model ${DEFAULT_MODEL} is also not available.`, - ); - } - return { store, model: selectedModel }; } @@ -72,19 +64,46 @@ export async function deleteDocument({ c: Context<{ Bindings: Env }>; store: CloudflareVectorizeStore; }) { - const toBeDeleted = `${url}-${user}`; + const toBeDeleted = `${url}#supermemory-web`; const random = seededRandom(toBeDeleted); const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15); - await c.env.KV.list({ prefix: uuid }).then(async (keys) => { - for (const key of keys.keys) { - await c.env.KV.delete(key.name); - await store.delete({ ids: [key.name] }); + const allIds = await c.env.KV.list({ prefix: uuid }); + + if (allIds.keys.length > 0) { + const savedVectorIds = allIds.keys.map((key) => key.name); + const vectors = await c.env.VECTORIZE_INDEX.getByIds(savedVectorIds); + // We don't actually delete document directly, we just remove the user from the metadata. + // If there's no user left, we can delete the document. + const newVectors = vectors.map((vector) => { + delete vector.metadata[`user-${user}`]; + + // Get count of how many users are left + const userCount = Object.keys(vector.metadata).filter((key) => + key.startsWith("user-"), + ).length; + + // If there's no user left, we can delete the document. + // need to make sure that every chunk is deleted otherwise it would be problematic. + if (userCount === 0) { + store.delete({ ids: savedVectorIds }); + void Promise.all(savedVectorIds.map((id) => c.env.KV.delete(id))); + return null; + } + + return vector; + }); + + // If all vectors are null (deleted), we can delete the KV too. Otherwise, we update (upsert) the vectors. + if (newVectors.every((v) => v === null)) { + await c.env.KV.delete(uuid); + } else { + await c.env.VECTORIZE_INDEX.upsert(newVectors.filter((v) => v !== null)); } - }); + } } export async function batchCreateChunksAndEmbeddings({ @@ -98,19 +117,47 @@ export async function batchCreateChunksAndEmbeddings({ chunks: string[]; context: Context<{ Bindings: Env }>; }) { - const ourID = `${body.url}-${body.user}`; - - await deleteDocument({ url: body.url, user: body.user, c: context, store }); - + //! NOTE that we use #supermemory-web to ensure that + //! If a user saves it through the extension, we don't want other users to be able to see it. + // Requests from the extension should ALWAYS have a unique ID with the USERiD in it. + // I cannot stress this enough, important for security. + const ourID = `${body.url}#supermemory-web`; const random = seededRandom(ourID); + const uuid = + random().toString(36).substring(2, 15) + + random().toString(36).substring(2, 15); + + const allIds = await context.env.KV.list({ prefix: uuid }); + + // If some chunks for that content already exist, we'll just update the metadata to include + // the user. + if (allIds.keys.length > 0) { + const savedVectorIds = allIds.keys.map((key) => key.name); + const vectors = await context.env.VECTORIZE_INDEX.getByIds(savedVectorIds); + + // Now, we'll update all vector metadatas with one more userId and all spaceIds + const newVectors = vectors.map((vector) => { + vector.metadata = { + ...vector.metadata, + [`user-${body.user}`]: 1, + + // For each space in body, add the spaceId to the vector metadata + ...(body.spaces ?? [])?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}), + }; + + return vector; + }); + + await context.env.VECTORIZE_INDEX.upsert(newVectors); + return; + } for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; - const uuid = - random().toString(36).substring(2, 15) + - random().toString(36).substring(2, 15) + - "-" + - i; + const chunkId = `${uuid}-${i}`; const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${chunk}`; @@ -121,19 +168,25 @@ export async function batchCreateChunksAndEmbeddings({ metadata: { title: body.title?.slice(0, 50) ?? "", description: body.description ?? "", - space: body.space ?? "", url: body.url, - user: body.user, + type: body.type ?? "page", + content: newPageContent, + + [`user-${body.user}`]: 1, + ...body.spaces?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}), }, }, ], { - ids: [uuid], + ids: [chunkId], }, ); console.log("Docs added: ", docs); - await context.env.KV.put(uuid, ourID); + await context.env.KV.put(chunkId, ourID); } } diff --git a/apps/cf-ai-backend/src/index.test.ts b/apps/cf-ai-backend/src/index.test.ts deleted file mode 100644 index bbf66fb5..00000000 --- a/apps/cf-ai-backend/src/index.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import app from "."; - -// TODO: write more tests -describe("Test the application", () => { - it("Should return 200 response", async () => { - const res = await app.request("http://localhost/"); - expect(res.status).toBe(200); - }), - it("Should return 404 response", async () => { - const res = await app.request("http://localhost/404"); - expect(res.status).toBe(404); - }); -}); diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 2dbb2d0c..e89d170c 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { Hono } from "hono"; -import { CoreMessage, streamText } from "ai"; +import { CoreMessage, generateText, streamText } from "ai"; import { chatObj, Env, vectorObj } from "./types"; import { batchCreateChunksAndEmbeddings, @@ -18,7 +18,12 @@ import { swaggerUI } from "@hono/swagger-ui"; const app = new Hono<{ Bindings: Env }>(); -app.get("/doc", swaggerUI({ url: "/doc" })); +app.get( + "/ui", + swaggerUI({ + url: "/doc", + }), +); // ------- MIDDLEWARES ------- app.use("*", poweredBy()); @@ -34,6 +39,17 @@ app.use("/api/", async (c, next) => { }); // ------- MIDDLEWARES END ------- +const fileSchema = z + .instanceof(File) + .refine( + (file) => file.size <= 10 * 1024 * 1024, + "File size should be less than 10MB", + ) // Validate file size + .refine( + (file) => ["image/jpeg", "image/png", "image/gif"].includes(file.type), + "Invalid file type", + ); // Validate file type + app.get("/", (c) => { return c.text("Supermemory backend API is running!"); }); @@ -57,6 +73,82 @@ app.post("/api/add", zValidator("json", vectorObj), async (c) => { return c.json({ status: "ok" }); }); +app.post( + "/api/add-with-image", + zValidator( + "form", + z.object({ + images: z + .array(fileSchema) + .min(1, "At least one image is required") + .optional(), + "images[]": z + .array(fileSchema) + .min(1, "At least one image is required") + .optional(), + text: z.string().optional(), + spaces: z.array(z.string()).optional(), + url: z.string(), + user: z.string(), + }), + (c) => { + console.log(c); + }, + ), + async (c) => { + const body = c.req.valid("form"); + + const { store } = await initQuery(c); + + if (!(body.images || body["images[]"])) { + return c.json({ status: "error", message: "No images found" }, 400); + } + + const imagePromises = (body.images ?? body["images[]"]).map( + async (image) => { + const buffer = await image.arrayBuffer(); + const input = { + image: [...new Uint8Array(buffer)], + prompt: + "What's in this image? caption everything you see in great detail. If it has text, do an OCR and extract all of it.", + max_tokens: 1024, + }; + const response = await c.env.AI.run( + "@cf/llava-hf/llava-1.5-7b-hf", + input, + ); + console.log(response.description); + return response.description; + }, + ); + + const imageDescriptions = await Promise.all(imagePromises); + + await batchCreateChunksAndEmbeddings({ + store, + body: { + url: body.url, + user: body.user, + type: "image", + description: + imageDescriptions.length > 1 + ? `A group of ${imageDescriptions.length} images on ${body.url}` + : imageDescriptions[0], + spaces: body.spaces, + pageContent: imageDescriptions.join("\n"), + title: "Image content from the web", + }, + chunks: [ + imageDescriptions, + ...(body.text ? chunkText(body.text, 1536) : []), + ].flat(), + context: c, + }); + + return c.json({ status: "ok" }); + }, +); + app.get( "/api/ask", zValidator( @@ -101,12 +193,14 @@ app.post( const body = c.req.valid("json"); const sourcesOnly = query.sourcesOnly === "true"; - const spaces = query.spaces?.split(",") ?? [""]; + const spaces = query.spaces?.split(",") ?? [undefined]; // Get the AI model maker and vector store const { model, store } = await initQuery(c, query.model); - const filter: VectorizeVectorMetadataFilter = { user: query.user }; + const filter: VectorizeVectorMetadataFilter = { + [`user-${query.user}`]: 1, + }; console.log("Spaces", spaces); // Converting the query to a vector so that we can search for similar vectors @@ -118,9 +212,9 @@ app.post( // SLICED to 5 to avoid too many queries for (const space of spaces.slice(0, 5)) { console.log("space", space); - if (space !== "") { + if (!space && spaces.length > 1) { // it's possible for space list to be [undefined] so we only add space filter conditionally - filter.space = space; + filter[`space-${query.user}-${space}`] = 1; } // Because there's no OR operator in the filter, we have to make multiple queries @@ -173,29 +267,20 @@ app.post( dataPoint.id.toString(), ); - // We are getting the content ID back, so that the frontend can show the actual sources properly. - // it IS a lot of DB calls, i completely agree. - // TODO: return metadata value here, so that the frontend doesn't have to re-fetch anything. const storedContent = await Promise.all( idsAsStrings.map(async (id) => await c.env.KV.get(id)), ); - return c.json({ ids: storedContent }); + const metadata = normalizedData.map((datapoint) => datapoint.metadata); + + return c.json({ ids: storedContent, metadata }); } - const vec = responses.matches.map((data) => ({ metadata: data.metadata })); - - const vecWithScores = vec.map((v, i) => ({ - ...v, - score: sortedHighScoreData[i].score, - normalisedScore: sortedHighScoreData[i].normalizedScore, - })); - - const preparedContext = vecWithScores.map( - ({ metadata, score, normalisedScore }) => ({ + const preparedContext = normalizedData.map( + ({ metadata, score, normalizedScore }) => ({ context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, score, - normalisedScore, + normalizedScore, }), ); @@ -245,4 +330,20 @@ app.delete( }, ); +// ERROR #1 - this is the api that the editor uses, it is just a scrape off of /api/chat so you may check that out +app.get('/api/editorai', zValidator( + "query", + z.object({ + context: z.string(), + request: z.string(), + }), +), async (c)=> { + const { context, request } = c.req.valid("query"); + const { model } = await initQuery(c); + + const response = await streamText({ model, prompt: `${request}-${context}`, maxTokens: 224 }); + + return response.toTextStreamResponse(); +}) + export default app; diff --git a/apps/cf-ai-backend/src/prompts/prompt1.ts b/apps/cf-ai-backend/src/prompts/prompt1.ts index d2ee988c..289495b6 100644 --- a/apps/cf-ai-backend/src/prompts/prompt1.ts +++ b/apps/cf-ai-backend/src/prompts/prompt1.ts @@ -18,13 +18,12 @@ export const template = ({ contexts, question }) => { // Map over contexts to generate the context and score parts const contextParts = contexts .map( - ({ context, score, normalisedScore }) => ` + ({ context, normalisedScore }) => ` ${context} - score: ${score} normalisedScore: ${normalisedScore} `, ) diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index bea4bf80..417d6320 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -2,7 +2,7 @@ import { z } from "zod"; export type Env = { VECTORIZE_INDEX: VectorizeIndex; - AI: Fetcher; + AI: Ai; SECURITY_KEY: string; OPENAI_API_KEY: string; GOOGLE_AI_API_KEY: string; @@ -43,7 +43,8 @@ export const vectorObj = z.object({ pageContent: z.string(), title: z.string().optional(), description: z.string().optional(), - space: z.string().optional(), + spaces: z.array(z.string()).optional(), url: z.string(), user: z.string(), + type: z.string().optional().default("page"), }); diff --git a/apps/cf-ai-backend/src/utils/chonker.ts b/apps/cf-ai-backend/src/utils/chonker.ts index 39d4b458..c63020be 100644 --- a/apps/cf-ai-backend/src/utils/chonker.ts +++ b/apps/cf-ai-backend/src/utils/chonker.ts @@ -1,5 +1,8 @@ import nlp from "compromise"; +/** + * Split text into chunks of specified max size with some overlap for continuity. + */ export default function chunkText( text: string, maxChunkSize: number, diff --git a/apps/cf-ai-backend/src/utils/seededRandom.ts b/apps/cf-ai-backend/src/utils/seededRandom.ts index 36a1e4f9..9e315ee8 100644 --- a/apps/cf-ai-backend/src/utils/seededRandom.ts +++ b/apps/cf-ai-backend/src/utils/seededRandom.ts @@ -1,5 +1,9 @@ import { MersenneTwister19937, integer } from "random-js"; +/** + * Hashes a string to a 32-bit integer. + * @param {string} seed - The input string to hash. + */ function hashString(seed: string) { let hash = 0; for (let i = 0; i < seed.length; i++) { @@ -10,6 +14,9 @@ function hashString(seed: string) { return hash; } +/** + * returns a funtion that generates same sequence of random numbers for a given seed between 0 and 1. + */ export function seededRandom(seed: string) { const seedHash = hashString(seed); const engine = MersenneTwister19937.seed(seedHash); diff --git a/apps/cf-ai-backend/tsconfig.json b/apps/cf-ai-backend/tsconfig.json index 2b75d5a0..fcdf6914 100644 --- a/apps/cf-ai-backend/tsconfig.json +++ b/apps/cf-ai-backend/tsconfig.json @@ -1,6 +1,7 @@ { "compilerOptions": { "lib": ["ES2020"], - "types": ["@cloudflare/workers-types"] + "types": ["@cloudflare/workers-types"], + "downlevelIteration": true } } diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index db0ae945..fa883195 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -5,7 +5,7 @@ node_compat = true [[vectorize]] binding = "VECTORIZE_INDEX" -index_name = "supermem-vector" +index_name = "supermem-vector-dev" [ai] binding = "AI" diff --git a/apps/web/app/(auth)/auth-buttons.tsx b/apps/web/app/(auth)/auth-buttons.tsx index 0e99213e..5b0ad06e 100644 --- a/apps/web/app/(auth)/auth-buttons.tsx +++ b/apps/web/app/(auth)/auth-buttons.tsx @@ -2,7 +2,7 @@ import { Button } from "@repo/ui/shadcn/button"; import React from "react"; -import { signIn } from "../helpers/server/auth"; +import { signIn } from "../../server/auth"; function SignIn() { return ( diff --git a/apps/web/app/(auth)/signin/page.tsx b/apps/web/app/(auth)/signin/page.tsx index ba84a94a..d7bad8da 100644 --- a/apps/web/app/(auth)/signin/page.tsx +++ b/apps/web/app/(auth)/signin/page.tsx @@ -1,7 +1,7 @@ import Image from "next/image"; import Link from "next/link"; import Logo from "@/public/logo.svg"; -import { signIn } from "@/app/helpers/server/auth"; +import { signIn } from "@/server/auth"; import { Google } from "@repo/ui/components/icons"; export const runtime = "edge"; diff --git a/apps/web/app/(canvas)/canvas.tsx b/apps/web/app/(canvas)/canvas.tsx index 9ec57d6d..498ab1eb 100644 --- a/apps/web/app/(canvas)/canvas.tsx +++ b/apps/web/app/(canvas)/canvas.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Editor, Tldraw, setUserPreferences, TLStoreWithStatus } from "tldraw"; import { createAssetFromUrl } from "./lib/createAssetUrl"; import "tldraw/tldraw.css"; @@ -7,10 +7,53 @@ import { twitterCardUtil } from "./twitterCard"; import createEmbedsFromUrl from "./lib/createEmbeds"; import { loadRemoteSnapshot } from "./lib/loadSnap"; import { SaveStatus } from "./savesnap"; -import { getAssetUrls } from '@tldraw/assets/selfHosted' -import { memo } from 'react'; +import { getAssetUrls } from "@tldraw/assets/selfHosted"; +import { memo } from "react"; +import DragContext from "./lib/context"; +import DropZone from "./dropComponent"; -export const Canvas = memo(()=>{ +export const Canvas = memo(() => { + const [isDraggingOver, setIsDraggingOver] = useState(false); + const Dragref = useRef(null) + + const handleDragOver = (event: any) => { + event.preventDefault(); + setIsDraggingOver(true); + console.log("entere") + }; + + const handleDragLeave = () => { + setIsDraggingOver(false); + console.log("leaver") + }; + + useEffect(() => { + const divElement = Dragref.current; + if (divElement) { + divElement.addEventListener('dragover', handleDragOver); + divElement.addEventListener('dragleave', handleDragLeave); + } + return () => { + if (divElement) { + divElement.removeEventListener('dragover', handleDragOver); + divElement.removeEventListener('dragleave', handleDragLeave); + } + }; + }, []); + + return ( + +
+ +
+ + ); +}); + +const TldrawComponent =memo(() => { const [storeWithStatus, setStoreWithStatus] = useState({ status: "loading", }); @@ -38,18 +81,22 @@ export const Canvas = memo(()=>{ setUserPreferences({ id: "supermemory", isDarkMode: true }); - const assetUrls = getAssetUrls() + const assetUrls = getAssetUrls(); return ( - -
- -
- +
+ +
+ +
+ + +
); }) diff --git a/apps/web/app/(canvas)/canvas/page.tsx b/apps/web/app/(canvas)/canvas/page.tsx index 7abfa583..366a4481 100644 --- a/apps/web/app/(canvas)/canvas/page.tsx +++ b/apps/web/app/(canvas)/canvas/page.tsx @@ -18,7 +18,7 @@ function page() { const [fullScreen, setFullScreen] = useState(false); return ( -
+
{setTimeout(()=> setFullScreen(false), 50)}} onCollapse={()=> {setTimeout(()=> setFullScreen(true), 50)}} defaultSize={30} collapsible={true} minSize={22}> diff --git a/apps/web/app/(canvas)/dropComponent.tsx b/apps/web/app/(canvas)/dropComponent.tsx new file mode 100644 index 00000000..03a32358 --- /dev/null +++ b/apps/web/app/(canvas)/dropComponent.tsx @@ -0,0 +1,76 @@ +import React, { useRef, useCallback, useEffect, useContext } from "react"; +import { useEditor } from "tldraw"; +import DragContext, { DragContextType } from "./lib/context"; +import { handleExternalDroppedContent } from "./lib/createEmbeds"; + +const stripHtmlTags = (html: string): string => { + const div = document.createElement("div"); + div.innerHTML = html; + return div.textContent || div.innerText || ""; +}; + +const useDrag = (): DragContextType => { + const context = useContext(DragContext); + if (!context) { + throw new Error('useCounter must be used within a CounterProvider'); + } + return context; +}; + + +function DropZone() { + const dropRef = useRef(null); + const {isDraggingOver, setIsDraggingOver} = useDrag(); + + const editor = useEditor(); + + const handleDrop = useCallback((event: React.DragEvent) => { + event.preventDefault(); + setIsDraggingOver(false); + const dt = event.dataTransfer; + const items = dt.items; + + for (let i = 0; i < items.length; i++) { + if (items[i]!.kind === "file" && items[i]!.type.startsWith("image/")) { + const file = items[i]!.getAsFile(); + if (file) { + const reader = new FileReader(); + reader.onload = (e) => { + if (e.target) { + // setDroppedImage(e.target.result as string); + } + }; + reader.readAsDataURL(file); + } + } else if (items[i]!.kind === "string") { + items[i]!.getAsString((data) => { + const cleanText = stripHtmlTags(data); + handleExternalDroppedContent({editor,text:cleanText}) + }); + } + } + }, []); + + useEffect(() => { + const divElement = dropRef.current; + if (divElement) { + // @ts-ignore + divElement.addEventListener("drop", handleDrop); + } + return () => { + if (divElement) { + // @ts-ignore + divElement.removeEventListener("drop", handleDrop); + } + }; + }, []); + + return ( +
+ ); +} + +export default DropZone; diff --git a/apps/web/app/(canvas)/enabledComp.tsx b/apps/web/app/(canvas)/enabledComp.tsx index 5dbe6ee7..85811b82 100644 --- a/apps/web/app/(canvas)/enabledComp.tsx +++ b/apps/web/app/(canvas)/enabledComp.tsx @@ -7,12 +7,12 @@ export const components: Partial = { TopPanel: null, DebugPanel: null, DebugMenu: null, + PageMenu: null, // Minimap: null, // ContextMenu: null, // HelpMenu: null, // ZoomMenu: null, // StylePanel: null, - // PageMenu: null, // NavigationPanel: null, // Toolbar: null, // KeyboardShortcutsDialog: null, diff --git a/apps/web/app/(canvas)/lib/context.tsx b/apps/web/app/(canvas)/lib/context.tsx new file mode 100644 index 00000000..36a106cf --- /dev/null +++ b/apps/web/app/(canvas)/lib/context.tsx @@ -0,0 +1,11 @@ +import { createContext } from 'react'; + +export interface DragContextType { + isDraggingOver: boolean; + setIsDraggingOver: React.Dispatch>; +} + + +const DragContext = createContext(undefined); + +export default DragContext; \ No newline at end of file diff --git a/apps/web/app/(canvas)/lib/createEmbeds.ts b/apps/web/app/(canvas)/lib/createEmbeds.ts index 322e697e..0db3c71b 100644 --- a/apps/web/app/(canvas)/lib/createEmbeds.ts +++ b/apps/web/app/(canvas)/lib/createEmbeds.ts @@ -2,8 +2,8 @@ import { AssetRecordType, Editor, TLAsset, TLAssetId, TLBookmarkShape, TLExterna export default async function createEmbedsFromUrl({url, point, sources, editor}: { url: string - point: VecLike | undefined - sources: TLExternalContentSource[] | undefined + point?: VecLike | undefined + sources?: TLExternalContentSource[] | undefined editor: Editor }){ @@ -50,10 +50,18 @@ export default async function createEmbedsFromUrl({url, point, sources, editor}: type: "url", url, }); - const fetchWebsite = await (await fetch(`https://unfurl-bookmark.pruthvirajthinks.workers.dev/?url=${url}`)).json() - if (fetchWebsite.title) bookmarkAsset.props.title = fetchWebsite.title; - if (fetchWebsite.image) bookmarkAsset.props.image = fetchWebsite.image; - if (fetchWebsite.description) bookmarkAsset.props.description = fetchWebsite.description; + const fetchWebsite: { + title?: string; + image?: string; + description?: string; + } = await (await fetch(`/api/unfirlsite?website=${url}`, { + method: "POST" + })).json() + if (bookmarkAsset){ + if (fetchWebsite.title) bookmarkAsset.props.title = fetchWebsite.title; + if (fetchWebsite.image) bookmarkAsset.props.image = fetchWebsite.image; + if (fetchWebsite.description) bookmarkAsset.props.description = fetchWebsite.description; + } if (!bookmarkAsset) throw Error("Could not create an asset"); asset = bookmarkAsset; } catch (e) { @@ -79,6 +87,38 @@ export default async function createEmbedsFromUrl({url, point, sources, editor}: }); } +function isURL(str: string) { + try { + new URL(str); + return true; + } catch { + return false; + } +} + + +export function handleExternalDroppedContent({text, editor}: {text:string, editor: Editor}){ + const position = editor.inputs.shiftKey + ? editor.inputs.currentPagePoint + : editor.getViewportPageBounds().center; + + if (isURL(text)){ + createEmbedsFromUrl({editor, url: text}) + } else{ + editor.createShape({ + type: "text", + x: position.x - 75, + y: position.y - 75, + props: { + text: text, + size: "s", + textAlign: "start", + }, + }); + + } +} + function centerSelectionAroundPoint(editor: Editor, position: VecLike) { // Re-position shapes so that the center of the group is at the provided point const viewportPageBounds = editor.getViewportPageBounds() diff --git a/apps/web/app/(dash)/chat/chatWindow.tsx b/apps/web/app/(dash)/chat/chatWindow.tsx index bb6a0be1..32fd1fce 100644 --- a/apps/web/app/(dash)/chat/chatWindow.tsx +++ b/apps/web/app/(dash)/chat/chatWindow.tsx @@ -1,7 +1,7 @@ "use client"; import { AnimatePresence } from "framer-motion"; -import React, { useEffect, useState } from "react"; +import React, { useEffect, useRef, useState } from "react"; import QueryInput from "../home/queryinput"; import { cn } from "@repo/ui/lib/utils"; import { motion } from "framer-motion"; @@ -19,7 +19,10 @@ import remarkMath from "remark-math"; import rehypeKatex from "rehype-katex"; import rehypeHighlight from "rehype-highlight"; import { code, p } from "./markdownRenderHelpers"; -import { codeLanguageSubset } from "@/app/helpers/constants"; +import { codeLanguageSubset } from "@/lib/constants"; +import { z } from "zod"; +import { toast } from "sonner"; +import Link from "next/link"; function ChatWindow({ q, @@ -33,19 +36,85 @@ function ChatWindow({ { question: q, answer: { - parts: [ - // { - // text: `It seems like there might be a typo in your question. Could you please clarify or provide more context? If you meant "interesting," please let me know what specific information or topic you find interesting, and I can help you with that.`, - // }, - ], + parts: [], sources: [], }, }, ]); + const [isAutoScroll, setIsAutoScroll] = useState(true); + + const removeJustificationFromText = (text: string) => { + // remove everything after the first "" word + const justificationLine = text.indexOf(""); + if (justificationLine !== -1) { + // Add that justification to the last chat message + const lastChatMessage = chatHistory[chatHistory.length - 1]; + if (lastChatMessage) { + lastChatMessage.answer.justification = text.slice(justificationLine); + } + return text.slice(0, justificationLine); + } + return text; + }; const router = useRouter(); const getAnswer = async (query: string, spaces: string[]) => { + const sourcesFetch = await fetch( + `/api/chat?q=${query}&spaces=${spaces}&sourcesOnly=true`, + { + method: "POST", + body: JSON.stringify({ chatHistory }), + }, + ); + + // TODO: handle this properly + const sources = await sourcesFetch.json(); + + const sourcesZod = z.object({ + ids: z.array(z.string()), + metadata: z.array(z.any()), + }); + + const sourcesParsed = sourcesZod.safeParse(sources); + + if (!sourcesParsed.success) { + console.log(sources); + console.error(sourcesParsed.error); + toast.error("Something went wrong while getting the sources"); + return; + } + + setChatHistory((prevChatHistory) => { + window.scrollTo({ + top: document.documentElement.scrollHeight, + behavior: "smooth", + }); + const newChatHistory = [...prevChatHistory]; + const lastAnswer = newChatHistory[newChatHistory.length - 1]; + if (!lastAnswer) return prevChatHistory; + const filteredSourceUrls = new Set( + sourcesParsed.data.metadata.map((source) => source.url), + ); + const uniqueSources = sourcesParsed.data.metadata.filter((source) => { + if (filteredSourceUrls.has(source.url)) { + filteredSourceUrls.delete(source.url); + return true; + } + return false; + }); + lastAnswer.answer.sources = uniqueSources.map((source) => ({ + title: source.title ?? "Untitled", + type: source.type ?? "page", + source: source.url ?? "https://supermemory.ai", + content: source.description ?? "No content available", + numChunks: sourcesParsed.data.metadata.filter( + (f) => f.url === source.url, + ).length, + })); + return newChatHistory; + }); + const resp = await fetch(`/api/chat?q=${query}&spaces=${spaces}`, { method: "POST", body: JSON.stringify({ chatHistory }), @@ -53,7 +122,6 @@ function ChatWindow({ const reader = resp.body?.getReader(); let done = false; - let result = ""; while (!done && reader) { const { value, done: d } = await reader.read(); done = d; @@ -62,23 +130,28 @@ function ChatWindow({ const newChatHistory = [...prevChatHistory]; const lastAnswer = newChatHistory[newChatHistory.length - 1]; if (!lastAnswer) return prevChatHistory; - lastAnswer.answer.parts.push({ text: new TextDecoder().decode(value) }); + const txt = new TextDecoder().decode(value); + + if (isAutoScroll) { + window.scrollTo({ + top: document.documentElement.scrollHeight, + behavior: "smooth", + }); + } + + lastAnswer.answer.parts.push({ text: txt }); return newChatHistory; }); } - - console.log(result); }; useEffect(() => { if (q.trim().length > 0) { + setLayout("chat"); getAnswer( q, spaces.map((s) => s.id), ); - setTimeout(() => { - setLayout("chat"); - }, 300); } else { router.push("/home"); } @@ -94,18 +167,23 @@ function ChatWindow({ className="max-w-3xl h-full justify-center items-center flex mx-auto w-full flex-col" >
- + {}} + initialQuery={q} + initialSpaces={[]} + disabled + />
) : (
{chatHistory.map((chat, idx) => (

))} {chat.answer.sources.map((source, idx) => ( -
-
- {source.type} +
+ {source.type} + + {source.numChunks > 1 && ( + {source.numChunks} chunks + )}
-
{source.title}
-
+
{source.title}
+
+ {source.content.length > 100 + ? source.content.slice(0, 100) + "..." + : source.content} +
+ ))} @@ -197,14 +285,67 @@ function ChatWindow({ }} className="flex flex-col gap-2" > - {chat.answer.parts.map((part) => part.text).join("")} + {removeJustificationFromText( + chat.answer.parts.map((part) => part.text).join(""), + )}

- + {/* Justification */} + {chat.answer.justification && + chat.answer.justification.length && ( +
0 ? "flex" : "hidden"}`} + > + + + + Justification + + + {chat.answer.justification.length > 0 + ? chat.answer.justification + .replaceAll("", "") + .replaceAll("", "") + : "No justification provided."} + + + +
+ )}
))} + +
+ { + setChatHistory((prevChatHistory) => { + return [ + ...prevChatHistory, + { + question: q, + answer: { + parts: [], + sources: [], + }, + }, + ]; + }); + await getAnswer( + q, + spaces.map((s) => `${s.id}`), + ); + }} + /> +
)} diff --git a/apps/web/app/(dash)/dynamicisland.tsx b/apps/web/app/(dash)/dynamicisland.tsx index c08f883a..98fafc7a 100644 --- a/apps/web/app/(dash)/dynamicisland.tsx +++ b/apps/web/app/(dash)/dynamicisland.tsx @@ -4,12 +4,12 @@ import { AddIcon } from "@repo/ui/icons"; import Image from "next/image"; import { AnimatePresence, useMotionValueEvent, useScroll } from "framer-motion"; -import { useEffect, useRef, useState } from "react"; +import { useActionState, useEffect, useRef, useState } from "react"; import { motion } from "framer-motion"; import { Label } from "@repo/ui/shadcn/label"; import { Input } from "@repo/ui/shadcn/input"; import { Textarea } from "@repo/ui/shadcn/textarea"; -import { createSpace } from "../actions/doers"; +import { createMemory, createSpace } from "../actions/doers"; import { Select, SelectContent, @@ -20,6 +20,7 @@ import { import { Space } from "../actions/types"; import { getSpaces } from "../actions/fetchers"; import { toast } from "sonner"; +import { useFormStatus } from "react-dom"; export function DynamicIsland() { const { scrollYProgress } = useScroll(); @@ -253,13 +254,39 @@ function PageForm({ cancelfn: () => void; spaces: Space[]; }) { + const [loading, setLoading] = useState(false); + + const { pending } = useFormStatus(); return ( -
+
{ + const content = e.get("content")?.toString(); + const space = e.get("space")?.toString(); + if (!content) { + toast.error("Content is required"); + return; + } + setLoading(true); + const cont = await createMemory({ + content: content, + spaces: space ? [space] : undefined, + }); + + console.log(cont); + setLoading(false); + if (cont.success) { + toast.success("Memory created"); + } else { + toast.error("Memory creation failed"); + } + }} + className="bg-secondary border border-muted-foreground px-4 py-3 rounded-2xl mt-2 flex flex-col gap-3" + >
- @@ -272,24 +299,28 @@ function PageForm({
+
+ {loading ?
Loading...
: "not loading"} +
-
- cancel -
+ Submit +
-
+ ); } diff --git a/apps/web/app/(dash)/home/page.tsx b/apps/web/app/(dash)/home/page.tsx index c539673d..bdf6a61e 100644 --- a/apps/web/app/(dash)/home/page.tsx +++ b/apps/web/app/(dash)/home/page.tsx @@ -1,11 +1,12 @@ -import React from "react"; -import Menu from "../menu"; -import Header from "../header"; -import QueryInput from "./queryinput"; -import { homeSearchParamsCache } from "@/app/helpers/lib/searchParams"; -import { getSpaces } from "@/app/actions/fetchers"; +"use client"; -async function Page({ +import React, { useEffect, useState } from "react"; +import QueryInput from "./queryinput"; +import { homeSearchParamsCache } from "@/lib/searchParams"; +import { getSpaces } from "@/app/actions/fetchers"; +import { useRouter } from "next/navigation"; + +function Page({ searchParams, }: { searchParams: Record; @@ -13,12 +14,18 @@ async function Page({ // TODO: use this to show a welcome page/modal const { firstTime } = homeSearchParamsCache.parse(searchParams); - let spaces = await getSpaces(); + const [spaces, setSpaces] = useState<{ id: number; name: string }[]>([]); - if (!spaces.success) { - // TODO: handle this error properly. - spaces.data = []; - } + useEffect(() => { + getSpaces().then((res) => { + if (res.success && res.data) { + setSpaces(res.data); + } + // TODO: HANDLE ERROR + }); + }, []); + + const { push } = useRouter(); return (
@@ -26,7 +33,17 @@ async function Page({ {/*
hi {firstTime ? 'first time' : ''}
*/}
- + { + const newQ = + "/chat?q=" + + encodeURI(q) + + (spaces ? "&spaces=" + JSON.stringify(spaces) : ""); + + push(newQ); + }} + initialSpaces={spaces} + />
); diff --git a/apps/web/app/(dash)/home/queryinput.tsx b/apps/web/app/(dash)/home/queryinput.tsx index fbd537e3..4fadfb6f 100644 --- a/apps/web/app/(dash)/home/queryinput.tsx +++ b/apps/web/app/(dash)/home/queryinput.tsx @@ -12,6 +12,9 @@ function QueryInput({ initialQuery = "", initialSpaces = [], disabled = false, + className, + mini = false, + handleSubmit, }: { initialQuery?: string; initialSpaces?: { @@ -19,32 +22,14 @@ function QueryInput({ name: string; }[]; disabled?: boolean; + className?: string; + mini?: boolean; + handleSubmit: (q: string, spaces: { id: number; name: string }[]) => void; }) { const [q, setQ] = useState(initialQuery); const [selectedSpaces, setSelectedSpaces] = useState([]); - const { push } = useRouter(); - - const parseQ = () => { - // preparedSpaces is list of spaces selected by user, with id and name - const preparedSpaces = initialSpaces - .filter((x) => selectedSpaces.includes(x.id)) - .map((x) => { - return { - id: x.id, - name: x.name, - }; - }); - - const newQ = - "/chat?q=" + - encodeURI(q) + - (selectedSpaces ? "&spaces=" + JSON.stringify(preparedSpaces) : ""); - - return newQ; - }; - const options = useMemo( () => initialSpaces.map((x) => ({ @@ -54,21 +39,43 @@ function QueryInput({ [initialSpaces], ); + const preparedSpaces = useMemo( + () => + initialSpaces + .filter((x) => selectedSpaces.includes(x.id)) + .map((x) => { + return { + id: x.id, + name: x.name, + }; + }), + [selectedSpaces, initialSpaces], + ); + return ( -
-
+
+
{/* input and action button */} -
push(parseQ())} className="flex gap-4 p-3"> + { + handleSubmit(q, preparedSpaces); + setQ(""); + }} + className="flex gap-4 p-3" + >