From 5b44dcb1d1f7fcea331b248800cad60129038eab Mon Sep 17 00:00:00 2001 From: Dhravya Date: Tue, 16 Jul 2024 19:26:47 -0500 Subject: [PATCH] use tabs --- .eslintrc.js | 12 +- .prettierrc | 6 +- .vscode/settings.json | 8 +- README.md | 8 +- apps/cf-ai-backend/package.json | 30 +- apps/cf-ai-backend/src/helper.ts | 336 +-- apps/cf-ai-backend/src/index.ts | 870 +++---- apps/cf-ai-backend/src/prompts/prompt1.ts | 18 +- apps/cf-ai-backend/src/types.ts | 70 +- .../cf-ai-backend/src/utils/OpenAIEmbedder.ts | 86 +- apps/cf-ai-backend/src/utils/chonker.ts | 68 +- apps/cf-ai-backend/src/utils/seededRandom.ts | 22 +- apps/cf-ai-backend/tsconfig.json | 10 +- apps/cf-ai-backend/vite.config.ts | 2 +- apps/extension/background.ts | 802 +++---- apps/extension/components.json | 32 +- apps/extension/content/ContentApp.tsx | 732 +++--- apps/extension/content/base.css | 8 +- apps/extension/content/content.tsx | 108 +- apps/extension/content/ui/shadcn/input.tsx | 28 +- apps/extension/content/ui/shadcn/label.tsx | 18 +- apps/extension/content/ui/shadcn/popover.tsx | 42 +- apps/extension/content/ui/shadcn/select.tsx | 220 +- apps/extension/content/ui/shadcn/textarea.tsx | 26 +- apps/extension/content/ui/shadcn/toast.tsx | 158 +- apps/extension/content/ui/shadcn/toaster.tsx | 52 +- apps/extension/content/ui/shadcn/tooltip.tsx | 22 +- apps/extension/content/ui/shadcn/use-toast.ts | 256 +-- apps/extension/content/utils.ts | 2 +- apps/extension/helpers.ts | 130 +- apps/extension/manifest.json | 52 +- apps/extension/package.json | 64 +- apps/extension/postcss.config.js | 8 +- apps/extension/public/output.css | 2016 ++++++++--------- apps/extension/tsconfig.json | 32 +- apps/web/.eslintrc.js | 12 +- apps/web/app/(auth)/privacy/page.tsx | 18 +- .../TextGradient/gradient.module.css | 138 +- apps/web/app/(auth)/signin/page.tsx | 200 +- apps/web/app/(auth)/tos/page.tsx | 18 +- apps/web/app/(canvas)/canvas/[id]/page.tsx | 22 +- apps/web/app/(canvas)/canvas/page.tsx | 26 +- .../web/app/(canvas)/canvas/search&create.tsx | 62 +- apps/web/app/(canvas)/canvas/thinkPad.tsx | 466 ++-- apps/web/app/(canvas)/canvas/thinkPads.tsx | 42 +- apps/web/app/(canvas)/canvasStyles.css | 22 +- apps/web/app/(canvas)/layout.tsx | 38 +- apps/web/app/(dash)/(memories)/content.tsx | 672 +++--- .../app/(dash)/(memories)/memories/page.tsx | 6 +- .../(memories)/space/[spaceid]/page.tsx | 28 +- apps/web/app/(dash)/chat/CodeBlock.tsx | 158 +- apps/web/app/(dash)/chat/[chatid]/page.tsx | 48 +- apps/web/app/(dash)/chat/chatWindow.tsx | 774 +++---- .../app/(dash)/chat/markdownRenderHelpers.tsx | 30 +- .../web/app/(dash)/header/autoBreadCrumbs.tsx | 68 +- apps/web/app/(dash)/header/header.tsx | 84 +- apps/web/app/(dash)/header/newChatButton.tsx | 26 +- apps/web/app/(dash)/home/homeVariants.ts | 96 +- apps/web/app/(dash)/home/page.tsx | 198 +- apps/web/app/(dash)/home/queryinput.tsx | 308 +-- apps/web/app/(dash)/layout.tsx | 42 +- apps/web/app/(dash)/menu.tsx | 616 ++--- apps/web/app/(dash)/note/[noteid]/page.tsx | 30 +- .../(landing)/CardPatterns/AnimatedBeam.tsx | 329 +-- .../CardPatterns/AnimatedBeamWithOutput.tsx | 961 ++++---- .../(landing)/CardPatterns/AnimatedGrid.tsx | 254 +-- apps/web/app/(landing)/CardPatterns/Glare.tsx | 251 +- apps/web/app/(landing)/Cta.tsx | 84 +- apps/web/app/(landing)/EmailInput.tsx | 144 +- apps/web/app/(landing)/FeatureCardContent.tsx | 191 +- apps/web/app/(landing)/FeatureContent.tsx | 96 +- apps/web/app/(landing)/Features.tsx | 108 +- .../web/app/(landing)/Features/chatbubble.tsx | 40 +- apps/web/app/(landing)/Features/features.tsx | 26 +- .../web/app/(landing)/Features/generating.tsx | 20 +- apps/web/app/(landing)/Features/index.tsx | 396 ++-- .../app/(landing)/GridPatterns/PlusGrid.tsx | 68 +- apps/web/app/(landing)/Headers/Navbar.tsx | 194 +- apps/web/app/(landing)/Hero.tsx | 142 +- apps/web/app/(landing)/ImageSliders.tsx | 324 +-- apps/web/app/(landing)/Navbar.tsx | 84 +- apps/web/app/(landing)/RotatingIcons.tsx | 174 +- apps/web/app/(landing)/Showcase.tsx | 486 ++-- apps/web/app/(landing)/footer.tsx | 62 +- apps/web/app/(landing)/formSubmitAction.ts | 74 +- apps/web/app/(landing)/linkArrow.tsx | 66 +- apps/web/app/(landing)/page.tsx | 70 +- apps/web/app/(landing)/twitterLink.tsx | 208 +- apps/web/app/(quicklinks)/extension/route.ts | 6 +- apps/web/app/actions/doers.ts | 1132 ++++----- apps/web/app/actions/fetchers.ts | 538 ++--- apps/web/app/actions/types.ts | 14 +- apps/web/app/api/canvas/route.ts | 10 +- apps/web/app/api/canvasai/route.ts | 46 +- apps/web/app/api/chat/route.ts | 126 +- apps/web/app/api/editorai/route.ts | 44 +- apps/web/app/api/ensureAuth.ts | 42 +- apps/web/app/api/getCount/route.ts | 66 +- apps/web/app/api/hello/route.ts | 26 +- apps/web/app/api/me/route.ts | 66 +- apps/web/app/api/mobile/newUser/route.ts | 13 + apps/web/app/api/spaces/route.ts | 32 +- apps/web/app/api/store/route.ts | 338 +-- apps/web/app/api/telegram/route.ts | 150 +- apps/web/app/api/unfirlsite/route.ts | 250 +- apps/web/app/api/upload_image/route.ts | 78 +- apps/web/app/global-error.tsx | 26 +- apps/web/app/layout.tsx | 126 +- apps/web/app/ref/page.tsx | 202 +- apps/web/cf-env.d.ts | 34 +- apps/web/components/canvas/canvas.tsx | 132 +- .../components/canvas/draggableComponent.tsx | 84 +- apps/web/components/canvas/dropComponent.tsx | 338 +-- .../components/canvas/enabledComp copy.tsx | 36 +- apps/web/components/canvas/enabledComp.tsx | 36 +- .../web/components/canvas/resizableLayout.tsx | 268 +-- apps/web/components/canvas/savesnap.tsx | 46 +- apps/web/components/canvas/textCard.tsx | 74 +- apps/web/components/canvas/twitterCard.tsx | 124 +- .../components/twitter/icons/icons.module.css | 14 +- .../twitter/icons/verified-business.tsx | 98 +- .../twitter/icons/verified-government.tsx | 28 +- .../web/components/twitter/icons/verified.tsx | 20 +- apps/web/components/twitter/render-tweet.tsx | 196 +- .../twitter/tweet-header.module.css | 106 +- .../twitter/verified-badge.module.css | 8 +- .../web/components/twitter/verified-badge.tsx | 46 +- apps/web/drizzle.config.ts | 16 +- apps/web/env.d.ts | 8 +- apps/web/instrumentation.ts | 12 +- apps/web/lib/constants.ts | 76 +- apps/web/lib/context.ts | 14 +- apps/web/lib/createAssetUrl.ts | 158 +- apps/web/lib/createEmbeds.ts | 388 ++-- apps/web/lib/get-metadata.ts | 54 +- apps/web/lib/get-theme-button.tsx | 10 +- apps/web/lib/handle-errors.ts | 30 +- apps/web/lib/loadSnap.ts | 12 +- apps/web/lib/searchParams.ts | 48 +- apps/web/middleware.ts | 24 +- apps/web/migrations/meta/0000_snapshot.json | 1608 ++++++------- apps/web/migrations/meta/_journal.json | 22 +- apps/web/next.config.mjs | 72 +- apps/web/package.json | 104 +- apps/web/public/site.webmanifest | 34 +- apps/web/sentry.client.config.ts | 34 +- apps/web/sentry.edge.config.ts | 10 +- apps/web/sentry.server.config.ts | 14 +- apps/web/server/auth.ts | 52 +- apps/web/server/db/schema.ts | 370 +-- apps/web/server/encrypt.ts | 128 +- apps/web/tailwind.config.ts | 1 - apps/web/tsconfig.json | 46 +- components.json | 32 +- package.json | 234 +- packages/eslint-config/library.js | 54 +- packages/eslint-config/next.js | 102 +- packages/eslint-config/package.json | 34 +- packages/eslint-config/react-internal.js | 52 +- packages/shared-types/index.ts | 96 +- packages/shared-types/package.json | 4 +- packages/shared-types/tsconfig.json | 10 +- packages/shared-types/utils.ts | 16 +- packages/tailwind-config/globals.css | 278 +-- packages/tailwind-config/package.json | 30 +- packages/tailwind-config/postcss.config.js | 8 +- packages/tailwind-config/tailwind.config.ts | 232 +- packages/typescript-config/base.json | 32 +- packages/typescript-config/nextjs.json | 22 +- packages/typescript-config/package.json | 14 +- packages/typescript-config/react-library.json | 12 +- packages/ui/.eslintrc.js | 14 +- packages/ui/components/cardClick.tsx | 126 +- packages/ui/components/icons.tsx | 566 ++--- packages/ui/hooks/use-callback-ref.ts | 14 +- packages/ui/hooks/use-controllable-state.ts | 82 +- packages/ui/icons/index.ts | 34 +- packages/ui/lib/utils.ts | 56 +- packages/ui/package.json | 40 +- packages/ui/shadcn/accordion.tsx | 54 +- packages/ui/shadcn/badge.tsx | 44 +- packages/ui/shadcn/breadcrumb.tsx | 144 +- packages/ui/shadcn/button.tsx | 78 +- packages/ui/shadcn/card.tsx | 100 +- packages/ui/shadcn/combobox.tsx | 132 +- packages/ui/shadcn/command.tsx | 180 +- packages/ui/shadcn/dialog.tsx | 152 +- packages/ui/shadcn/divider.tsx | 2 +- packages/ui/shadcn/drawer.tsx | 136 +- packages/ui/shadcn/dropdown-menu.tsx | 266 +-- packages/ui/shadcn/form.tsx | 216 +- packages/ui/shadcn/input.tsx | 28 +- packages/ui/shadcn/label.tsx | 18 +- packages/ui/shadcn/popover.tsx | 28 +- packages/ui/shadcn/progress.tsx | 30 +- packages/ui/shadcn/scroll-area.tsx | 60 +- packages/ui/shadcn/select.tsx | 216 +- packages/ui/shadcn/separator.tsx | 31 +- packages/ui/shadcn/sonner.tsx | 38 +- packages/ui/shadcn/switch.tsx | 32 +- packages/ui/shadcn/tabs.tsx | 60 +- packages/ui/shadcn/textarea.tsx | 34 +- packages/ui/shadcn/theme-toggle.tsx | 6 +- packages/ui/shadcn/toast.tsx | 158 +- packages/ui/shadcn/toaster.tsx | 52 +- packages/ui/shadcn/tooltip.tsx | 22 +- packages/ui/shadcn/use-toast.ts | 256 +-- packages/ui/tsconfig.json | 12 +- packages/ui/tsconfig.lint.json | 12 +- packages/ui/turbo/generators/config.ts | 48 +- .../turbo/generators/templates/component.hbs | 4 +- tsconfig.json | 2 +- turbo.json | 52 +- 213 files changed, 14379 insertions(+), 14336 deletions(-) create mode 100644 apps/web/app/api/mobile/newUser/route.ts diff --git a/.eslintrc.js b/.eslintrc.js index e928b868..d30c0a1e 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,10 +1,10 @@ // This configuration only applies to the package manager root. /** @type {import("eslint").Linter.Config} */ module.exports = { - ignorePatterns: ["apps/**", "packages/**"], - extends: ["@repo/eslint-config/library.js"], - parser: "@typescript-eslint/parser", - parserOptions: { - project: true, - }, + ignorePatterns: ["apps/**", "packages/**"], + extends: ["@repo/eslint-config/library.js"], + parser: "@typescript-eslint/parser", + parserOptions: { + project: true, + }, }; diff --git a/.prettierrc b/.prettierrc index 1f37a581..cbfd12dc 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,5 +1,5 @@ { - "tabWidth": 2, - "trailingComma": "all", - "useTabs": false + "tabWidth": 2, + "trailingComma": "all", + "useTabs": true } diff --git a/.vscode/settings.json b/.vscode/settings.json index 2b006d2e..4bc75484 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { - "css.validate": false, - "editor.quickSuggestions": { - "strings": true - } + "css.validate": false, + "editor.quickSuggestions": { + "strings": true + } } diff --git a/README.md b/README.md index 5958bbf2..4bd34791 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,14 @@ To use the chrome extension, image ### Import Twitter Bookmarks + 1. Make sure you signed into supermemory and installed chrome extension 2. Open Twitter/X, you will see the save icon as follows image -4. Click on save button and give it 10 - 20 secs, where supermemory extension will sync all your twitter bookmarks to supermemory.ai -5. Volia! Now your second brain has all your twitter bookmarks. +3. Click on save button and give it 10 - 20 secs, where supermemory extension will sync all your twitter bookmarks to supermemory.ai +4. Volia! Now your second brain has all your twitter bookmarks. ## 👨‍💻 The Stack @@ -85,7 +86,6 @@ The database, auth etc logic is here ![image](https://github.com/MaheshtheDev/supermemory/assets/38828053/0c44708d-600a-43a8-a641-835cb1f349fc) - Built with: - Nextjs 14 @@ -140,4 +140,4 @@ Contributions are very welcome! A contribution can be as small as a ⭐ or even - \ No newline at end of file + diff --git a/apps/cf-ai-backend/package.json b/apps/cf-ai-backend/package.json index 78353e08..fee0c0d8 100644 --- a/apps/cf-ai-backend/package.json +++ b/apps/cf-ai-backend/package.json @@ -1,17 +1,17 @@ { - "name": "new-cf-ai-backend", - "private": true, - "version": "0.0.1", - "main": "src/index.ts", - "scripts": { - "test": "jest --verbose", - "deploy": "wrangler deploy", - "dev": "wrangler dev --remote --port 8686", - "start": "wrangler dev", - "unsafe-reset-vector-db": "wrangler vectorize delete supermem-vector && wrangler vectorize create --dimensions=1536 supermem-vector-1 --metric=cosine" - }, - "license": "MIT", - "dependencies": { - "@hono/zod-validator": "^0.2.1" - } + "name": "new-cf-ai-backend", + "private": true, + "version": "0.0.1", + "main": "src/index.ts", + "scripts": { + "test": "jest --verbose", + "deploy": "wrangler deploy", + "dev": "wrangler dev --remote --port 8686", + "start": "wrangler dev", + "unsafe-reset-vector-db": "wrangler vectorize delete supermem-vector && wrangler vectorize create --dimensions=1536 supermem-vector-1 --metric=cosine" + }, + "license": "MIT", + "dependencies": { + "@hono/zod-validator": "^0.2.1" + } } diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 8502ca35..dad84a49 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -9,211 +9,211 @@ import { z } from "zod"; import { seededRandom } from "./utils/seededRandom"; export async function initQuery( - c: Context<{ Bindings: Env }>, - model: string = "gpt-4o", + c: Context<{ Bindings: Env }>, + model: string = "gpt-4o", ) { - const embeddings = new OpenAIEmbeddings({ - apiKey: c.env.OPENAI_API_KEY, - modelName: "text-embedding-3-small", - }); + const embeddings = new OpenAIEmbeddings({ + apiKey: c.env.OPENAI_API_KEY, + modelName: "text-embedding-3-small", + }); - const store = new CloudflareVectorizeStore(embeddings, { - index: c.env.VECTORIZE_INDEX, - }); + const store = new CloudflareVectorizeStore(embeddings, { + index: c.env.VECTORIZE_INDEX, + }); - let selectedModel: - | ReturnType> - | ReturnType> - | ReturnType>; + let selectedModel: + | ReturnType> + | ReturnType> + | ReturnType>; - switch (model) { - case "claude-3-opus": - const anthropic = createAnthropic({ - apiKey: c.env.ANTHROPIC_API_KEY, - baseURL: - "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/anthropic", - }); - selectedModel = anthropic.chat("claude-3-opus-20240229"); - console.log("Selected model: ", selectedModel); - break; - case "gemini-1.5-pro": - const googleai = createGoogleGenerativeAI({ - apiKey: c.env.GOOGLE_AI_API_KEY, - baseURL: - "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/google-vertex-ai", - }); - selectedModel = googleai.chat("models/gemini-1.5-pro-latest"); - console.log("Selected model: ", selectedModel); - break; - case "gpt-4o": - default: - const openai = createOpenAI({ - apiKey: c.env.OPENAI_API_KEY, - baseURL: - "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/openai", - compatibility: "strict", - }); - selectedModel = openai.chat("gpt-4o"); - break; - } + switch (model) { + case "claude-3-opus": + const anthropic = createAnthropic({ + apiKey: c.env.ANTHROPIC_API_KEY, + baseURL: + "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/anthropic", + }); + selectedModel = anthropic.chat("claude-3-opus-20240229"); + console.log("Selected model: ", selectedModel); + break; + case "gemini-1.5-pro": + const googleai = createGoogleGenerativeAI({ + apiKey: c.env.GOOGLE_AI_API_KEY, + baseURL: + "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/google-vertex-ai", + }); + selectedModel = googleai.chat("models/gemini-1.5-pro-latest"); + console.log("Selected model: ", selectedModel); + break; + case "gpt-4o": + default: + const openai = createOpenAI({ + apiKey: c.env.OPENAI_API_KEY, + baseURL: + "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/openai", + compatibility: "strict", + }); + selectedModel = openai.chat("gpt-4o"); + break; + } - return { store, model: selectedModel }; + return { store, model: selectedModel }; } export async function deleteDocument({ - url, - user, - c, - store, + url, + user, + c, + store, }: { - url: string; - user: string; - c: Context<{ Bindings: Env }>; - store: CloudflareVectorizeStore; + url: string; + user: string; + c: Context<{ Bindings: Env }>; + store: CloudflareVectorizeStore; }) { - const toBeDeleted = `${url}#supermemory-web`; - const random = seededRandom(toBeDeleted); + const toBeDeleted = `${url}#supermemory-web`; + const random = seededRandom(toBeDeleted); - const uuid = - random().toString(36).substring(2, 15) + - random().toString(36).substring(2, 15); + const uuid = + random().toString(36).substring(2, 15) + + random().toString(36).substring(2, 15); - const allIds = await c.env.KV.list({ prefix: uuid }); + const allIds = await c.env.KV.list({ prefix: uuid }); - if (allIds.keys.length > 0) { - const savedVectorIds = allIds.keys.map((key) => key.name); - const vectors = await c.env.VECTORIZE_INDEX.getByIds(savedVectorIds); - // We don't actually delete document directly, we just remove the user from the metadata. - // If there's no user left, we can delete the document. - const newVectors = vectors.map((vector) => { - delete vector.metadata[`user-${user}`]; + if (allIds.keys.length > 0) { + const savedVectorIds = allIds.keys.map((key) => key.name); + const vectors = await c.env.VECTORIZE_INDEX.getByIds(savedVectorIds); + // We don't actually delete document directly, we just remove the user from the metadata. + // If there's no user left, we can delete the document. + const newVectors = vectors.map((vector) => { + delete vector.metadata[`user-${user}`]; - // Get count of how many users are left - const userCount = Object.keys(vector.metadata).filter((key) => - key.startsWith("user-"), - ).length; + // Get count of how many users are left + const userCount = Object.keys(vector.metadata).filter((key) => + key.startsWith("user-"), + ).length; - // If there's no user left, we can delete the document. - // need to make sure that every chunk is deleted otherwise it would be problematic. - if (userCount === 0) { - store.delete({ ids: savedVectorIds }); - void Promise.all(savedVectorIds.map((id) => c.env.KV.delete(id))); - return null; - } + // If there's no user left, we can delete the document. + // need to make sure that every chunk is deleted otherwise it would be problematic. + if (userCount === 0) { + store.delete({ ids: savedVectorIds }); + void Promise.all(savedVectorIds.map((id) => c.env.KV.delete(id))); + return null; + } - return vector; - }); + return vector; + }); - // If all vectors are null (deleted), we can delete the KV too. Otherwise, we update (upsert) the vectors. - if (newVectors.every((v) => v === null)) { - await c.env.KV.delete(uuid); - } else { - await c.env.VECTORIZE_INDEX.upsert(newVectors.filter((v) => v !== null)); - } - } + // If all vectors are null (deleted), we can delete the KV too. Otherwise, we update (upsert) the vectors. + if (newVectors.every((v) => v === null)) { + await c.env.KV.delete(uuid); + } else { + await c.env.VECTORIZE_INDEX.upsert(newVectors.filter((v) => v !== null)); + } + } } function sanitizeKey(key: string): string { - if (!key) throw new Error("Key cannot be empty"); + if (!key) throw new Error("Key cannot be empty"); - // Remove or replace invalid characters - let sanitizedKey = key.replace(/[.$"]/g, "_"); + // Remove or replace invalid characters + let sanitizedKey = key.replace(/[.$"]/g, "_"); - // Ensure key does not start with $ - if (sanitizedKey.startsWith("$")) { - sanitizedKey = sanitizedKey.substring(1); - } + // Ensure key does not start with $ + if (sanitizedKey.startsWith("$")) { + sanitizedKey = sanitizedKey.substring(1); + } - return sanitizedKey; + return sanitizedKey; } export async function batchCreateChunksAndEmbeddings({ - store, - body, - chunks, - context, + store, + body, + chunks, + context, }: { - store: CloudflareVectorizeStore; - body: z.infer; - chunks: string[]; - context: Context<{ Bindings: Env }>; + store: CloudflareVectorizeStore; + body: z.infer; + chunks: string[]; + context: Context<{ Bindings: Env }>; }) { - //! NOTE that we use #supermemory-web to ensure that - //! If a user saves it through the extension, we don't want other users to be able to see it. - // Requests from the extension should ALWAYS have a unique ID with the USERiD in it. - // I cannot stress this enough, important for security. - const ourID = `${body.url}#supermemory-web`; - const random = seededRandom(ourID); - const uuid = - random().toString(36).substring(2, 15) + - random().toString(36).substring(2, 15); + //! NOTE that we use #supermemory-web to ensure that + //! If a user saves it through the extension, we don't want other users to be able to see it. + // Requests from the extension should ALWAYS have a unique ID with the USERiD in it. + // I cannot stress this enough, important for security. + const ourID = `${body.url}#supermemory-web`; + const random = seededRandom(ourID); + const uuid = + random().toString(36).substring(2, 15) + + random().toString(36).substring(2, 15); - const allIds = await context.env.KV.list({ prefix: uuid }); + const allIds = await context.env.KV.list({ prefix: uuid }); - let pageContent = ""; - // If some chunks for that content already exist, we'll just update the metadata to include - // the user. - if (allIds.keys.length > 0) { - const savedVectorIds = allIds.keys.map((key) => key.name); - const vectors = await context.env.VECTORIZE_INDEX.getByIds(savedVectorIds); + let pageContent = ""; + // If some chunks for that content already exist, we'll just update the metadata to include + // the user. + if (allIds.keys.length > 0) { + const savedVectorIds = allIds.keys.map((key) => key.name); + const vectors = await context.env.VECTORIZE_INDEX.getByIds(savedVectorIds); - // Now, we'll update all vector metadatas with one more userId and all spaceIds - const newVectors = vectors.map((vector) => { - vector.metadata = { - ...vector.metadata, - [`user-${body.user}`]: 1, + // Now, we'll update all vector metadatas with one more userId and all spaceIds + const newVectors = vectors.map((vector) => { + vector.metadata = { + ...vector.metadata, + [`user-${body.user}`]: 1, - // For each space in body, add the spaceId to the vector metadata - ...(body.spaces ?? [])?.reduce((acc, space) => { - acc[`space-${body.user}-${space}`] = 1; - return acc; - }, {}), - }; - const content = - vector.metadata.content.toString().split("Content: ")[1] || - vector.metadata.content; - pageContent += `<---chunkId: ${vector.id}\n${content}\n---->`; - return vector; - }); + // For each space in body, add the spaceId to the vector metadata + ...(body.spaces ?? [])?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}), + }; + const content = + vector.metadata.content.toString().split("Content: ")[1] || + vector.metadata.content; + pageContent += `<---chunkId: ${vector.id}\n${content}\n---->`; + return vector; + }); - await context.env.VECTORIZE_INDEX.upsert(newVectors); - return pageContent; //Return the page content that goes to d1 db - } + await context.env.VECTORIZE_INDEX.upsert(newVectors); + return pageContent; //Return the page content that goes to d1 db + } - for (let i = 0; i < chunks.length; i++) { - const chunk = chunks[i]; - const chunkId = `${uuid}-${i}`; + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]; + const chunkId = `${uuid}-${i}`; - const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${chunk}`; + const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${chunk}`; - const docs = await store.addDocuments( - [ - { - pageContent: newPageContent, - metadata: { - title: body.title?.slice(0, 50) ?? "", - description: body.description ?? "", - url: body.url, - type: body.type ?? "page", - content: newPageContent, + const docs = await store.addDocuments( + [ + { + pageContent: newPageContent, + metadata: { + title: body.title?.slice(0, 50) ?? "", + description: body.description ?? "", + url: body.url, + type: body.type ?? "page", + content: newPageContent, - [sanitizeKey(`user-${body.user}`)]: 1, - ...body.spaces?.reduce((acc, space) => { - acc[`space-${body.user}-${space}`] = 1; - return acc; - }, {}), - }, - }, - ], - { - ids: [chunkId], - }, - ); + [sanitizeKey(`user-${body.user}`)]: 1, + ...body.spaces?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}), + }, + }, + ], + { + ids: [chunkId], + }, + ); - console.log("Docs added: ", docs); + console.log("Docs added: ", docs); - await context.env.KV.put(chunkId, ourID); - pageContent += `<---chunkId: ${chunkId}\n${chunk}\n---->`; - } - return pageContent; // Return the pageContent that goes to the d1 db + await context.env.KV.put(chunkId, ourID); + pageContent += `<---chunkId: ${chunkId}\n${chunk}\n---->`; + } + return pageContent; // Return the pageContent that goes to the d1 db } diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 0844c22e..72b4d527 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -3,9 +3,9 @@ import { Hono } from "hono"; import { CoreMessage, generateText, streamText, tool } from "ai"; import { chatObj, Env, vectorObj } from "./types"; import { - batchCreateChunksAndEmbeddings, - deleteDocument, - initQuery, + batchCreateChunksAndEmbeddings, + deleteDocument, + initQuery, } from "./helper"; import { timing } from "hono/timing"; import { logger } from "hono/logger"; @@ -19,10 +19,10 @@ import { swaggerUI } from "@hono/swagger-ui"; const app = new Hono<{ Bindings: Env }>(); app.get( - "/ui", - swaggerUI({ - url: "/doc", - }), + "/ui", + swaggerUI({ + url: "/doc", + }), ); // ------- MIDDLEWARES ------- @@ -31,345 +31,349 @@ app.use("*", timing()); app.use("*", logger()); app.use("/api/", async (c, next) => { - if (c.env.NODE_ENV !== "development") { - const auth = bearerAuth({ token: c.env.SECURITY_KEY }); - return auth(c, next); - } - return next(); + if (c.env.NODE_ENV !== "development") { + const auth = bearerAuth({ token: c.env.SECURITY_KEY }); + return auth(c, next); + } + return next(); }); // ------- MIDDLEWARES END ------- const fileSchema = z - .instanceof(File) - .refine( - (file) => file.size <= 10 * 1024 * 1024, - "File size should be less than 10MB", - ) // Validate file size - .refine( - (file) => ["image/jpeg", "image/png", "image/gif"].includes(file.type), - "Invalid file type", - ); // Validate file type + .instanceof(File) + .refine( + (file) => file.size <= 10 * 1024 * 1024, + "File size should be less than 10MB", + ) // Validate file size + .refine( + (file) => ["image/jpeg", "image/png", "image/gif"].includes(file.type), + "Invalid file type", + ); // Validate file type app.get("/", (c) => { - return c.text("Supermemory backend API is running!"); + return c.text("Supermemory backend API is running!"); }); app.get("/api/health", (c) => { - return c.json({ status: "ok" }); + return c.json({ status: "ok" }); }); app.post("/api/add", zValidator("json", vectorObj), async (c) => { - const body = c.req.valid("json"); + const body = c.req.valid("json"); - const { store } = await initQuery(c); + const { store } = await initQuery(c); - console.log(body.spaces); - const chunks = chunkText(body.pageContent, 1536); - if (chunks.length > 20) { - return c.json({ status: "error", message: "We are unable to process documents this size just yet, try something smaller" }); - } - const chunkedInput = await batchCreateChunksAndEmbeddings({ - store, - body, - chunks: chunks, - context: c, - }); + console.log(body.spaces); + const chunks = chunkText(body.pageContent, 1536); + if (chunks.length > 20) { + return c.json({ + status: "error", + message: + "We are unable to process documents this size just yet, try something smaller", + }); + } + const chunkedInput = await batchCreateChunksAndEmbeddings({ + store, + body, + chunks: chunks, + context: c, + }); - return c.json({ status: "ok", chunkedInput }); + return c.json({ status: "ok", chunkedInput }); }); app.post( - "/api/add-with-image", - zValidator( - "form", - z.object({ - images: z - .array(fileSchema) - .min(1, "At least one image is required") - .optional(), - "images[]": z - .array(fileSchema) - .min(1, "At least one image is required") - .optional(), - text: z.string().optional(), - spaces: z.array(z.string()).optional(), - url: z.string(), - user: z.string(), - }), - (c) => { - console.log(c); - }, - ), - async (c) => { - const body = c.req.valid("form"); + "/api/add-with-image", + zValidator( + "form", + z.object({ + images: z + .array(fileSchema) + .min(1, "At least one image is required") + .optional(), + "images[]": z + .array(fileSchema) + .min(1, "At least one image is required") + .optional(), + text: z.string().optional(), + spaces: z.array(z.string()).optional(), + url: z.string(), + user: z.string(), + }), + (c) => { + console.log(c); + }, + ), + async (c) => { + const body = c.req.valid("form"); - const { store } = await initQuery(c); + const { store } = await initQuery(c); - if (!(body.images || body["images[]"])) { - return c.json({ status: "error", message: "No images found" }, 400); - } + if (!(body.images || body["images[]"])) { + return c.json({ status: "error", message: "No images found" }, 400); + } - const imagePromises = (body.images ?? body["images[]"]).map( - async (image) => { - const buffer = await image.arrayBuffer(); - const input = { - image: [...new Uint8Array(buffer)], - prompt: - "What's in this image? caption everything you see in great detail. If it has text, do an OCR and extract all of it.", - max_tokens: 1024, - }; - const response = await c.env.AI.run( - "@cf/llava-hf/llava-1.5-7b-hf", - input, - ); - console.log(response.description); - return response.description; - }, - ); + const imagePromises = (body.images ?? body["images[]"]).map( + async (image) => { + const buffer = await image.arrayBuffer(); + const input = { + image: [...new Uint8Array(buffer)], + prompt: + "What's in this image? caption everything you see in great detail. If it has text, do an OCR and extract all of it.", + max_tokens: 1024, + }; + const response = await c.env.AI.run( + "@cf/llava-hf/llava-1.5-7b-hf", + input, + ); + console.log(response.description); + return response.description; + }, + ); - const imageDescriptions = await Promise.all(imagePromises); + const imageDescriptions = await Promise.all(imagePromises); - await batchCreateChunksAndEmbeddings({ - store, - body: { - url: body.url, - user: body.user, - type: "image", - description: - imageDescriptions.length > 1 - ? `A group of ${imageDescriptions.length} images on ${body.url}` - : imageDescriptions[0], - spaces: body.spaces, - pageContent: imageDescriptions.join("\n"), - title: "Image content from the web", - }, - chunks: [ - imageDescriptions, - ...(body.text ? chunkText(body.text, 1536) : []), - ].flat(), - context: c, - }); + await batchCreateChunksAndEmbeddings({ + store, + body: { + url: body.url, + user: body.user, + type: "image", + description: + imageDescriptions.length > 1 + ? `A group of ${imageDescriptions.length} images on ${body.url}` + : imageDescriptions[0], + spaces: body.spaces, + pageContent: imageDescriptions.join("\n"), + title: "Image content from the web", + }, + chunks: [ + imageDescriptions, + ...(body.text ? chunkText(body.text, 1536) : []), + ].flat(), + context: c, + }); - return c.json({ status: "ok" }); - }, + return c.json({ status: "ok" }); + }, ); app.get( - "/api/ask", - zValidator( - "query", - z.object({ - query: z.string(), - }), - ), - async (c) => { - const query = c.req.valid("query"); + "/api/ask", + zValidator( + "query", + z.object({ + query: z.string(), + }), + ), + async (c) => { + const query = c.req.valid("query"); - const { model } = await initQuery(c); + const { model } = await initQuery(c); - const response = await streamText({ model, prompt: query.query }); - const r = response.toTextStreamResponse(); + const response = await streamText({ model, prompt: query.query }); + const r = response.toTextStreamResponse(); - return r; - }, + return r; + }, ); app.get( - "/api/search", - zValidator("query", z.object({ query: z.string(), user: z.string() })), - async (c) => { - const { query, user } = c.req.valid("query"); - const filter: VectorizeVectorMetadataFilter = { - [`user-${user}`]: 1, - }; + "/api/search", + zValidator("query", z.object({ query: z.string(), user: z.string() })), + async (c) => { + const { query, user } = c.req.valid("query"); + const filter: VectorizeVectorMetadataFilter = { + [`user-${user}`]: 1, + }; - const { store } = await initQuery(c); - const queryAsVector = await store.embeddings.embedQuery(query); + const { store } = await initQuery(c); + const queryAsVector = await store.embeddings.embedQuery(query); - const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { - topK: 5, - filter, - returnMetadata: true, - }); + const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { + topK: 5, + filter, + returnMetadata: true, + }); - const minScore = Math.min(...resp.matches.map(({ score }) => score)); - const maxScore = Math.max(...resp.matches.map(({ score }) => score)); + const minScore = Math.min(...resp.matches.map(({ score }) => score)); + const maxScore = Math.max(...resp.matches.map(({ score }) => score)); - // This entire chat part is basically just a dumb down version of the /api/chat endpoint. - const normalizedData = resp.matches.map((data) => ({ - ...data, - normalizedScore: - maxScore !== minScore - ? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98 - : 50, - })); + // This entire chat part is basically just a dumb down version of the /api/chat endpoint. + const normalizedData = resp.matches.map((data) => ({ + ...data, + normalizedScore: + maxScore !== minScore + ? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98 + : 50, + })); - const preparedContext = normalizedData.map( - ({ metadata, score, normalizedScore }) => ({ - context: `Title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, - score, - normalizedScore, - }), - ); + const preparedContext = normalizedData.map( + ({ metadata, score, normalizedScore }) => ({ + context: `Title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, + score, + normalizedScore, + }), + ); - return c.json({ - status: "ok", - response: preparedContext, - }); - }, + return c.json({ + status: "ok", + response: preparedContext, + }); + }, ); // This is a special endpoint for our "chatbot-only" solutions. // It does both - adding content AND chatting with it. app.post( - "/api/autoChatOrAdd", - zValidator( - "query", - z.object({ - query: z.string(), - user: z.string(), - }), - ), - zValidator("json", chatObj), - async (c) => { - const { query, user } = c.req.valid("query"); - const { chatHistory } = c.req.valid("json"); + "/api/autoChatOrAdd", + zValidator( + "query", + z.object({ + query: z.string(), + user: z.string(), + }), + ), + zValidator("json", chatObj), + async (c) => { + const { query, user } = c.req.valid("query"); + const { chatHistory } = c.req.valid("json"); - const { store, model } = await initQuery(c); + const { store, model } = await initQuery(c); - let task: "add" | "chat" = "chat"; - let thingToAdd: "page" | "image" | "text" | undefined = undefined; - let addContent: string | undefined = undefined; + let task: "add" | "chat" = "chat"; + let thingToAdd: "page" | "image" | "text" | undefined = undefined; + let addContent: string | undefined = undefined; - // This is a "router". this finds out if the user wants to add a document, or chat with the AI to get a response. - const routerQuery = await generateText({ - model: model, - system: `You are Supermemory chatbot. You can either add a document to the supermemory database, or return a chat response. Based on this query, + // This is a "router". this finds out if the user wants to add a document, or chat with the AI to get a response. + const routerQuery = await generateText({ + model: model, + system: `You are Supermemory chatbot. You can either add a document to the supermemory database, or return a chat response. Based on this query, You must determine what to do. Basically if it feels like a "question", then you should intiate a chat. If it feels like a "command" or feels like something that could be forwarded to the AI, then you should add a document. You must also extract the "thing" to add and what type of thing it is.`, - prompt: `Question from user: ${query}`, - tools: { - decideTask: tool({ - description: - "Decide if the user wants to add a document or chat with the AI", - parameters: z.object({ - generatedTask: z.enum(["add", "chat"]), - contentToAdd: z.object({ - thing: z.enum(["page", "image", "text"]), - content: z.string(), - }), - }), - execute: async ({ generatedTask, contentToAdd }) => { - task = generatedTask; - thingToAdd = contentToAdd.thing; - addContent = contentToAdd.content; - }, - }), - }, - }); + prompt: `Question from user: ${query}`, + tools: { + decideTask: tool({ + description: + "Decide if the user wants to add a document or chat with the AI", + parameters: z.object({ + generatedTask: z.enum(["add", "chat"]), + contentToAdd: z.object({ + thing: z.enum(["page", "image", "text"]), + content: z.string(), + }), + }), + execute: async ({ generatedTask, contentToAdd }) => { + task = generatedTask; + thingToAdd = contentToAdd.thing; + addContent = contentToAdd.content; + }, + }), + }, + }); - if ((task as string) === "add") { - // addString is the plaintext string that the user wants to add to the database - let addString: string = addContent; + if ((task as string) === "add") { + // addString is the plaintext string that the user wants to add to the database + let addString: string = addContent; - if (thingToAdd === "page") { - // TODO: Sometimes this query hangs, and errors out. we need to do proper error management here. - const response = await fetch("https://md.dhr.wtf/?url=" + addContent, { - headers: { - Authorization: "Bearer " + c.env.SECURITY_KEY, - }, - }); + if (thingToAdd === "page") { + // TODO: Sometimes this query hangs, and errors out. we need to do proper error management here. + const response = await fetch("https://md.dhr.wtf/?url=" + addContent, { + headers: { + Authorization: "Bearer " + c.env.SECURITY_KEY, + }, + }); - addString = await response.text(); - } + addString = await response.text(); + } - // At this point, we can just go ahead and create the embeddings! - await batchCreateChunksAndEmbeddings({ - store, - body: { - url: addContent, - user, - type: thingToAdd, - pageContent: addString, - title: `${addString.slice(0, 30)}... (Added from chatbot)`, - }, - chunks: chunkText(addString, 1536), - context: c, - }); + // At this point, we can just go ahead and create the embeddings! + await batchCreateChunksAndEmbeddings({ + store, + body: { + url: addContent, + user, + type: thingToAdd, + pageContent: addString, + title: `${addString.slice(0, 30)}... (Added from chatbot)`, + }, + chunks: chunkText(addString, 1536), + context: c, + }); - return c.json({ - status: "ok", - response: - "I added the document to your personal second brain! You can now use it to answer questions or chat with me.", - contentAdded: { - type: thingToAdd, - content: addString, - url: - thingToAdd === "page" - ? addContent - : `https://supermemory.ai/note/${Date.now()}`, - }, - }); - } else { - const filter: VectorizeVectorMetadataFilter = { - [`user-${user}`]: 1, - }; + return c.json({ + status: "ok", + response: + "I added the document to your personal second brain! You can now use it to answer questions or chat with me.", + contentAdded: { + type: thingToAdd, + content: addString, + url: + thingToAdd === "page" + ? addContent + : `https://supermemory.ai/note/${Date.now()}`, + }, + }); + } else { + const filter: VectorizeVectorMetadataFilter = { + [`user-${user}`]: 1, + }; - const queryAsVector = await store.embeddings.embedQuery(query); + const queryAsVector = await store.embeddings.embedQuery(query); - const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { - topK: 5, - filter, - returnMetadata: true, - }); + const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { + topK: 5, + filter, + returnMetadata: true, + }); - const minScore = Math.min(...resp.matches.map(({ score }) => score)); - const maxScore = Math.max(...resp.matches.map(({ score }) => score)); + const minScore = Math.min(...resp.matches.map(({ score }) => score)); + const maxScore = Math.max(...resp.matches.map(({ score }) => score)); - // This entire chat part is basically just a dumb down version of the /api/chat endpoint. - const normalizedData = resp.matches.map((data) => ({ - ...data, - normalizedScore: - maxScore !== minScore - ? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98 - : 50, - })); + // This entire chat part is basically just a dumb down version of the /api/chat endpoint. + const normalizedData = resp.matches.map((data) => ({ + ...data, + normalizedScore: + maxScore !== minScore + ? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98 + : 50, + })); - const preparedContext = normalizedData.map( - ({ metadata, score, normalizedScore }) => ({ - context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, - score, - normalizedScore, - }), - ); + const preparedContext = normalizedData.map( + ({ metadata, score, normalizedScore }) => ({ + context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, + score, + normalizedScore, + }), + ); - const prompt = template({ - contexts: preparedContext, - question: query, - }); + const prompt = template({ + contexts: preparedContext, + question: query, + }); - const initialMessages: CoreMessage[] = [ - { - role: "system", - content: `You are an AI chatbot called "Supermemory.ai". When asked a question by a user, you must take all the context provided to you and give a good, small, but helpful response.`, - }, - { role: "assistant", content: "Hello, how can I help?" }, - ]; + const initialMessages: CoreMessage[] = [ + { + role: "system", + content: `You are an AI chatbot called "Supermemory.ai". When asked a question by a user, you must take all the context provided to you and give a good, small, but helpful response.`, + }, + { role: "assistant", content: "Hello, how can I help?" }, + ]; - const userMessage: CoreMessage = { role: "user", content: prompt }; + const userMessage: CoreMessage = { role: "user", content: prompt }; - const response = await generateText({ - model, - messages: [ - ...initialMessages, - ...((chatHistory || []) as CoreMessage[]), - userMessage, - ], - }); + const response = await generateText({ + model, + messages: [ + ...initialMessages, + ...((chatHistory || []) as CoreMessage[]), + userMessage, + ], + }); - return c.json({ status: "ok", response: response.text }); - } - }, + return c.json({ status: "ok", response: response.text }); + } + }, ); /* TODO: Eventually, we should not have to save each user's content in a seperate vector. @@ -378,195 +382,195 @@ But, it's not scalable *enough*. How can we store the same vectors for the same Hard problem to solve, Vectorize doesn't have an OR filter, so we can't just filter by URL and user. */ app.post( - "/api/chat", - zValidator( - "query", - z.object({ - query: z.string(), - user: z.string(), - topK: z.number().optional().default(10), - spaces: z.string().optional(), - sourcesOnly: z.string().optional().default("false"), - model: z.string().optional().default("gpt-4o"), - }), - ), - zValidator("json", chatObj), - async (c) => { - const query = c.req.valid("query"); - const body = c.req.valid("json"); + "/api/chat", + zValidator( + "query", + z.object({ + query: z.string(), + user: z.string(), + topK: z.number().optional().default(10), + spaces: z.string().optional(), + sourcesOnly: z.string().optional().default("false"), + model: z.string().optional().default("gpt-4o"), + }), + ), + zValidator("json", chatObj), + async (c) => { + const query = c.req.valid("query"); + const body = c.req.valid("json"); - const sourcesOnly = query.sourcesOnly === "true"; + const sourcesOnly = query.sourcesOnly === "true"; - // Return early for dumb requests - if (sourcesOnly && body.sources) { - return c.json(body.sources); - } + // Return early for dumb requests + if (sourcesOnly && body.sources) { + return c.json(body.sources); + } - const spaces = query.spaces?.split(",") ?? [undefined]; - console.log(spaces); + const spaces = query.spaces?.split(",") ?? [undefined]; + console.log(spaces); - // Get the AI model maker and vector store - const { model, store } = await initQuery(c, query.model); + // Get the AI model maker and vector store + const { model, store } = await initQuery(c, query.model); - if (!body.sources) { - const filter: VectorizeVectorMetadataFilter = { - [`user-${query.user}`]: 1, - }; - console.log("Spaces", spaces); + if (!body.sources) { + const filter: VectorizeVectorMetadataFilter = { + [`user-${query.user}`]: 1, + }; + console.log("Spaces", spaces); - // Converting the query to a vector so that we can search for similar vectors - const queryAsVector = await store.embeddings.embedQuery(query.query); - const responses: VectorizeMatches = { matches: [], count: 0 }; + // Converting the query to a vector so that we can search for similar vectors + const queryAsVector = await store.embeddings.embedQuery(query.query); + const responses: VectorizeMatches = { matches: [], count: 0 }; - console.log("hello world", spaces); + console.log("hello world", spaces); - // SLICED to 5 to avoid too many queries - for (const space of spaces.slice(0, 5)) { - if (space && space.length >= 1) { - // it's possible for space list to be [undefined] so we only add space filter conditionally - filter[`space-${query.user}-${space}`] = 1; - } + // SLICED to 5 to avoid too many queries + for (const space of spaces.slice(0, 5)) { + if (space && space.length >= 1) { + // it's possible for space list to be [undefined] so we only add space filter conditionally + filter[`space-${query.user}-${space}`] = 1; + } - // Because there's no OR operator in the filter, we have to make multiple queries - const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { - topK: query.topK, - filter, - returnMetadata: true, - }); + // Because there's no OR operator in the filter, we have to make multiple queries + const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { + topK: query.topK, + filter, + returnMetadata: true, + }); - // Basically recreating the response object - if (resp.count > 0) { - responses.matches.push(...resp.matches); - responses.count += resp.count; - } - } + // Basically recreating the response object + if (resp.count > 0) { + responses.matches.push(...resp.matches); + responses.count += resp.count; + } + } - const minScore = Math.min(...responses.matches.map(({ score }) => score)); - const maxScore = Math.max(...responses.matches.map(({ score }) => score)); + const minScore = Math.min(...responses.matches.map(({ score }) => score)); + const maxScore = Math.max(...responses.matches.map(({ score }) => score)); - // We are "normalising" the scores - if all of them are on top, we want to make sure that - // we have a way to filter out the noise. - const normalizedData = responses.matches.map((data) => ({ - ...data, - normalizedScore: - maxScore !== minScore - ? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98 - : 50, // If all scores are the same, set them to the middle of the scale - })); + // We are "normalising" the scores - if all of them are on top, we want to make sure that + // we have a way to filter out the noise. + const normalizedData = responses.matches.map((data) => ({ + ...data, + normalizedScore: + maxScore !== minScore + ? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98 + : 50, // If all scores are the same, set them to the middle of the scale + })); - let highScoreData = normalizedData.filter( - ({ normalizedScore }) => normalizedScore > 50, - ); + let highScoreData = normalizedData.filter( + ({ normalizedScore }) => normalizedScore > 50, + ); - // If the normalsation is not done properly, we have a fallback to just get the - // top 3 scores - if (highScoreData.length === 0) { - highScoreData = normalizedData - .sort((a, b) => b.score - a.score) - .slice(0, 3); - } + // If the normalsation is not done properly, we have a fallback to just get the + // top 3 scores + if (highScoreData.length === 0) { + highScoreData = normalizedData + .sort((a, b) => b.score - a.score) + .slice(0, 3); + } - const sortedHighScoreData = highScoreData.sort( - (a, b) => b.normalizedScore - a.normalizedScore, - ); + const sortedHighScoreData = highScoreData.sort( + (a, b) => b.normalizedScore - a.normalizedScore, + ); - body.sources = { - normalizedData, - }; + body.sources = { + normalizedData, + }; - // So this is kinda hacky, but the frontend needs to do 2 calls to get sources and chat. - // I think this is fine for now, but we can improve this later. - if (sourcesOnly) { - const idsAsStrings = sortedHighScoreData.map((dataPoint) => - dataPoint.id.toString(), - ); + // So this is kinda hacky, but the frontend needs to do 2 calls to get sources and chat. + // I think this is fine for now, but we can improve this later. + if (sourcesOnly) { + const idsAsStrings = sortedHighScoreData.map((dataPoint) => + dataPoint.id.toString(), + ); - const storedContent = await Promise.all( - idsAsStrings.map(async (id) => await c.env.KV.get(id)), - ); + const storedContent = await Promise.all( + idsAsStrings.map(async (id) => await c.env.KV.get(id)), + ); - const metadata = normalizedData.map((datapoint) => datapoint.metadata); + const metadata = normalizedData.map((datapoint) => datapoint.metadata); - return c.json({ ids: storedContent, metadata, normalizedData }); - } - } + return c.json({ ids: storedContent, metadata, normalizedData }); + } + } - const preparedContext = body.sources.normalizedData.map( - ({ metadata, score, normalizedScore }) => ({ - context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, - score, - normalizedScore, - }), - ); + const preparedContext = body.sources.normalizedData.map( + ({ metadata, score, normalizedScore }) => ({ + context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, + score, + normalizedScore, + }), + ); - const initialMessages: CoreMessage[] = [ - { role: "user", content: systemPrompt }, - { role: "assistant", content: "Hello, how can I help?" }, - ]; + const initialMessages: CoreMessage[] = [ + { role: "user", content: systemPrompt }, + { role: "assistant", content: "Hello, how can I help?" }, + ]; - const prompt = template({ - contexts: preparedContext, - question: query.query, - }); + const prompt = template({ + contexts: preparedContext, + question: query.query, + }); - const userMessage: CoreMessage = { role: "user", content: prompt }; + const userMessage: CoreMessage = { role: "user", content: prompt }; - const response = await streamText({ - model: model, - messages: [ - ...initialMessages, - ...((body.chatHistory || []) as CoreMessage[]), - userMessage, - ], - // temperature: 0.4, - }); + const response = await streamText({ + model: model, + messages: [ + ...initialMessages, + ...((body.chatHistory || []) as CoreMessage[]), + userMessage, + ], + // temperature: 0.4, + }); - return response.toTextStreamResponse(); - }, + return response.toTextStreamResponse(); + }, ); app.delete( - "/api/delete", - zValidator( - "query", - z.object({ - websiteUrl: z.string(), - user: z.string(), - }), - ), - async (c) => { - const { websiteUrl, user } = c.req.valid("query"); + "/api/delete", + zValidator( + "query", + z.object({ + websiteUrl: z.string(), + user: z.string(), + }), + ), + async (c) => { + const { websiteUrl, user } = c.req.valid("query"); - const { store } = await initQuery(c); + const { store } = await initQuery(c); - await deleteDocument({ url: websiteUrl, user, c, store }); + await deleteDocument({ url: websiteUrl, user, c, store }); - return c.json({ message: "Document deleted" }); - }, + return c.json({ message: "Document deleted" }); + }, ); // ERROR #1 - this is the api that the editor uses, it is just a scrape off of /api/chat so you may check that out app.get( - "/api/editorai", - zValidator( - "query", - z.object({ - context: z.string(), - request: z.string(), - }), - ), - async (c) => { - const { context, request } = c.req.valid("query"); - const { model } = await initQuery(c); + "/api/editorai", + zValidator( + "query", + z.object({ + context: z.string(), + request: z.string(), + }), + ), + async (c) => { + const { context, request } = c.req.valid("query"); + const { model } = await initQuery(c); - const response = await streamText({ - model, - prompt: `${request}-${context}`, - maxTokens: 224, - }); + const response = await streamText({ + model, + prompt: `${request}-${context}`, + maxTokens: 224, + }); - return response.toTextStreamResponse(); - }, + return response.toTextStreamResponse(); + }, ); export default app; diff --git a/apps/cf-ai-backend/src/prompts/prompt1.ts b/apps/cf-ai-backend/src/prompts/prompt1.ts index e52fe2cd..e311d575 100644 --- a/apps/cf-ai-backend/src/prompts/prompt1.ts +++ b/apps/cf-ai-backend/src/prompts/prompt1.ts @@ -15,19 +15,19 @@ To generate your answer: If no context is provided, introduce yourself and explain that the user can save content which will allow you to answer questions about that content in the future. Do not provide an answer if no context is provided.`; export const template = ({ contexts, question }) => { - // Map over contexts to generate the context and score parts - const contextParts = contexts - .map( - ({ context, normalisedScore }) => ` + // Map over contexts to generate the context and score parts + const contextParts = contexts + .map( + ({ context, normalisedScore }) => ` ${context ? ` ${context} ` : ""} ${normalisedScore ? ` normalisedScore: ${normalisedScore} ` : ""}`, - ) - .join("\n"); + ) + .join("\n"); - // Construct the final prompt using a template literal - const finalPrompt = ` + // Construct the final prompt using a template literal + const finalPrompt = ` Here's the given context and question for the task: ${contextParts} @@ -38,5 +38,5 @@ export const template = ({ contexts, question }) => { `; - return finalPrompt.trim(); + return finalPrompt.trim(); }; diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index dc97777c..4db568a1 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -2,51 +2,51 @@ import { sourcesZod } from "@repo/shared-types"; import { z } from "zod"; export type Env = { - VECTORIZE_INDEX: VectorizeIndex; - AI: Ai; - SECURITY_KEY: string; - OPENAI_API_KEY: string; - GOOGLE_AI_API_KEY: string; - MY_QUEUE: Queue; - KV: KVNamespace; - MYBROWSER: unknown; - ANTHROPIC_API_KEY: string; - NODE_ENV: string; + VECTORIZE_INDEX: VectorizeIndex; + AI: Ai; + SECURITY_KEY: string; + OPENAI_API_KEY: string; + GOOGLE_AI_API_KEY: string; + MY_QUEUE: Queue; + KV: KVNamespace; + MYBROWSER: unknown; + ANTHROPIC_API_KEY: string; + NODE_ENV: string; }; export interface TweetData { - tweetText: string; - postUrl: string; - authorName: string; - handle: string; - time: string; - saveToUser: string; + tweetText: string; + postUrl: string; + authorName: string; + handle: string; + time: string; + saveToUser: string; } export const contentObj = z.object({ - role: z.string(), - parts: z - .array( - z.object({ - text: z.string(), - }), - ) - .transform((val) => val.map((v) => v.text)) - .optional(), - content: z.string().optional(), + role: z.string(), + parts: z + .array( + z.object({ + text: z.string(), + }), + ) + .transform((val) => val.map((v) => v.text)) + .optional(), + content: z.string().optional(), }); export const chatObj = z.object({ - chatHistory: z.array(contentObj).optional(), - sources: sourcesZod.optional(), + chatHistory: z.array(contentObj).optional(), + sources: sourcesZod.optional(), }); export const vectorObj = z.object({ - pageContent: z.string(), - title: z.string().optional(), - description: z.string().optional(), - spaces: z.array(z.string()).optional(), - url: z.string(), - user: z.string(), - type: z.string().optional().default("page"), + pageContent: z.string(), + title: z.string().optional(), + description: z.string().optional(), + spaces: z.array(z.string()).optional(), + url: z.string(), + user: z.string(), + type: z.string().optional().default("page"), }); diff --git a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts index 8364ed0d..4a6ff593 100644 --- a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts +++ b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts @@ -1,58 +1,58 @@ import { z } from "zod"; interface OpenAIEmbeddingsParams { - apiKey: string; - modelName: string; + apiKey: string; + modelName: string; } export class OpenAIEmbeddings { - private apiKey: string; - private modelName: string; + private apiKey: string; + private modelName: string; - constructor({ apiKey, modelName }: OpenAIEmbeddingsParams) { - this.apiKey = apiKey; - this.modelName = modelName; - } + constructor({ apiKey, modelName }: OpenAIEmbeddingsParams) { + this.apiKey = apiKey; + this.modelName = modelName; + } - async embedDocuments(texts: string[]): Promise { - const responses = await Promise.all( - texts.map((text) => this.embedQuery(text)), - ); - return responses; - } + async embedDocuments(texts: string[]): Promise { + const responses = await Promise.all( + texts.map((text) => this.embedQuery(text)), + ); + return responses; + } - async embedQuery(text: string): Promise { - const response = await fetch( - "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/openai/embeddings", - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${this.apiKey}`, - }, - body: JSON.stringify({ - input: text, - model: this.modelName, - }), - }, - ); + async embedQuery(text: string): Promise { + const response = await fetch( + "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/openai/embeddings", + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + }, + body: JSON.stringify({ + input: text, + model: this.modelName, + }), + }, + ); - const data = await response.json(); + const data = await response.json(); - const zodTypeExpected = z.object({ - data: z.array( - z.object({ - embedding: z.array(z.number()), - }), - ), - }); + const zodTypeExpected = z.object({ + data: z.array( + z.object({ + embedding: z.array(z.number()), + }), + ), + }); - const json = zodTypeExpected.safeParse(data); + const json = zodTypeExpected.safeParse(data); - if (!json.success) { - throw new Error("Invalid response from OpenAI: " + json.error.message); - } + if (!json.success) { + throw new Error("Invalid response from OpenAI: " + json.error.message); + } - return json.data.data[0].embedding; - } + return json.data.data[0].embedding; + } } diff --git a/apps/cf-ai-backend/src/utils/chonker.ts b/apps/cf-ai-backend/src/utils/chonker.ts index c63020be..18788dab 100644 --- a/apps/cf-ai-backend/src/utils/chonker.ts +++ b/apps/cf-ai-backend/src/utils/chonker.ts @@ -4,44 +4,44 @@ import nlp from "compromise"; * Split text into chunks of specified max size with some overlap for continuity. */ export default function chunkText( - text: string, - maxChunkSize: number, - overlap: number = 0.2, + text: string, + maxChunkSize: number, + overlap: number = 0.2, ): string[] { - const sentences = nlp(text).sentences().out("array"); - const chunks = []; - let currentChunk: string[] = []; - let currentSize = 0; + const sentences = nlp(text).sentences().out("array"); + const chunks = []; + let currentChunk: string[] = []; + let currentSize = 0; - for (let i = 0; i < sentences.length; i++) { - const sentence = sentences[i]; - currentChunk.push(sentence); - currentSize += sentence.length; + for (let i = 0; i < sentences.length; i++) { + const sentence = sentences[i]; + currentChunk.push(sentence); + currentSize += sentence.length; - if (currentSize >= maxChunkSize) { - // Calculate overlap - const overlapSize = Math.floor(currentChunk.length * overlap); - const chunkText = currentChunk.join(" "); - chunks.push({ - text: chunkText, - start: i - currentChunk.length + 1, - end: i, - }); + if (currentSize >= maxChunkSize) { + // Calculate overlap + const overlapSize = Math.floor(currentChunk.length * overlap); + const chunkText = currentChunk.join(" "); + chunks.push({ + text: chunkText, + start: i - currentChunk.length + 1, + end: i, + }); - // Prepare the next chunk with overlap - currentChunk = currentChunk.slice(-overlapSize); - currentSize = currentChunk.reduce((sum, s) => sum + s.length, 0); - } - } + // Prepare the next chunk with overlap + currentChunk = currentChunk.slice(-overlapSize); + currentSize = currentChunk.reduce((sum, s) => sum + s.length, 0); + } + } - if (currentChunk.length > 0) { - const chunkText = currentChunk.join(" "); - chunks.push({ - text: chunkText, - start: sentences.length - currentChunk.length, - end: sentences.length, - }); - } + if (currentChunk.length > 0) { + const chunkText = currentChunk.join(" "); + chunks.push({ + text: chunkText, + start: sentences.length - currentChunk.length, + end: sentences.length, + }); + } - return chunks.map((chunk) => chunk.text); + return chunks.map((chunk) => chunk.text); } diff --git a/apps/cf-ai-backend/src/utils/seededRandom.ts b/apps/cf-ai-backend/src/utils/seededRandom.ts index 9e315ee8..3e41a53f 100644 --- a/apps/cf-ai-backend/src/utils/seededRandom.ts +++ b/apps/cf-ai-backend/src/utils/seededRandom.ts @@ -5,21 +5,21 @@ import { MersenneTwister19937, integer } from "random-js"; * @param {string} seed - The input string to hash. */ function hashString(seed: string) { - let hash = 0; - for (let i = 0; i < seed.length; i++) { - const char = seed.charCodeAt(i); - hash = (hash << 5) - hash + char; - hash |= 0; // Convert to 32bit integer - } - return hash; + let hash = 0; + for (let i = 0; i < seed.length; i++) { + const char = seed.charCodeAt(i); + hash = (hash << 5) - hash + char; + hash |= 0; // Convert to 32bit integer + } + return hash; } /** * returns a funtion that generates same sequence of random numbers for a given seed between 0 and 1. */ export function seededRandom(seed: string) { - const seedHash = hashString(seed); - const engine = MersenneTwister19937.seed(seedHash); - return () => - integer(0, Number.MAX_SAFE_INTEGER)(engine) / Number.MAX_SAFE_INTEGER; + const seedHash = hashString(seed); + const engine = MersenneTwister19937.seed(seedHash); + return () => + integer(0, Number.MAX_SAFE_INTEGER)(engine) / Number.MAX_SAFE_INTEGER; } diff --git a/apps/cf-ai-backend/tsconfig.json b/apps/cf-ai-backend/tsconfig.json index fcdf6914..323a4de4 100644 --- a/apps/cf-ai-backend/tsconfig.json +++ b/apps/cf-ai-backend/tsconfig.json @@ -1,7 +1,7 @@ { - "compilerOptions": { - "lib": ["ES2020"], - "types": ["@cloudflare/workers-types"], - "downlevelIteration": true - } + "compilerOptions": { + "lib": ["ES2020"], + "types": ["@cloudflare/workers-types"], + "downlevelIteration": true + } } diff --git a/apps/cf-ai-backend/vite.config.ts b/apps/cf-ai-backend/vite.config.ts index b7647b0d..06136158 100644 --- a/apps/cf-ai-backend/vite.config.ts +++ b/apps/cf-ai-backend/vite.config.ts @@ -2,5 +2,5 @@ import { defineConfig } from "vite"; import honox from "honox/vite"; export default defineConfig({ - plugins: [honox()], + plugins: [honox()], }); diff --git a/apps/extension/background.ts b/apps/extension/background.ts index 406493f1..2641f93d 100644 --- a/apps/extension/background.ts +++ b/apps/extension/background.ts @@ -2,7 +2,7 @@ import { Tweet } from "react-tweet/api"; import { features, transformTweetData } from "./helpers"; const tweetToMd = (tweet: Tweet) => { - return `Tweet from @${tweet.user?.name ?? tweet.user?.screen_name ?? "Unknown"} + return `Tweet from @${tweet.user?.name ?? tweet.user?.screen_name ?? "Unknown"} ${tweet.text} Images: ${tweet.photos ? tweet.photos.map((photo) => photo.url).join(", ") : "none"} @@ -19,478 +19,478 @@ const BACKEND_URL = "https://supermemory.ai"; let lastTwitterFetch = 0; const batchImportAll = async (cursor = "", totalImported = 0) => { - chrome.storage.session.get(["cookie", "csrf", "auth"], (result) => { - if (!result.cookie || !result.csrf || !result.auth) { - console.log("cookie, csrf, or auth is missing"); - return; - } + chrome.storage.session.get(["cookie", "csrf", "auth"], (result) => { + if (!result.cookie || !result.csrf || !result.auth) { + console.log("cookie, csrf, or auth is missing"); + return; + } - const myHeaders = new Headers(); - myHeaders.append("Cookie", result.cookie); - myHeaders.append("X-Csrf-token", result.csrf); - myHeaders.append("Authorization", result.auth); + const myHeaders = new Headers(); + myHeaders.append("Cookie", result.cookie); + myHeaders.append("X-Csrf-token", result.csrf); + myHeaders.append("Authorization", result.auth); - const requestOptions: RequestInit = { - method: "GET", - headers: myHeaders, - redirect: "follow", - }; + const requestOptions: RequestInit = { + method: "GET", + headers: myHeaders, + redirect: "follow", + }; - const variables = { - count: 100, - cursor: cursor, - includePromotedContent: false, - }; + const variables = { + count: 100, + cursor: cursor, + includePromotedContent: false, + }; - const urlWithCursor = cursor - ? `${BOOKMARKS_URL}&variables=${encodeURIComponent(JSON.stringify(variables))}` - : BOOKMARKS_URL; + const urlWithCursor = cursor + ? `${BOOKMARKS_URL}&variables=${encodeURIComponent(JSON.stringify(variables))}` + : BOOKMARKS_URL; - fetch(urlWithCursor, requestOptions) - .then((response) => response.json()) - .then((data) => { - const tweets = getAllTweets(data); - let importedCount = 0; + fetch(urlWithCursor, requestOptions) + .then((response) => response.json()) + .then((data) => { + const tweets = getAllTweets(data); + let importedCount = 0; - for (const tweet of tweets) { - console.log(tweet); + for (const tweet of tweets) { + console.log(tweet); - const tweetMd = tweetToMd(tweet); - (async () => { - chrome.storage.local.get(["jwt"], ({ jwt }) => { - if (!jwt) { - console.error("No JWT found"); - return; - } - fetch(`${BACKEND_URL}/api/store`, { - method: "POST", - headers: { - Authorization: `Bearer ${jwt}`, - }, - body: JSON.stringify({ - pageContent: tweetMd, - url: `https://twitter.com/supermemoryai/status/${tweet.id_str}`, - title: `Tweet by ${tweet.user.name}`, - description: tweet.text.slice(0, 200), - type: "tweet", - }), - }).then(async (ers) => { - console.log(ers.status); - importedCount++; - totalImported++; - console.log(totalImported); - chrome.tabs.query( - { active: true, currentWindow: true }, - async function (tabs) { - if (tabs.length > 0) { - let currentTabId = tabs[0].id; + const tweetMd = tweetToMd(tweet); + (async () => { + chrome.storage.local.get(["jwt"], ({ jwt }) => { + if (!jwt) { + console.error("No JWT found"); + return; + } + fetch(`${BACKEND_URL}/api/store`, { + method: "POST", + headers: { + Authorization: `Bearer ${jwt}`, + }, + body: JSON.stringify({ + pageContent: tweetMd, + url: `https://twitter.com/supermemoryai/status/${tweet.id_str}`, + title: `Tweet by ${tweet.user.name}`, + description: tweet.text.slice(0, 200), + type: "tweet", + }), + }).then(async (ers) => { + console.log(ers.status); + importedCount++; + totalImported++; + console.log(totalImported); + chrome.tabs.query( + { active: true, currentWindow: true }, + async function (tabs) { + if (tabs.length > 0) { + let currentTabId = tabs[0].id; - if (!currentTabId) { - return; - } + if (!currentTabId) { + return; + } - await chrome.tabs.sendMessage(currentTabId, { - type: "import-update", - importedCount: totalImported, - }); - } - }, - ); - }); - }); - })(); - } + await chrome.tabs.sendMessage(currentTabId, { + type: "import-update", + importedCount: totalImported, + }); + } + }, + ); + }); + }); + })(); + } - console.log("tweets", tweets); - console.log("data", data); + console.log("tweets", tweets); + console.log("data", data); - const instructions = - data.data?.bookmark_timeline_v2?.timeline?.instructions; - const lastInstruction = instructions?.[0].entries.pop(); + const instructions = + data.data?.bookmark_timeline_v2?.timeline?.instructions; + const lastInstruction = instructions?.[0].entries.pop(); - if (lastInstruction?.entryId.startsWith("cursor-bottom-")) { - let nextCursor = lastInstruction?.content?.value; + if (lastInstruction?.entryId.startsWith("cursor-bottom-")) { + let nextCursor = lastInstruction?.content?.value; - if (!nextCursor) { - for (let i = instructions.length - 1; i >= 0; i--) { - if (instructions[i].entryId.startsWith("cursor-bottom-")) { - nextCursor = instructions[i].content.value; - break; - } - } - } + if (!nextCursor) { + for (let i = instructions.length - 1; i >= 0; i--) { + if (instructions[i].entryId.startsWith("cursor-bottom-")) { + nextCursor = instructions[i].content.value; + break; + } + } + } - if (nextCursor) { - batchImportAll(nextCursor, totalImported); // Recursively call with new cursor - } else { - console.log("All bookmarks imported"); + if (nextCursor) { + batchImportAll(nextCursor, totalImported); // Recursively call with new cursor + } else { + console.log("All bookmarks imported"); - chrome.tabs.query( - { active: true, currentWindow: true }, - async function (tabs) { - if (tabs.length > 0) { - let currentTabId = tabs[0].id; + chrome.tabs.query( + { active: true, currentWindow: true }, + async function (tabs) { + if (tabs.length > 0) { + let currentTabId = tabs[0].id; - if (!currentTabId) { - return; - } + if (!currentTabId) { + return; + } - await chrome.runtime.sendMessage({ - type: "import-done", - importedCount: totalImported, - }); - } - }, - ); - } - } else { - console.log("All bookmarks imported"); - // Send a "done" message to the content script - chrome.tabs.query( - { active: true, currentWindow: true }, - async function (tabs) { - if (tabs.length > 0) { - let currentTabId = tabs[0].id; + await chrome.runtime.sendMessage({ + type: "import-done", + importedCount: totalImported, + }); + } + }, + ); + } + } else { + console.log("All bookmarks imported"); + // Send a "done" message to the content script + chrome.tabs.query( + { active: true, currentWindow: true }, + async function (tabs) { + if (tabs.length > 0) { + let currentTabId = tabs[0].id; - if (!currentTabId) { - return; - } + if (!currentTabId) { + return; + } - await chrome.runtime.sendMessage({ - type: "import-done", - importedCount: totalImported, - }); - } - }, - ); - } - }) - .catch((error) => console.error(error)); - }); + await chrome.runtime.sendMessage({ + type: "import-done", + importedCount: totalImported, + }); + } + }, + ); + } + }) + .catch((error) => console.error(error)); + }); }; chrome.webRequest.onBeforeSendHeaders.addListener( - (details) => { - if ( - !(details.url.includes("x.com") || details.url.includes("twitter.com")) - ) { - return; - } - const authHeader = details.requestHeaders!.find( - (header) => header.name.toLowerCase() === "authorization", - ); - const auth = authHeader ? authHeader.value : ""; + (details) => { + if ( + !(details.url.includes("x.com") || details.url.includes("twitter.com")) + ) { + return; + } + const authHeader = details.requestHeaders!.find( + (header) => header.name.toLowerCase() === "authorization", + ); + const auth = authHeader ? authHeader.value : ""; - const cookieHeader = details.requestHeaders!.find( - (header) => header.name.toLowerCase() === "cookie", - ); - const cookie = cookieHeader ? cookieHeader.value : ""; + const cookieHeader = details.requestHeaders!.find( + (header) => header.name.toLowerCase() === "cookie", + ); + const cookie = cookieHeader ? cookieHeader.value : ""; - const csrfHeader = details.requestHeaders!.find( - (header) => header.name.toLowerCase() === "x-csrf-token", - ); - const csrf = csrfHeader ? csrfHeader.value : ""; + const csrfHeader = details.requestHeaders!.find( + (header) => header.name.toLowerCase() === "x-csrf-token", + ); + const csrf = csrfHeader ? csrfHeader.value : ""; - if (!auth || !cookie || !csrf) { - console.log("auth, cookie, or csrf is missing"); - return; - } - chrome.storage.session.set({ cookie, csrf, auth }); - chrome.storage.local.get(["twitterBookmarks"], (result) => { - console.log("twitterBookmarks", result.twitterBookmarks); - if (result.twitterBookmarks !== "true") { - console.log("twitterBookmarks is NOT true"); - } else { - if ( - !details.requestHeaders || - details.requestHeaders.length === 0 || - details.requestHeaders === undefined - ) { - return; - } + if (!auth || !cookie || !csrf) { + console.log("auth, cookie, or csrf is missing"); + return; + } + chrome.storage.session.set({ cookie, csrf, auth }); + chrome.storage.local.get(["twitterBookmarks"], (result) => { + console.log("twitterBookmarks", result.twitterBookmarks); + if (result.twitterBookmarks !== "true") { + console.log("twitterBookmarks is NOT true"); + } else { + if ( + !details.requestHeaders || + details.requestHeaders.length === 0 || + details.requestHeaders === undefined + ) { + return; + } - // Check cache first - chrome.storage.local.get(["lastFetch", "cachedData"], (result) => { - const now = new Date().getTime(); - if (result.lastFetch && now - result.lastFetch < 30 * 60 * 1000) { - // Cached data is less than 30 minutes old, use it - console.log("Using cached data"); - console.log(result.cachedData); - return; - } + // Check cache first + chrome.storage.local.get(["lastFetch", "cachedData"], (result) => { + const now = new Date().getTime(); + if (result.lastFetch && now - result.lastFetch < 30 * 60 * 1000) { + // Cached data is less than 30 minutes old, use it + console.log("Using cached data"); + console.log(result.cachedData); + return; + } - // No valid cache, proceed to fetch - const authHeader = details.requestHeaders!.find( - (header) => header.name.toLowerCase() === "authorization", - ); - const auth = authHeader ? authHeader.value : ""; + // No valid cache, proceed to fetch + const authHeader = details.requestHeaders!.find( + (header) => header.name.toLowerCase() === "authorization", + ); + const auth = authHeader ? authHeader.value : ""; - const cookieHeader = details.requestHeaders!.find( - (header) => header.name.toLowerCase() === "cookie", - ); - const cookie = cookieHeader ? cookieHeader.value : ""; + const cookieHeader = details.requestHeaders!.find( + (header) => header.name.toLowerCase() === "cookie", + ); + const cookie = cookieHeader ? cookieHeader.value : ""; - const csrfHeader = details.requestHeaders!.find( - (header) => header.name.toLowerCase() === "x-csrf-token", - ); - const csrf = csrfHeader ? csrfHeader.value : ""; + const csrfHeader = details.requestHeaders!.find( + (header) => header.name.toLowerCase() === "x-csrf-token", + ); + const csrf = csrfHeader ? csrfHeader.value : ""; - if (!auth || !cookie || !csrf) { - console.log("auth, cookie, or csrf is missing"); - return; - } - chrome.storage.session.set({ cookie, csrf, auth }); + if (!auth || !cookie || !csrf) { + console.log("auth, cookie, or csrf is missing"); + return; + } + chrome.storage.session.set({ cookie, csrf, auth }); - const myHeaders = new Headers(); - myHeaders.append("Cookie", cookie); - myHeaders.append("X-Csrf-token", csrf); - myHeaders.append("Authorization", auth); + const myHeaders = new Headers(); + myHeaders.append("Cookie", cookie); + myHeaders.append("X-Csrf-token", csrf); + myHeaders.append("Authorization", auth); - const requestOptions: RequestInit = { - method: "GET", - headers: myHeaders, - redirect: "follow", - }; + const requestOptions: RequestInit = { + method: "GET", + headers: myHeaders, + redirect: "follow", + }; - const variables = { - count: 200, - includePromotedContent: false, - }; + const variables = { + count: 200, + includePromotedContent: false, + }; - // only fetch once in 1 minute - if (now - lastTwitterFetch < 60 * 1000) { - console.log("Waiting for ratelimits"); - return; - } + // only fetch once in 1 minute + if (now - lastTwitterFetch < 60 * 1000) { + console.log("Waiting for ratelimits"); + return; + } - fetch( - `${BOOKMARKS_URL}&variables=${encodeURIComponent(JSON.stringify(variables))}`, - requestOptions, - ) - .then((response) => response.text()) - .then((result) => { - const tweets = getAllTweets(JSON.parse(result)); + fetch( + `${BOOKMARKS_URL}&variables=${encodeURIComponent(JSON.stringify(variables))}`, + requestOptions, + ) + .then((response) => response.text()) + .then((result) => { + const tweets = getAllTweets(JSON.parse(result)); - console.log("tweets", tweets); - // Cache the result along with the current timestamp - chrome.storage.local.set({ - lastFetch: new Date().getTime(), - cachedData: tweets, - }); + console.log("tweets", tweets); + // Cache the result along with the current timestamp + chrome.storage.local.set({ + lastFetch: new Date().getTime(), + cachedData: tweets, + }); - lastTwitterFetch = now; - }) - .catch((error) => console.error(error)); - }); - return; - } - }); - }, - { urls: ["*://x.com/*", "*://twitter.com/*"] }, - ["requestHeaders", "extraHeaders"], + lastTwitterFetch = now; + }) + .catch((error) => console.error(error)); + }); + return; + } + }); + }, + { urls: ["*://x.com/*", "*://twitter.com/*"] }, + ["requestHeaders", "extraHeaders"], ); const getAllTweets = (rawJson: any): Tweet[] => { - const entries = - rawJson?.data?.bookmark_timeline_v2?.timeline?.instructions[0]?.entries; + const entries = + rawJson?.data?.bookmark_timeline_v2?.timeline?.instructions[0]?.entries; - console.log("Entries: ", entries); + console.log("Entries: ", entries); - if (!entries) { - console.error("No entries found"); - return []; - } + if (!entries) { + console.error("No entries found"); + return []; + } - const tweets = entries - .map((entry: any) => transformTweetData(entry)) - .filter((tweet: Tweet | null) => tweet !== null) as Tweet[]; + const tweets = entries + .map((entry: any) => transformTweetData(entry)) + .filter((tweet: Tweet | null) => tweet !== null) as Tweet[]; - console.log(tweets); + console.log(tweets); - return tweets; + return tweets; }; chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { - console.log(request); - if (request.type === "getJwt") { - chrome.storage.local.get(["jwt"], ({ jwt }) => { - sendResponse({ jwt }); - }); + console.log(request); + if (request.type === "getJwt") { + chrome.storage.local.get(["jwt"], ({ jwt }) => { + sendResponse({ jwt }); + }); - return true; - } else if (request.type === "urlSave") { - const content = request.content; - const url = request.url; - const title = request.title; - const description = request.description; - const ogImage = request.ogImage; - const favicon = request.favicon; - console.log(request.content, request.url); + return true; + } else if (request.type === "urlSave") { + const content = request.content; + const url = request.url; + const title = request.title; + const description = request.description; + const ogImage = request.ogImage; + const favicon = request.favicon; + console.log(request.content, request.url); - (async () => { - chrome.storage.local.get(["jwt"], ({ jwt }) => { - if (!jwt) { - console.error("No JWT found"); - return; - } - fetch(`${BACKEND_URL}/api/store`, { - method: "POST", - headers: { - Authorization: `Bearer ${jwt}`, - }, - body: JSON.stringify({ - pageContent: content, - url: url + "#supermemory-user-" + Math.random(), - title, - spaces: request.spaces, - description, - ogImage, - image: favicon, - }), - }).then((ers) => console.log(ers.status)); - }); - })(); - } else if (request.type === "batchImportAll") { - batchImportAll(); - return true; - } + (async () => { + chrome.storage.local.get(["jwt"], ({ jwt }) => { + if (!jwt) { + console.error("No JWT found"); + return; + } + fetch(`${BACKEND_URL}/api/store`, { + method: "POST", + headers: { + Authorization: `Bearer ${jwt}`, + }, + body: JSON.stringify({ + pageContent: content, + url: url + "#supermemory-user-" + Math.random(), + title, + spaces: request.spaces, + description, + ogImage, + image: favicon, + }), + }).then((ers) => console.log(ers.status)); + }); + })(); + } else if (request.type === "batchImportAll") { + batchImportAll(); + return true; + } }); chrome.runtime.onInstalled.addListener(function (details) { - if (details.reason === "install") { - chrome.tabs.create({ - url: "https://supermemory.ai/signin?extension=true", - active: true, - }); - } + if (details.reason === "install") { + chrome.tabs.create({ + url: "https://supermemory.ai/signin?extension=true", + active: true, + }); + } }); chrome.runtime.onInstalled.addListener(() => { - chrome.contextMenus.create({ - id: "saveSelection", - title: "Save note to Supermemory", - contexts: ["selection"], - }); + chrome.contextMenus.create({ + id: "saveSelection", + title: "Save note to Supermemory", + contexts: ["selection"], + }); - chrome.contextMenus.create({ - id: "savePage", - title: "Save page to Supermemory", - contexts: ["page"], - }); + chrome.contextMenus.create({ + id: "savePage", + title: "Save page to Supermemory", + contexts: ["page"], + }); - // TODO - // chrome.contextMenus.create({ - // id: 'saveLink', - // title: 'Save link to Supermemory', - // contexts: ['link'], - // }); + // TODO + // chrome.contextMenus.create({ + // id: 'saveLink', + // title: 'Save link to Supermemory', + // contexts: ['link'], + // }); }); interface FetchDataParams { - content: string; - url: string; - title: string; - description: string; - ogImage: string; - favicon: string; - isExternalContent: boolean; // Indicates if the content is from an external API + content: string; + url: string; + title: string; + description: string; + ogImage: string; + favicon: string; + isExternalContent: boolean; // Indicates if the content is from an external API } const fetchData = ({ - content, - url, - title, - description, - ogImage, - favicon, - isExternalContent, + content, + url, + title, + description, + ogImage, + favicon, + isExternalContent, }: FetchDataParams) => { - // Construct the URL - const finalUrl = isExternalContent - ? url - : `${url}#supermemory-stuff-${Math.random()}`; + // Construct the URL + const finalUrl = isExternalContent + ? url + : `${url}#supermemory-stuff-${Math.random()}`; - // Construct the body - const body = JSON.stringify({ - pageContent: content, - url: finalUrl, - title, - spaces: [], - description, - ogImage, - image: favicon, - }); + // Construct the body + const body = JSON.stringify({ + pageContent: content, + url: finalUrl, + title, + spaces: [], + description, + ogImage, + image: favicon, + }); - // Make the fetch call - fetch(`${BACKEND_URL}/api/store`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: body, - }) - .then((response) => { - console.log("Data saved successfully"); - }) - .catch((error) => { - console.error("Error saving data:", error); - }); + // Make the fetch call + fetch(`${BACKEND_URL}/api/store`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: body, + }) + .then((response) => { + console.log("Data saved successfully"); + }) + .catch((error) => { + console.error("Error saving data:", error); + }); - return Promise.resolve(); + return Promise.resolve(); }; chrome.contextMenus.onClicked.addListener((info, tab) => { - if (!tab || !tab.id) return; + if (!tab || !tab.id) return; - const tabId = tab.id; + const tabId = tab.id; - const sendMessageToTab = (message: string) => { - chrome.tabs.sendMessage(tabId, { message, type: "supermemory-message" }); - }; + const sendMessageToTab = (message: string) => { + chrome.tabs.sendMessage(tabId, { message, type: "supermemory-message" }); + }; - if (info.menuItemId === "saveSelection" && info.selectionText) { - sendMessageToTab("Saving selection..."); - fetchData({ - content: info.selectionText || "No content", - url: info.pageUrl, - title: tab.title || "Selection Title", - description: "User-selected content from the page", - ogImage: "", - favicon: "", - isExternalContent: false, - }) - .then(() => { - sendMessageToTab("Selection saved successfully."); - }) - .catch(() => { - sendMessageToTab("Failed to save selection."); - }); - } else if (info.menuItemId === "savePage") { - sendMessageToTab("Saving page..."); - chrome.scripting.executeScript( - { - target: { tabId: tabId }, - func: () => document.body.innerText, - }, - (results) => { - if (results.length > 0 && results[0].result) { - fetchData({ - content: results[0].result as string, - url: info.pageUrl, - title: tab.title || "Page Title", - description: "Full page content", - ogImage: "", - favicon: "", - isExternalContent: false, - }) - .then(() => { - sendMessageToTab("Page saved successfully."); - }) - .catch(() => { - sendMessageToTab("Failed to save page."); - }); - } - }, - ); - } + if (info.menuItemId === "saveSelection" && info.selectionText) { + sendMessageToTab("Saving selection..."); + fetchData({ + content: info.selectionText || "No content", + url: info.pageUrl, + title: tab.title || "Selection Title", + description: "User-selected content from the page", + ogImage: "", + favicon: "", + isExternalContent: false, + }) + .then(() => { + sendMessageToTab("Selection saved successfully."); + }) + .catch(() => { + sendMessageToTab("Failed to save selection."); + }); + } else if (info.menuItemId === "savePage") { + sendMessageToTab("Saving page..."); + chrome.scripting.executeScript( + { + target: { tabId: tabId }, + func: () => document.body.innerText, + }, + (results) => { + if (results.length > 0 && results[0].result) { + fetchData({ + content: results[0].result as string, + url: info.pageUrl, + title: tab.title || "Page Title", + description: "Full page content", + ogImage: "", + favicon: "", + isExternalContent: false, + }) + .then(() => { + sendMessageToTab("Page saved successfully."); + }) + .catch(() => { + sendMessageToTab("Failed to save page."); + }); + } + }, + ); + } }); diff --git a/apps/extension/components.json b/apps/extension/components.json index d3099362..7d301176 100644 --- a/apps/extension/components.json +++ b/apps/extension/components.json @@ -1,18 +1,18 @@ { - "$schema": "https://ui.shadcn.com/schema.json", - "style": "default", - "rsc": true, - "tsx": true, - "tailwind": { - "config": "tailwind.config.ts", - "css": "../../packages/tailwind-config/globals.css", - "baseColor": "stone", - "cssVariables": true, - "prefix": "" - }, - "aliases": { - "components": "content/ui/components", - "utils": "content/utils", - "ui": "content/ui/shadcn" - } + "$schema": "https://ui.shadcn.com/schema.json", + "style": "default", + "rsc": true, + "tsx": true, + "tailwind": { + "config": "tailwind.config.ts", + "css": "../../packages/tailwind-config/globals.css", + "baseColor": "stone", + "cssVariables": true, + "prefix": "" + }, + "aliases": { + "components": "content/ui/components", + "utils": "content/utils", + "ui": "content/ui/shadcn" + } } diff --git a/apps/extension/content/ContentApp.tsx b/apps/extension/content/ContentApp.tsx index d82857e2..8882e346 100644 --- a/apps/extension/content/ContentApp.tsx +++ b/apps/extension/content/ContentApp.tsx @@ -1,21 +1,21 @@ import React, { useEffect, useRef, useState } from "react"; import { Readability } from "@mozilla/readability"; import { - Tooltip, - TooltipContent, - TooltipProvider, - TooltipTrigger, + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, } from "./ui/shadcn/tooltip"; import { Popover, PopoverContent, PopoverTrigger } from "./ui/shadcn/popover"; import { Toaster } from "./ui/shadcn/toaster"; import { - Select, - SelectContent, - SelectGroup, - SelectItem, - SelectLabel, - SelectTrigger, - SelectValue, + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectLabel, + SelectTrigger, + SelectValue, } from "./ui/shadcn/select"; import { useToast } from "./ui/shadcn/use-toast"; import { Input } from "./ui/shadcn/input"; @@ -25,405 +25,405 @@ import { Textarea } from "./ui/shadcn/textarea"; const BACKEND_URL = "https://supermemory.ai"; export default function ContentApp({ - token, - shadowRoot, + token, + shadowRoot, }: { - token: string | undefined; - shadowRoot: ShadowRoot; + token: string | undefined; + shadowRoot: ShadowRoot; }) { - const [hover, setHover] = useState(false); + const [hover, setHover] = useState(false); - const { toast } = useToast(); + const { toast } = useToast(); - const [loading, setLoading] = useState(false); + const [loading, setLoading] = useState(false); - const [webNote, setWebNote] = useState(""); + const [webNote, setWebNote] = useState(""); - const [importedCount, setImportedCount] = useState(0); - const [isImporting, setIsImporting] = useState(false); - const [importDone, setImportDone] = useState(false); + const [importedCount, setImportedCount] = useState(0); + const [isImporting, setIsImporting] = useState(false); + const [importDone, setImportDone] = useState(false); - const [portalContainer, setPortalContainer] = useState( - null, - ); - const [isPopoverOpen, setIsPopoverOpen] = useState(false); - const [isPopover2Open, setIsPopover2Open] = useState(false); + const [portalContainer, setPortalContainer] = useState( + null, + ); + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + const [isPopover2Open, setIsPopover2Open] = useState(false); - const [spacesOptions, setSpacesOptions] = useState< - { id: number; name: string }[] - >([]); - const [selectedSpace, setSelectedSpace] = useState(); + const [spacesOptions, setSpacesOptions] = useState< + { id: number; name: string }[] + >([]); + const [selectedSpace, setSelectedSpace] = useState(); - const [userNotLoggedIn, setUserNotLoggedIn] = useState(false); + const [userNotLoggedIn, setUserNotLoggedIn] = useState(false); - const showLoginToast = async () => { - setUserNotLoggedIn(true); + const showLoginToast = async () => { + setUserNotLoggedIn(true); - const NOSHOW_TOAST = ["accounts.google.com", "supermemory.ai"]; + const NOSHOW_TOAST = ["accounts.google.com", "supermemory.ai"]; - const noLoginWarning = await chrome.storage.local.get("noLoginWarning"); - if (Object.keys(noLoginWarning).length > 0) { - return; - } + const noLoginWarning = await chrome.storage.local.get("noLoginWarning"); + if (Object.keys(noLoginWarning).length > 0) { + return; + } - if (!NOSHOW_TOAST.includes(window.location.host)) { - const t = toast({ - title: "Please login to supermemory.ai to use this extension.", - action: ( -
- + if (!NOSHOW_TOAST.includes(window.location.host)) { + const t = toast({ + title: "Please login to supermemory.ai to use this extension.", + action: ( +
+ - -
- ), - }); - } - }; + +
+ ), + }); + } + }; - useEffect(() => { - document.addEventListener("mousemove", (e) => { - const percentageX = (e.clientX / window.innerWidth) * 100; - const percentageY = (e.clientY / window.innerHeight) * 100; + useEffect(() => { + document.addEventListener("mousemove", (e) => { + const percentageX = (e.clientX / window.innerWidth) * 100; + const percentageY = (e.clientY / window.innerHeight) * 100; - if (percentageX > 75 && percentageY > 75) { - setHover(true); - } else { - setHover(false); - } - }); + if (percentageX > 75 && percentageY > 75) { + setHover(true); + } else { + setHover(false); + } + }); - const getUserData = () => { - chrome.runtime.sendMessage({ type: "getJwt" }); - }; + const getUserData = () => { + chrome.runtime.sendMessage({ type: "getJwt" }); + }; - getUserData(); + getUserData(); - chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { - if (request.type === "import-update") { - setIsImporting(true); - setImportedCount(request.importedCount); - } + chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { + if (request.type === "import-update") { + setIsImporting(true); + setImportedCount(request.importedCount); + } - if (request.type === "import-done") { - setIsImporting(false); - setImportDone(true); - } + if (request.type === "import-done") { + setIsImporting(false); + setImportDone(true); + } - if (request.type === "supermemory-message") { - toast({ - title: request.message, - }); - } - }); + if (request.type === "supermemory-message") { + toast({ + title: request.message, + }); + } + }); - const portalDiv = document.createElement("div"); - portalDiv.id = "popover-portal"; - shadowRoot.appendChild(portalDiv); - setPortalContainer(portalDiv); + const portalDiv = document.createElement("div"); + portalDiv.id = "popover-portal"; + shadowRoot.appendChild(portalDiv); + setPortalContainer(portalDiv); - const getSpaces = async () => { - const response = await fetch(`${BACKEND_URL}/api/spaces`, { - headers: { - Authorization: `Bearer ${token}`, - }, - }); + const getSpaces = async () => { + const response = await fetch(`${BACKEND_URL}/api/spaces`, { + headers: { + Authorization: `Bearer ${token}`, + }, + }); - if (response.status === 401) { - showLoginToast(); - return; - } + if (response.status === 401) { + showLoginToast(); + return; + } - try { - const data = await response.json(); - setSpacesOptions(data.data); - } catch (e) { - console.error( - `Error in supermemory.ai extension: ${e}. Please contact the developer https://x.com/dhravyashah`, - ); - } - }; + try { + const data = await response.json(); + setSpacesOptions(data.data); + } catch (e) { + console.error( + `Error in supermemory.ai extension: ${e}. Please contact the developer https://x.com/dhravyashah`, + ); + } + }; - getSpaces(); + getSpaces(); - return () => { - document.removeEventListener("mousemove", () => {}); - }; - }, []); + return () => { + document.removeEventListener("mousemove", () => {}); + }; + }, []); - async function sendUrlToAPI(spaces: string[]) { - setLoading(true); + async function sendUrlToAPI(spaces: string[]) { + setLoading(true); - setTimeout(() => { - setLoading(false); - }, 1500); + setTimeout(() => { + setLoading(false); + }, 1500); - // get the current URL - const url = window.location.href; + // get the current URL + const url = window.location.href; - const blacklist: string[] = []; - // check if the URL is blacklisted - if (blacklist.some((blacklisted) => url.includes(blacklisted))) { - return; - } else { - const clone = document.cloneNode(true) as Document; - const article = new Readability(clone).parse(); + const blacklist: string[] = []; + // check if the URL is blacklisted + if (blacklist.some((blacklisted) => url.includes(blacklisted))) { + return; + } else { + const clone = document.cloneNode(true) as Document; + const article = new Readability(clone).parse(); - const ogImage = document - .querySelector('meta[property="og:image"]') - ?.getAttribute("content"); + const ogImage = document + .querySelector('meta[property="og:image"]') + ?.getAttribute("content"); - const favicon = ( - document.querySelector('link[rel="icon"]') as HTMLLinkElement - )?.href; + const favicon = ( + document.querySelector('link[rel="icon"]') as HTMLLinkElement + )?.href; - setLoading(true); + setLoading(true); - setIsPopoverOpen(false); + setIsPopoverOpen(false); - await fetch(`${BACKEND_URL}/api/store`, { - method: "POST", - headers: { - Authorization: `Bearer ${token}`, - }, - body: JSON.stringify({ - pageContent: - (webNote ? `Note about this website: ${webNote}\n\n` : "") + - article?.textContent, - url: url + "#supermemory-user-" + Math.random(), - title: article?.title.slice(0, 500), - spaces: spaces, - description: article?.excerpt.slice(0, 250), - ogImage: ogImage?.slice(0, 1000), - image: favicon, - }), - }).then(async (rep) => { - if (rep.status === 401) { - showLoginToast(); - return; - } + await fetch(`${BACKEND_URL}/api/store`, { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ + pageContent: + (webNote ? `Note about this website: ${webNote}\n\n` : "") + + article?.textContent, + url: url + "#supermemory-user-" + Math.random(), + title: article?.title.slice(0, 500), + spaces: spaces, + description: article?.excerpt.slice(0, 250), + ogImage: ogImage?.slice(0, 1000), + image: favicon, + }), + }).then(async (rep) => { + if (rep.status === 401) { + showLoginToast(); + return; + } - const d = await rep.json(); + const d = await rep.json(); - if (rep.status === 200) { - toast({ - title: "Saved to supermemory.ai", - }); - } else { - toast({ - title: `Failed to save to supermemory.ai: ${d.error ?? "Unknown error"}`, - }); - } - setLoading(false); - return rep; - }); - } - } + if (rep.status === 200) { + toast({ + title: "Saved to supermemory.ai", + }); + } else { + toast({ + title: `Failed to save to supermemory.ai: ${d.error ?? "Unknown error"}`, + }); + } + setLoading(false); + return rep; + }); + } + } - if (!shadowRoot || !portalContainer) { - return null; - } + if (!shadowRoot || !portalContainer) { + return null; + } - return ( -
- + return ( +
+ - - - - - - - - - - - - {userNotLoggedIn ? ( - <>You need to login to use this extension. - ) : ( -

Add to supermemory.ai

- )} - - - - - {userNotLoggedIn ? ( -
- -
- ) : ( -
- + + + + + + + + + + + + {userNotLoggedIn ? ( + <>You need to login to use this extension. + ) : ( +

Add to supermemory.ai

+ )} + + + + + {userNotLoggedIn ? ( +
+ +
+ ) : ( +
+ - -