diff --git a/.vscode/settings.json b/.vscode/settings.json index 61a3bb7c..15394cb2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,6 @@ "css.validate": false, "editor.quickSuggestions": { "strings": true - }, - "typescript.tsdk": "node_modules/typescript/lib" + }, + "typescript.tsdk": "node_modules/typescript/lib" } diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index c54dde9f..3a15ac4d 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -1,5 +1,5 @@ import { Context } from "hono"; -import { Env, vectorObj } from "./types"; +import { Env, vectorObj, Chunks } from "./types"; import { CloudflareVectorizeStore } from "@langchain/cloudflare"; import { OpenAIEmbeddings } from "./utils/OpenAIEmbedder"; import { createOpenAI } from "@ai-sdk/openai"; @@ -7,6 +7,7 @@ import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { createAnthropic } from "@ai-sdk/anthropic"; import { z } from "zod"; import { seededRandom } from "./utils/seededRandom"; +import { bulkInsertKv } from "./utils/kvBulkInsert"; export async function initQuery( c: Context<{ Bindings: Env }>, @@ -135,7 +136,7 @@ export async function batchCreateChunksAndEmbeddings({ }: { store: CloudflareVectorizeStore; body: z.infer; - chunks: string[]; + chunks: Chunks; context: Context<{ Bindings: Env }>; }) { //! NOTE that we use #supermemory-web to ensure that @@ -150,15 +151,25 @@ export async function batchCreateChunksAndEmbeddings({ const allIds = await context.env.KV.list({ prefix: uuid }); - let pageContent = ""; // If some chunks for that content already exist, we'll just update the metadata to include // the user. if (allIds.keys.length > 0) { const savedVectorIds = allIds.keys.map((key) => key.name); - const vectors = await context.env.VECTORIZE_INDEX.getByIds(savedVectorIds); - + const vectors = []; + //Search in a batch of 20 + for (let i = 0; i < savedVectorIds.length; i += 20) { + const batch = savedVectorIds.slice(i, i + 20); + const batchVectors = await context.env.VECTORIZE_INDEX.getByIds(batch); + vectors.push(...batchVectors); + } + console.log( + vectors.map((vector) => { + return vector.id; + }), + ); // Now, we'll update all vector metadatas with one more userId and all spaceIds const newVectors = vectors.map((vector) => { + console.log(JSON.stringify(vector.metadata)); vector.metadata = { ...vector.metadata, [`user-${body.user}`]: 1, @@ -169,51 +180,183 @@ export async function batchCreateChunksAndEmbeddings({ return acc; }, {}), }; - const content = - vector.metadata.content.toString().split("Content: ")[1] || - vector.metadata.content; - pageContent += `<---chunkId: ${vector.id}\n${content}\n---->`; return vector; }); - await context.env.VECTORIZE_INDEX.upsert(newVectors); - return pageContent; //Return the page content that goes to d1 db - } + // upsert in batch of 20 + const results = []; + for (let i = 0; i < newVectors.length; i += 20) { + results.push(newVectors.slice(i, i + 20)); + console.log(JSON.stringify(newVectors[1].id)); + } - for (let i = 0; i < chunks.length; i++) { - const chunk = chunks[i]; - const chunkId = `${uuid}-${i}`; - - const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${chunk}`; - - const docs = await store.addDocuments( - [ - { - pageContent: newPageContent, - metadata: { - title: body.title?.slice(0, 50) ?? "", - description: body.description ?? "", - url: body.url, - type: body.type ?? "page", - content: newPageContent, - - [sanitizeKey(`user-${body.user}`)]: 1, - ...body.spaces?.reduce((acc, space) => { - acc[`space-${body.user}-${space}`] = 1; - return acc; - }, {}), - }, - }, - ], - { - ids: [chunkId], - }, + await Promise.all( + results.map((result) => { + return context.env.VECTORIZE_INDEX.upsert(result); + }), ); - - console.log("Docs added: ", docs); - - await context.env.KV.put(chunkId, ourID); - pageContent += `<---chunkId: ${chunkId}\n${chunk}\n---->`; + return; } - return pageContent; // Return the pageContent that goes to the d1 db + + switch (chunks.type) { + case "tweet": + { + const commonMetaData = { + type: body.type ?? "tweet", + title: body.title, + description: body.description ?? "", + url: body.url, + [sanitizeKey(`user-${body.user}`)]: 1, + }; + + const spaceMetadata = body.spaces?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}); + + const ids = []; + const preparedDocuments = chunks.chunks + .map((tweet, i) => { + return tweet.chunkedTweet.map((chunk) => { + const id = `${uuid}-${i}`; + ids.push(id); + const { tweetLinks, tweetVids, tweetId, tweetImages } = + tweet.metadata; + return { + pageContent: chunk, + metadata: { + links: tweetLinks, + videos: tweetVids, + tweetId: tweetId, + tweetImages: tweetImages, + ...commonMetaData, + ...spaceMetadata, + }, + }; + }); + }) + .flat(); + + const docs = await store.addDocuments(preparedDocuments, { + ids: ids, + }); + console.log("these are the doucment ids", ids); + console.log("Docs added:", docs); + const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = + context.env; + await bulkInsertKv( + { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, + { chunkIds: ids, urlid: ourID }, + ); + } + break; + case "page": + { + const commonMetaData = { + type: body.type ?? "page", + title: body.title, + description: body.description ?? "", + url: body.url, + [sanitizeKey(`user-${body.user}`)]: 1, + }; + const spaceMetadata = body.spaces?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}); + + const ids = []; + const preparedDocuments = chunks.chunks.map((chunk, i) => { + const id = `${uuid}-${i}`; + ids.push(id); + return { + pageContent: chunk, + metadata: { + ...commonMetaData, + ...spaceMetadata, + }, + }; + }); + + const docs = await store.addDocuments(preparedDocuments, { ids: ids }); + console.log("Docs added:", docs); + const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = + context.env; + await bulkInsertKv( + { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, + { chunkIds: ids, urlid: ourID }, + ); + } + break; + case "note": + { + const commonMetaData = { + type: body.type ?? "page", + description: body.description ?? "", + url: body.url, + [sanitizeKey(`user-${body.user}`)]: 1, + }; + const spaceMetadata = body.spaces?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}); + + const ids = []; + const preparedDocuments = chunks.chunks.map((chunk, i) => { + const id = `${uuid}-${i}`; + ids.push(id); + return { + pageContent: chunk, + metadata: { + ...commonMetaData, + ...spaceMetadata, + }, + }; + }); + + const docs = await store.addDocuments(preparedDocuments, { ids: ids }); + console.log("Docs added:", docs); + const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = + context.env; + await bulkInsertKv( + { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, + { chunkIds: ids, urlid: ourID }, + ); + } + break; + case "image": { + const commonMetaData = { + type: body.type ?? "image", + title: body.title, + description: body.description ?? "", + url: body.url, + [sanitizeKey(`user-${body.user}`)]: 1, + }; + const spaceMetadata = body.spaces?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}); + + const ids = []; + const preparedDocuments = chunks.chunks.map((chunk, i) => { + const id = `${uuid}-${i}`; + ids.push(id); + return { + pageContent: chunk, + metadata: { + ...commonMetaData, + ...spaceMetadata, + }, + }; + }); + + const docs = await store.addDocuments(preparedDocuments, { ids: ids }); + console.log("Docs added:", docs); + const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = context.env; + await bulkInsertKv( + { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, + { chunkIds: ids, urlid: ourID }, + ); + } + } + return; } diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index edfe31c2..a3ac1380 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -1,7 +1,15 @@ import { z } from "zod"; import { Hono } from "hono"; import { CoreMessage, generateText, streamText, tool } from "ai"; -import { chatObj, Env, vectorObj } from "./types"; +import { + chatObj, + Chunks, + Env, + ImageChunks, + PageOrNoteChunks, + TweetChunks, + vectorObj, +} from "./types"; import { batchCreateChunksAndEmbeddings, deleteDocument, @@ -15,6 +23,8 @@ import { zValidator } from "@hono/zod-validator"; import chunkText from "./utils/chonker"; import { systemPrompt, template } from "./prompts/prompt1"; import { swaggerUI } from "@hono/swagger-ui"; +import { chunkThread } from "./utils/chunkTweet"; +import { chunkNote, chunkPage } from "./utils/chunkPageOrNotes"; const app = new Hono<{ Bindings: Env }>(); @@ -59,42 +69,42 @@ app.get("/api/health", (c) => { }); app.post("/api/add", zValidator("json", vectorObj), async (c) => { - const body = c.req.valid("json"); + try { + const body = c.req.valid("json"); - const { store } = await initQuery(c); + const { store } = await initQuery(c); - console.log(body.spaces); + console.log(body.spaces); + let chunks: TweetChunks | PageOrNoteChunks; + // remove everything in tags + const newPageContent = body.pageContent?.replace(/.*?<\/raw>/g, ""); - // remove everything in tags - const newPageContent = body.pageContent?.replace(/.*?<\/raw>/g, ""); + switch (body.type) { + case "tweet": + chunks = chunkThread(newPageContent); + break; - const chunks = chunkText(newPageContent, 1536); + case "page": + chunks = chunkPage(newPageContent); + break; - const chunksOf20 = chunks.reduce((acc, chunk, index) => { - if (index % 20 === 0) { - acc.push([chunk]); - } else { - acc[acc.length - 1].push(chunk); + case "note": + chunks = chunkNote(newPageContent); + break; } - return acc; - }, [] as string[][]); - const accumChunkedInputs = []; - - const promises = chunksOf20.map(async (chunkGroup) => { - const chunkedInput = await batchCreateChunksAndEmbeddings({ + await batchCreateChunksAndEmbeddings({ store, body, - chunks: chunkGroup, + chunks: chunks, context: c, }); - accumChunkedInputs.push(chunkedInput); - }); - - await Promise.all(promises); - - return c.json({ status: "ok", chunkedInput: accumChunkedInputs }); + return c.json({ status: "ok" }); + } catch (error) { + console.error("Error processing request:", error); + return c.json({ status: "error", message: error.message }, 500); + } }); app.post( @@ -147,6 +157,13 @@ app.post( ); const imageDescriptions = await Promise.all(imagePromises); + const chunks: ImageChunks = { + type: "image", + chunks: [ + imageDescriptions, + ...(body.text ? chunkText(body.text, 1536) : []), + ].flat(), + }; await batchCreateChunksAndEmbeddings({ store, @@ -162,10 +179,7 @@ app.post( pageContent: imageDescriptions.join("\n"), title: "Image content from the web", }, - chunks: [ - imageDescriptions, - ...(body.text ? chunkText(body.text, 1536) : []), - ].flat(), + chunks: chunks, context: c, }); @@ -263,7 +277,7 @@ app.post( // This is a "router". this finds out if the user wants to add a document, or chat with the AI to get a response. const routerQuery = await generateText({ model: model, - system: `You are Supermemory chatbot. You can either add a document to the supermemory database, or return a chat response. Based on this query, + system: `You are Supermemory chatbot. You can either add a document to the supermemory database, or return a chat response. Based on this query, You must determine what to do. Basically if it feels like a "question", then you should intiate a chat. If it feels like a "command" or feels like something that could be forwarded to the AI, then you should add a document. You must also extract the "thing" to add and what type of thing it is.`, prompt: `Question from user: ${query}`, @@ -289,7 +303,9 @@ app.post( if ((task as string) === "add") { // addString is the plaintext string that the user wants to add to the database + //chunk the note let addString: string = addContent; + let vectorContent: Chunks = chunkNote(addContent); if (thingToAdd === "page") { // TODO: Sometimes this query hangs, and errors out. we need to do proper error management here. @@ -300,6 +316,7 @@ app.post( }); addString = await response.text(); + vectorContent = chunkPage(addString); } // At this point, we can just go ahead and create the embeddings! @@ -312,7 +329,7 @@ app.post( pageContent: addString, title: `${addString.slice(0, 30)}... (Added from chatbot)`, }, - chunks: chunkText(addString, 1536), + chunks: vectorContent, context: c, }); diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index 4db568a1..5ef81f20 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -1,5 +1,6 @@ import { sourcesZod } from "@repo/shared-types"; import { z } from "zod"; +import { ThreadTweetData } from "./utils/chunkTweet"; export type Env = { VECTORIZE_INDEX: VectorizeIndex; @@ -7,6 +8,9 @@ export type Env = { SECURITY_KEY: string; OPENAI_API_KEY: string; GOOGLE_AI_API_KEY: string; + CF_KV_AUTH_TOKEN: string; + KV_NAMESPACE_ID: string; + CF_ACCOUNT_ID: string; MY_QUEUE: Queue; KV: KVNamespace; MYBROWSER: unknown; @@ -23,6 +27,32 @@ export interface TweetData { saveToUser: string; } +interface BaseChunks { + type: "tweet" | "page" | "note" | "image"; +} + +export interface TweetChunks extends BaseChunks { + type: "tweet"; + chunks: Array; +} + +export interface PageOrNoteChunks extends BaseChunks { + type: "page" | "note"; + chunks: string[]; +} +export interface ImageChunks extends BaseChunks { + type: "image"; + chunks: string[]; +} + +export type Chunks = TweetChunks | PageOrNoteChunks | ImageChunks; + +export interface KVBulkItem { + key: string; + value: string; + base64: boolean; +} + export const contentObj = z.object({ role: z.string(), parts: z diff --git a/apps/cf-ai-backend/src/utils/chunkPageOrNotes.ts b/apps/cf-ai-backend/src/utils/chunkPageOrNotes.ts new file mode 100644 index 00000000..f04ed0c5 --- /dev/null +++ b/apps/cf-ai-backend/src/utils/chunkPageOrNotes.ts @@ -0,0 +1,13 @@ +import chunkText from "./chonker"; +import { PageOrNoteChunks } from "../types"; +export function chunkPage(pageContent: string): PageOrNoteChunks { + const chunks = chunkText(pageContent, 1536); + + return { type: "page", chunks: chunks }; +} + +export function chunkNote(noteContent: string): PageOrNoteChunks { + const chunks = chunkText(noteContent, 1536); + + return { type: "note", chunks: chunks }; +} diff --git a/apps/cf-ai-backend/src/utils/chunkTweet.ts b/apps/cf-ai-backend/src/utils/chunkTweet.ts new file mode 100644 index 00000000..224c6c05 --- /dev/null +++ b/apps/cf-ai-backend/src/utils/chunkTweet.ts @@ -0,0 +1,40 @@ +import { TweetChunks } from "../types"; +import chunkText from "./chonker"; + +interface Tweet { + id: string; + text: string; + links: Array; + images: Array; + videos: Array; +} +interface Metadata { + tweetId: string; + tweetLinks: any[]; + tweetVids: any[]; + tweetImages: any[]; +} + +export interface ThreadTweetData { + chunkedTweet: string[]; + metadata: Metadata; +} + +export function chunkThread(threadText: string): TweetChunks { + const thread = JSON.parse(threadText); + + const chunkedTweets = thread.map((tweet: Tweet) => { + const chunkedTweet = chunkText(tweet.text, 1536); + + const metadata = { + tweetId: tweet.id, + tweetLinks: tweet.links, + tweetVids: tweet.videos, + tweetImages: tweet.images, + }; + + return { chunkedTweet, metadata }; + }); + + return { type: "tweet", chunks: chunkedTweets }; +} diff --git a/apps/cf-ai-backend/src/utils/kvBulkInsert.ts b/apps/cf-ai-backend/src/utils/kvBulkInsert.ts new file mode 100644 index 00000000..62236412 --- /dev/null +++ b/apps/cf-ai-backend/src/utils/kvBulkInsert.ts @@ -0,0 +1,43 @@ +import { KVBulkItem } from "../types"; + +export const bulkInsertKv = async ( + credentials: { + CF_KV_AUTH_TOKEN: string; + KV_NAMESPACE_ID: string; + CF_ACCOUNT_ID: string; + }, + keyData: { + chunkIds: Array; + urlid: string; + }, +) => { + const data: Array = keyData.chunkIds.map((chunkId) => ({ + key: chunkId, + value: keyData.urlid, + base64: false, + })); + + try { + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${credentials.CF_ACCOUNT_ID}/storage/kv/namespaces/${credentials.KV_NAMESPACE_ID}/bulk`, + { + method: "PUT", + headers: { + Authorization: `Bearer ${credentials.CF_KV_AUTH_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(data), + }, + ); + + if (!response.ok) { + throw new Error( + `can't insert bulk to kv because ${response.status} ${response.statusText} ${JSON.stringify(response.body)}`, + ); + } + return await response.json(); + } catch (e) { + //dosomething + throw e; + } +}; diff --git a/apps/web/app/(auth)/onboarding/page.tsx b/apps/web/app/(auth)/onboarding/page.tsx index b40042d1..10dd2325 100644 --- a/apps/web/app/(auth)/onboarding/page.tsx +++ b/apps/web/app/(auth)/onboarding/page.tsx @@ -24,7 +24,7 @@ export default function Home() { useEffect(() => { const updateDb = async () => { await completeOnboarding(); - } + }; if (currStep > 3) { updateDb().then(() => { push("/home?q=what%20is%20supermemory"); @@ -389,7 +389,7 @@ function Navbar() { const handleSkip = async () => { await completeOnboarding(); router.push("/home?q=what%20is%20supermemory"); - } + }; return (
@@ -399,7 +399,9 @@ function Navbar() { className="hover:brightness-125 duration-200 size-12" /> - +
); } diff --git a/apps/web/app/(dash)/header/signOutButton.tsx b/apps/web/app/(dash)/header/signOutButton.tsx index 3ef5fa2a..4c61c74d 100644 --- a/apps/web/app/(dash)/header/signOutButton.tsx +++ b/apps/web/app/(dash)/header/signOutButton.tsx @@ -5,13 +5,18 @@ export default function SignOutButton() { return (
{ - "use server" - await signOut() + "use server"; + await signOut(); }} > -
); -} \ No newline at end of file +} diff --git a/apps/web/app/actions/doers.ts b/apps/web/app/actions/doers.ts index 84cd4bfe..da2bfb5f 100644 --- a/apps/web/app/actions/doers.ts +++ b/apps/web/app/actions/doers.ts @@ -88,7 +88,7 @@ export const createSpace = async ( } }; -const typeDecider = (content: string) => { +const typeDecider = (content: string): "page" | "tweet" | "note" => { // if the content is a URL, then it's a page. if its a URL with https://x.com/user/status/123, then it's a tweet. else, it's a note. // do strict checking with regex if (content.match(/https?:\/\/(x\.com|twitter\.com)\/[\w]+\/[\w]+\/[\d]+/)) { @@ -199,6 +199,7 @@ export const createMemory = async (input: { let pageContent = input.content; let metadata: Awaited>; + let vectorData: string; if (!(await limit(data.user.id, type))) { return { @@ -217,7 +218,7 @@ export const createMemory = async (input: { }, }); pageContent = await response.text(); - + vectorData = pageContent; try { metadata = await getMetaData(input.content); } catch (e) { @@ -227,8 +228,42 @@ export const createMemory = async (input: { }; } } else if (type === "tweet") { + //Request the worker for the entire thread + + let thread: string; + let errorOccurred: boolean = false; + + try { + const cf_thread_endpoint = process.env.THREAD_CF_WORKER; + const authKey = process.env.THREAD_CF_AUTH; + + const threadRequest = await fetch(cf_thread_endpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: authKey, + }, + body: JSON.stringify({ url: input.content }), + }); + + if (threadRequest.status !== 200) { + throw new Error( + `Failed to fetch the thread: ${input.content}, Reason: ${threadRequest.statusText}`, + ); + } + + thread = await threadRequest.text(); + } catch (e) { + console.log("[THREAD FETCHING SERVICE] Failed to fetch the thread", e); + errorOccurred = true; + } + const tweet = await getTweetData(input.content.split("/").pop() as string); + pageContent = tweetToMd(tweet); + console.log("THis ishte page content!!", pageContent); + //@ts-ignore + vectorData = errorOccurred ? pageContent : thread; metadata = { baseUrl: input.content, description: tweet.text.slice(0, 200), @@ -237,6 +272,7 @@ export const createMemory = async (input: { }; } else if (type === "note") { pageContent = input.content; + vectorData = pageContent; noteId = new Date().getTime(); metadata = { baseUrl: `https://supermemory.ai/note/${noteId}`, @@ -263,7 +299,7 @@ export const createMemory = async (input: { { method: "POST", body: JSON.stringify({ - pageContent, + pageContent: vectorData, title: metadata.title, description: metadata.description, url: metadata.baseUrl, diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts index 9127261c..a14c96df 100644 --- a/apps/web/app/api/chat/route.ts +++ b/apps/web/app/api/chat/route.ts @@ -48,22 +48,22 @@ export async function POST(req: NextRequest) { // Only allow 5 requests per hour for each user, something lke this but this one is bad because chathistory.userid doesnt exist, we have to do a join and get it from the threads table const result = await db - .select({ - count: sql`count(*)`.mapWith(Number), - }) - .from(chatHistoryDb) - .innerJoin(chatThreads, eq(chatHistoryDb.threadId, chatThreads.id)) - .where( - and( - eq(chatThreads.userId, session.user.id), - gt(chatHistoryDb.createdAt, lastHour) - ) - ) - .execute(); + .select({ + count: sql`count(*)`.mapWith(Number), + }) + .from(chatHistoryDb) + .innerJoin(chatThreads, eq(chatHistoryDb.threadId, chatThreads.id)) + .where( + and( + eq(chatThreads.userId, session.user.id), + gt(chatHistoryDb.createdAt, lastHour), + ), + ) + .execute(); if (result[0]?.count && result[0]?.count >= 5) { // return new Response(`Too many requests ${result[0]?.count}`, { status: 429 }); - console.log(result[0]?.count) + console.log(result[0]?.count); } else { console.log("count", result); } diff --git a/apps/web/app/api/store/route.ts b/apps/web/app/api/store/route.ts index da393d2e..992c2a0e 100644 --- a/apps/web/app/api/store/route.ts +++ b/apps/web/app/api/store/route.ts @@ -30,7 +30,12 @@ const createMemoryFromAPI = async (input: { count: sql`count(*)`.mapWith(Number), }) .from(storedContent) - .where(and(gt(storedContent.savedAt, last2Hours), eq(storedContent.userId, input.userId))) + .where( + and( + gt(storedContent.savedAt, last2Hours), + eq(storedContent.userId, input.userId), + ), + ); if (numberOfItemsSavedInLast2Hours[0]!.count >= 20) { return { diff --git a/apps/web/cf-env.d.ts b/apps/web/cf-env.d.ts index ecf8f6a9..09b0690b 100644 --- a/apps/web/cf-env.d.ts +++ b/apps/web/cf-env.d.ts @@ -18,10 +18,13 @@ declare global { CLOUDFLARE_DATABASE_ID: string; CLOUDFLARE_D1_TOKEN: string; + THREAD_CF_WORKER: string; + THREAD_CF_AUTH: string; + MOBILE_TRUST_TOKEN: string; RATELIMITER: { - limit: ({key: string}) => {success: boolean} + limit: ({ key: string }) => { success: boolean }; }; } } diff --git a/apps/web/server/db/schema.ts b/apps/web/server/db/schema.ts index 35267eb8..32b80719 100644 --- a/apps/web/server/db/schema.ts +++ b/apps/web/server/db/schema.ts @@ -212,7 +212,9 @@ export const chatHistory = createTable( answer: text("answerParts"), // Single answer part as string answerSources: text("answerSources"), // JSON stringified array of objects answerJustification: text("answerJustification"), - createdAt: int("createdAt", { mode: "timestamp" }).notNull().default(new Date()), + createdAt: int("createdAt", { mode: "timestamp" }) + .notNull() + .default(new Date()), }, (history) => ({ threadIdx: index("chatHistory_thread_idx").on(history.threadId), diff --git a/package.json b/package.json index e10119d7..5a11d364 100644 --- a/package.json +++ b/package.json @@ -1,126 +1,126 @@ { - "name": "supermemory-ai", - "private": true, - "type": "module", - "scripts": { - "build": "turbo build", - "dev": "turbo dev", - "lint": "turbo lint", - "format": "prettier --write \"**/*.{ts,tsx,md}\"", - "deploy": "turbo deploy" - }, - "devDependencies": { - "@clack/prompts": "^0.7.0", - "@cloudflare/next-on-pages": "1", - "@cloudflare/workers-types": "^4.20240614.0", - "@repo/eslint-config": "*", - "@repo/shared-types": "*", - "@repo/tailwind-config": "*", - "@repo/typescript-config": "*", - "@repo/ui": "*", - "@tailwindcss/typography": "^0.5.13", - "@types/turndown": "^5.0.4", - "autoprefixer": "^10.4.19", - "drizzle-kit": "0.21.2", - "eslint-plugin-next-on-pages": "^1.11.3", - "lint-staged": "^15.2.5", - "postcss": "^8.4.38", - "prettier": "^3.3.3", - "readline-sync": "^1.4.10", - "tailwindcss": "^3.4.3", - "tailwindcss-animate": "^1.0.7", - "turbo": "2.0.3", - "vercel": "^34.2.0" - }, - "engines": { - "node": ">=18" - }, - "packageManager": "yarn@1.22.21", - "workspaces": [ - "apps/*", - "packages/*" - ], - "dependencies": { - "@ai-sdk/anthropic": "^0.0.15", - "@ai-sdk/google": "^0.0.15", - "@ai-sdk/openai": "^0.0.14", - "@auth/drizzle-adapter": "^1.1.0", - "@aws-sdk/client-s3": "^3.577.0", - "@aws-sdk/s3-request-presigner": "^3.577.0", - "@babel/plugin-transform-runtime": "^7.24.7", - "@cloudflare/puppeteer": "^0.0.11", - "@headlessui/react": "^2.0.4", - "@heroicons/react": "^2.1.4", - "@hono/swagger-ui": "^0.2.2", - "@hookform/resolvers": "^3.4.2", - "@iarna/toml": "^2.2.5", - "@langchain/cloudflare": "^0.0.6", - "@million/lint": "^1.0.0-rc.81", - "@mozilla/readability": "^0.5.0", - "@radix-ui/react-accordion": "^1.1.2", - "@radix-ui/react-dropdown-menu": "^2.1.1", - "@radix-ui/react-icons": "^1.3.0", - "@radix-ui/react-label": "^2.0.2", - "@radix-ui/react-popover": "^1.1.1", - "@radix-ui/react-progress": "^1.0.3", - "@radix-ui/react-scroll-area": "^1.0.5", - "@radix-ui/react-select": "^2.0.0", - "@radix-ui/react-separator": "^1.0.3", - "@radix-ui/react-slot": "^1.1.0", - "@radix-ui/react-switch": "^1.1.0", - "@radix-ui/react-tabs": "^1.0.4", - "@radix-ui/react-toast": "^1.1.5", - "@radix-ui/react-tooltip": "^1.1.2", - "@tldraw/assets": "^2.2.0", - "@types/jsonwebtoken": "^9.0.6", - "@types/react-responsive-masonry": "^2.1.3", - "@types/readline-sync": "^1.4.8", - "ai": "^3.1.14", - "aws4fetch": "^1.0.18", - "cheerio": "^1.0.0-rc.12", - "compromise": "^14.13.0", - "crypto-browserify": "^3.12.0", - "drizzle-orm": "0.30.0", - "eslint-config-turbo": "^2.0.6", - "framer-motion": "^11.2.6", - "geist": "^1.3.0", - "google-auth-library": "^9.11.0", - "grammy": "^1.25.1", - "http": "^0.0.1-security", - "https": "^1.0.0", - "jsonwebtoken": "^9.0.2", - "katex": "^0.16.10", - "lucide-react": "^0.379.0", - "million": "^3.1.11", - "next-app-theme": "^0.1.10", - "next-auth": "^5.0.0-beta.18", - "next-themes": "^0.3.0", - "random-js": "^2.1.0", - "react-dropzone": "^14.2.3", - "react-hook-form": "^7.51.5", - "react-layout-masonry": "^1.1.0", - "react-markdown": "^9.0.1", - "react-responsive-masonry": "^2.2.1", - "react-tweet": "^3.2.1", - "react-use-measure": "^2.1.1", - "react-web-share": "^2.0.2", - "rehype-highlight": "^7.0.0", - "rehype-katex": "^7.0.0", - "remark-gfm": "^4.0.0", - "remark-math": "^6.0.0", - "sonner": "^1.5.0", - "tailwind-scrollbar": "^3.1.0", - "tldraw": "^2.1.4", - "turndown": "^7.2.0", - "uploadthing": "^6.10.4", - "vaul": "^0.9.1", - "zod": "^3.23.8" - }, - "trustedDependencies": [ - "core-js-pure", - "es5-ext" - ], - "lint-staged": { - "**/*": "prettier --write --ignore-unknown" - } + "name": "supermemory-ai", + "private": true, + "type": "module", + "scripts": { + "build": "turbo build", + "dev": "turbo dev", + "lint": "turbo lint", + "format": "prettier --write \"**/*.{ts,tsx,md}\"", + "deploy": "turbo deploy" + }, + "devDependencies": { + "@clack/prompts": "^0.7.0", + "@cloudflare/next-on-pages": "1", + "@cloudflare/workers-types": "^4.20240614.0", + "@repo/eslint-config": "*", + "@repo/shared-types": "*", + "@repo/tailwind-config": "*", + "@repo/typescript-config": "*", + "@repo/ui": "*", + "@tailwindcss/typography": "^0.5.13", + "@types/turndown": "^5.0.4", + "autoprefixer": "^10.4.19", + "drizzle-kit": "0.21.2", + "eslint-plugin-next-on-pages": "^1.11.3", + "lint-staged": "^15.2.5", + "postcss": "^8.4.38", + "prettier": "^3.3.3", + "readline-sync": "^1.4.10", + "tailwindcss": "^3.4.3", + "tailwindcss-animate": "^1.0.7", + "turbo": "2.0.3", + "vercel": "^34.2.0" + }, + "engines": { + "node": ">=18" + }, + "packageManager": "yarn@1.22.21", + "workspaces": [ + "apps/*", + "packages/*" + ], + "dependencies": { + "@ai-sdk/anthropic": "^0.0.15", + "@ai-sdk/google": "^0.0.15", + "@ai-sdk/openai": "^0.0.14", + "@auth/drizzle-adapter": "^1.1.0", + "@aws-sdk/client-s3": "^3.577.0", + "@aws-sdk/s3-request-presigner": "^3.577.0", + "@babel/plugin-transform-runtime": "^7.24.7", + "@cloudflare/puppeteer": "^0.0.11", + "@headlessui/react": "^2.0.4", + "@heroicons/react": "^2.1.4", + "@hono/swagger-ui": "^0.2.2", + "@hookform/resolvers": "^3.4.2", + "@iarna/toml": "^2.2.5", + "@langchain/cloudflare": "^0.0.6", + "@million/lint": "^1.0.0-rc.81", + "@mozilla/readability": "^0.5.0", + "@radix-ui/react-accordion": "^1.1.2", + "@radix-ui/react-dropdown-menu": "^2.1.1", + "@radix-ui/react-icons": "^1.3.0", + "@radix-ui/react-label": "^2.0.2", + "@radix-ui/react-popover": "^1.1.1", + "@radix-ui/react-progress": "^1.0.3", + "@radix-ui/react-scroll-area": "^1.0.5", + "@radix-ui/react-select": "^2.0.0", + "@radix-ui/react-separator": "^1.0.3", + "@radix-ui/react-slot": "^1.1.0", + "@radix-ui/react-switch": "^1.1.0", + "@radix-ui/react-tabs": "^1.0.4", + "@radix-ui/react-toast": "^1.1.5", + "@radix-ui/react-tooltip": "^1.1.2", + "@tldraw/assets": "^2.2.0", + "@types/jsonwebtoken": "^9.0.6", + "@types/react-responsive-masonry": "^2.1.3", + "@types/readline-sync": "^1.4.8", + "ai": "^3.1.14", + "aws4fetch": "^1.0.18", + "cheerio": "^1.0.0-rc.12", + "compromise": "^14.13.0", + "crypto-browserify": "^3.12.0", + "drizzle-orm": "0.30.0", + "eslint-config-turbo": "^2.0.6", + "framer-motion": "^11.2.6", + "geist": "^1.3.0", + "google-auth-library": "^9.11.0", + "grammy": "^1.25.1", + "http": "^0.0.1-security", + "https": "^1.0.0", + "jsonwebtoken": "^9.0.2", + "katex": "^0.16.10", + "lucide-react": "^0.379.0", + "million": "^3.1.11", + "next-app-theme": "^0.1.10", + "next-auth": "^5.0.0-beta.18", + "next-themes": "^0.3.0", + "random-js": "^2.1.0", + "react-dropzone": "^14.2.3", + "react-hook-form": "^7.51.5", + "react-layout-masonry": "^1.1.0", + "react-markdown": "^9.0.1", + "react-responsive-masonry": "^2.2.1", + "react-tweet": "^3.2.1", + "react-use-measure": "^2.1.1", + "react-web-share": "^2.0.2", + "rehype-highlight": "^7.0.0", + "rehype-katex": "^7.0.0", + "remark-gfm": "^4.0.0", + "remark-math": "^6.0.0", + "sonner": "^1.5.0", + "tailwind-scrollbar": "^3.1.0", + "tldraw": "^2.1.4", + "turndown": "^7.2.0", + "uploadthing": "^6.10.4", + "vaul": "^0.9.1", + "zod": "^3.23.8" + }, + "trustedDependencies": [ + "core-js-pure", + "es5-ext" + ], + "lint-staged": { + "**/*": "prettier --write --ignore-unknown" + } } diff --git a/packages/ui/types.d.ts b/packages/ui/types.d.ts index 5783313c..51b04577 100644 --- a/packages/ui/types.d.ts +++ b/packages/ui/types.d.ts @@ -1,4 +1,4 @@ declare module "*.svg" { - const content: React.FunctionComponent>; - export default content; -} \ No newline at end of file + const content: React.FunctionComponent>; + export default content; +}