From f7cf59b84d6c0deed40f97ce098c7c07c5c48624 Mon Sep 17 00:00:00 2001 From: Kush Thaker Date: Thu, 8 Aug 2024 01:05:53 +0530 Subject: [PATCH 1/2] fix sapces; fix incorrect title for memory sources --- apps/cf-ai-backend/src/index.ts | 15 +----------- apps/cf-ai-backend/src/queueConsumer/index.ts | 24 +++++++++---------- apps/cf-ai-backend/src/types.ts | 2 +- 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 5957eb04..43143640 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -86,23 +86,12 @@ app.post("/api/add", zValidator("json", vectorBody), async (c) => { "#supermemory-user-" + body.user; - console.log( - "---------------------------------------------------------------------------------------------------------------------------------------------", - saveToDbUrl, - ); const alreadyExist = await db .select() .from(storedContent) .where(eq(storedContent.baseUrl, saveToDbUrl)); - console.log( - "------------------------------------------------", - JSON.stringify(alreadyExist), - ); if (alreadyExist.length > 0) { - console.log( - "------------------------------------------------------------------------------------------------I exist------------------------", - ); return c.json({ status: "error", message: "the content already exists" }); } @@ -129,11 +118,10 @@ app.post("/api/add", zValidator("json", vectorBody), async (c) => { // unique contraint check if (isWithinLimit) { - const spaceNumbers = body.spaces.map((s: string) => Number(s)); await c.env.EMBEDCHUNKS_QUEUE.send({ content: body.url, user: body.user, - space: spaceNumbers, + space: body.spaces, type: type, }); } else { @@ -620,7 +608,6 @@ app.post( ); const metadata = normalizedData.map((datapoint) => datapoint.metadata); - return c.json({ ids: storedContent.filter(Boolean), metadata, diff --git a/apps/cf-ai-backend/src/queueConsumer/index.ts b/apps/cf-ai-backend/src/queueConsumer/index.ts index 761aeb56..538eaff7 100644 --- a/apps/cf-ai-backend/src/queueConsumer/index.ts +++ b/apps/cf-ai-backend/src/queueConsumer/index.ts @@ -56,10 +56,9 @@ export async function queue( env: Env, ): Promise { const db = database(env); - console.log(env.CF_ACCOUNT_ID, env.CF_KV_AUTH_TOKEN); + for (let message of batch.messages) { - console.log(env.CF_ACCOUNT_ID, env.CF_KV_AUTH_TOKEN); - console.log("is thie even running?", message.body); + const body = message.body; const type = body.type; @@ -168,9 +167,9 @@ export async function queue( } case "tweet": { - console.log("tweet hit"); + const tweet = await getTweetData(body.content.split("/").pop()); - console.log(env.THREAD_CF_WORKER, env.THREAD_CF_AUTH); + const thread = await getThreadData({ tweetUrl: body.content, env: env, @@ -180,16 +179,18 @@ export async function queue( throw tweet.error; } pageContent = tweetToMd(tweet.value); - console.log(pageContent); + metadata = { baseUrl: body.content, description: tweet.value.text.slice(0, 200), image: tweet.value.user.profile_image_url_https, title: `Tweet by ${tweet.value.user.name}`, }; + console.log("this is the tweet metadata", metadata.title); if (isErr(thread)) { console.log("Thread worker is down!"); vectorData = JSON.stringify(pageContent); + console.log(vectorData); console.error(thread.error); } else { console.log("thread worker is fine"); @@ -216,13 +217,12 @@ export async function queue( }, { role: "user", - content: pageContent, + content: vectorData.replace(/.*?<\/raw>/g, ""), }, ], user_id: body.user, }), }); - // see what's up with the storedToSpaces in this block const { store } = await initQuery(env); @@ -235,7 +235,7 @@ export async function queue( type: type, url: metadata.baseUrl, description: metadata.description, - title: metadata.description, + title: metadata.title, }; try { @@ -268,6 +268,7 @@ export async function queue( (metadata.baseUrl.split("#supermemory-user-")[0] ?? metadata.baseUrl) + "#supermemory-user-" + body.user; + let contentId: number; const insertResponse = await wrap( @@ -303,9 +304,8 @@ export async function queue( }); throw insertResponse.error; } - console.log(JSON.stringify(insertResponse)); - contentId = insertResponse[0]?.id; - console.log("this is the content Id", contentId); + + contentId = insertResponse.value[0].id; if (storeToSpaces.length > 0) { // Adding the many-to-many relationship between content and spaces const spaceData = await wrap( diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index dd1b477a..93a548d2 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -95,7 +95,7 @@ export const vectorObj = z.object({ type: z.string().optional().default("page"), }); export const vectorBody = z.object({ - spaces: z.array(z.string()).optional(), + spaces: z.array(z.number()).optional(), url: z.string(), user: z.string(), }); From 8e893131de5c764f07685e2200a7c239c77e8920 Mon Sep 17 00:00:00 2001 From: Kush Thaker Date: Thu, 8 Aug 2024 01:09:57 +0530 Subject: [PATCH 2/2] remove unwanted console logs --- apps/cf-ai-backend/src/queueConsumer/index.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/cf-ai-backend/src/queueConsumer/index.ts b/apps/cf-ai-backend/src/queueConsumer/index.ts index 538eaff7..b6338a1a 100644 --- a/apps/cf-ai-backend/src/queueConsumer/index.ts +++ b/apps/cf-ai-backend/src/queueConsumer/index.ts @@ -58,7 +58,6 @@ export async function queue( const db = database(env); for (let message of batch.messages) { - const body = message.body; const type = body.type; @@ -167,7 +166,6 @@ export async function queue( } case "tweet": { - const tweet = await getTweetData(body.content.split("/").pop()); const thread = await getThreadData({ @@ -186,11 +184,10 @@ export async function queue( image: tweet.value.user.profile_image_url_https, title: `Tweet by ${tweet.value.user.name}`, }; - console.log("this is the tweet metadata", metadata.title); + if (isErr(thread)) { console.log("Thread worker is down!"); vectorData = JSON.stringify(pageContent); - console.log(vectorData); console.error(thread.error); } else { console.log("thread worker is fine");