mirror of
https://github.com/supermemoryai/supermemory.git
synced 2026-10-10 03:28:14 +00:00
De duplication and updating chunks
This commit is contained in:
Dhravya Shah
parent
75fb461501
commit
12b2638209
20 changed files with 2095 additions and 231 deletions
|
|
@ -1,10 +0,0 @@
|
|||
import { defineConfig } from "drizzle-kit";
|
||||
|
||||
export default defineConfig({
|
||||
dialect: "postgresql",
|
||||
schema: "../../packages/db",
|
||||
out: "./drizzle",
|
||||
dbCredentials: {
|
||||
url: process.env.PROD_DATABASE_URL!,
|
||||
},
|
||||
});
|
||||
|
|
@ -6,6 +6,8 @@ config();
|
|||
|
||||
if (process.env.NODE_ENV !== "production" && !process.env.DATABASE_URL) {
|
||||
throw new Error("DATABASE_URL is not set");
|
||||
} else if (process.env.NODE_ENV === "production" && !process.env.PROD_DATABASE_URL) {
|
||||
throw new Error("PROD_DATABASE_URL is not set");
|
||||
}
|
||||
|
||||
export default defineConfig({
|
||||
|
|
@ -13,6 +15,6 @@ export default defineConfig({
|
|||
schema: "../../packages/db",
|
||||
out: "./drizzle",
|
||||
dbCredentials: {
|
||||
url: process.env.DATABASE_URL!,
|
||||
url: process.env.NODE_ENV === "production" ? process.env.PROD_DATABASE_URL! : process.env.DATABASE_URL!,
|
||||
},
|
||||
});
|
||||
|
|
|
|||
16
apps/backend/drizzle/0015_messy_karma.sql
Normal file
16
apps/backend/drizzle/0015_messy_karma.sql
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
CREATE TABLE IF NOT EXISTS "processed_content" (
|
||||
"id" serial PRIMARY KEY NOT NULL,
|
||||
"content_hash" text NOT NULL,
|
||||
"content" text NOT NULL,
|
||||
"is_successfully_processed" boolean DEFAULT false,
|
||||
"error_message" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "processed_content_content_hash_unique" UNIQUE("content_hash")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "chunks" ALTER COLUMN "text_content" SET NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "chunks" ADD COLUMN "content_hash" text;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "processed_content_hash_idx" ON "processed_content" USING btree ("content_hash");--> statement-breakpoint
|
||||
CREATE INDEX IF NOT EXISTS "chunk_content_hash_idx" ON "chunks" USING btree ("content_hash");--> statement-breakpoint
|
||||
CREATE INDEX IF NOT EXISTS "document_content_hash_idx" ON "documents" USING btree ("content_hash");
|
||||
1322
apps/backend/drizzle/meta/0015_snapshot.json
Normal file
1322
apps/backend/drizzle/meta/0015_snapshot.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -106,6 +106,13 @@
|
|||
"when": 1736852938881,
|
||||
"tag": "0014_mighty_the_captain",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 15,
|
||||
"version": "7",
|
||||
"when": 1737670732529,
|
||||
"tag": "0015_messy_karma",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,15 +1,19 @@
|
|||
{
|
||||
"name": "supermemory-backend",
|
||||
"name": "@supermemory/backend",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "bunx wrangler -v && wrangler dev",
|
||||
"deploy": "bunx wrangler deploy --minify",
|
||||
"generate-migration": "dotenv -- npx drizzle-kit generate",
|
||||
"migrate:local": "bun run ./scripts/migrate.ts",
|
||||
"tail": "bunx wrangler tail"
|
||||
"tail": "bunx wrangler tail",
|
||||
"test": "vitest",
|
||||
"test:watch": "vitest watch"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/google": "^0.0.51",
|
||||
"@ai-sdk/openai": "^0.0.70",
|
||||
"@cloudflare/vitest-pool-workers": "^0.6.6",
|
||||
"@hono/zod-validator": "^0.4.1",
|
||||
"@supermemory/db": "workspace:*",
|
||||
"ai": "4.0.16",
|
||||
|
|
@ -21,10 +25,12 @@
|
|||
"openai": "^4.68.4",
|
||||
"postgres": "^3.4.4",
|
||||
"uuid": "^11.0.1",
|
||||
"vitest": "2.1.8",
|
||||
"zod": "^3.23.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@cloudflare/workers-types": "^4.20240925.0"
|
||||
"@cloudflare/workers-types": "^4.20240925.0",
|
||||
"vitest-environment-miniflare": "^2.14.4"
|
||||
},
|
||||
"overrides": {
|
||||
"iron-webcrypto": "^1.2.1"
|
||||
|
|
|
|||
|
|
@ -93,7 +93,6 @@ export const auth = async (
|
|||
};
|
||||
|
||||
const session = await getSessionFromRequest(c.req.raw, context);
|
||||
console.log("Session", session);
|
||||
c.set("session", session);
|
||||
|
||||
if (session?.user?.id) {
|
||||
|
|
@ -129,7 +128,6 @@ export const auth = async (
|
|||
|
||||
user = Array.isArray(user) ? user[0] : user;
|
||||
c.set("user", user);
|
||||
console.log("User", user);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,6 +48,10 @@ export const app = new Hono<{ Variables: Variables; Bindings: Env }>()
|
|||
.use("/api/*", (c, next) => {
|
||||
const user = c.get("user");
|
||||
|
||||
if (c.env.NODE_ENV === "development") {
|
||||
return next();
|
||||
}
|
||||
|
||||
// RATELIMITS
|
||||
const rateLimitConfig = {
|
||||
// Endpoints that bypass rate limiting
|
||||
|
|
|
|||
|
|
@ -717,40 +717,6 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
|
|||
|
||||
const db = database(c.env.HYPERDRIVE.connectionString);
|
||||
|
||||
// Calculate document hash early to enable faster duplicate detection
|
||||
const content = body.prefetched?.contentToVectorize || body.content;
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(content);
|
||||
const hashBuffer = await crypto.subtle.digest("SHA-256", data);
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
const documentHash = hashArray
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
|
||||
// Check for duplicates using hash
|
||||
const existingDocs = await db
|
||||
.select()
|
||||
.from(documents)
|
||||
.where(
|
||||
and(
|
||||
eq(documents.userId, user.id),
|
||||
or(
|
||||
eq(documents.contentHash, documentHash),
|
||||
and(
|
||||
eq(documents.type, type.value),
|
||||
or(eq(documents.url, body.content), eq(documents.raw, content))
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
if (existingDocs.length > 0) {
|
||||
return c.json(
|
||||
{ error: `That ${type.value} already exists in your memories` },
|
||||
409
|
||||
);
|
||||
}
|
||||
|
||||
// Check space permissions if spaces are specified
|
||||
if (body.spaces && body.spaces.length > 0) {
|
||||
const spacePermissions = await Promise.all(
|
||||
|
|
@ -828,38 +794,90 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
|
|||
? body.content
|
||||
: `https://supermemory.ai/content/${contentId}`;
|
||||
|
||||
// Insert into documents table with hash
|
||||
// Insert minimal document record
|
||||
try {
|
||||
await db.insert(documents).values({
|
||||
uuid: contentId,
|
||||
userId: user.id,
|
||||
type: type.value,
|
||||
url: indexedUrl,
|
||||
title: body.prefetched?.title,
|
||||
description: body.prefetched?.description,
|
||||
ogImage: body.prefetched?.ogImage,
|
||||
contentHash: documentHash,
|
||||
raw:
|
||||
(body.prefetched ?? body.content) + "\n\n" + body.spaces?.join(" "),
|
||||
});
|
||||
// Check if document with same URL exists
|
||||
console.log(
|
||||
"[Add] Checking for existing document with URL:",
|
||||
indexedUrl
|
||||
);
|
||||
const existingDoc = await db
|
||||
.select()
|
||||
.from(documents)
|
||||
.where(
|
||||
and(
|
||||
eq(documents.userId, user.id),
|
||||
eq(documents.url, indexedUrl),
|
||||
sql`${documents.url} IS NOT NULL`
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
let documentId = contentId;
|
||||
|
||||
if (existingDoc.length > 0) {
|
||||
console.log("[Add] Found existing document:", {
|
||||
id: existingDoc[0].id,
|
||||
uuid: existingDoc[0].uuid,
|
||||
url: existingDoc[0].url,
|
||||
});
|
||||
documentId = existingDoc[0].uuid;
|
||||
// Update the raw content of existing document
|
||||
console.log("[Add] Updating existing document content");
|
||||
await db
|
||||
.update(documents)
|
||||
.set({
|
||||
raw:
|
||||
(body.prefetched ?? body.content) +
|
||||
"\n\n" +
|
||||
body.spaces?.join(" "),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(documents.id, existingDoc[0].id));
|
||||
console.log("[Add] Document updated successfully");
|
||||
} else {
|
||||
console.log("[Add] No existing document found, creating new one");
|
||||
// Insert new document
|
||||
await db.insert(documents).values({
|
||||
uuid: contentId,
|
||||
userId: user.id,
|
||||
type: type.value,
|
||||
url: indexedUrl,
|
||||
raw:
|
||||
(body.prefetched ?? body.content) +
|
||||
"\n\n" +
|
||||
body.spaces?.join(" "),
|
||||
});
|
||||
console.log("[Add] New document created successfully");
|
||||
}
|
||||
|
||||
console.log("[Add] Starting workflow with params:", {
|
||||
documentId,
|
||||
url: indexedUrl,
|
||||
isUpdate: existingDoc.length > 0,
|
||||
});
|
||||
// Start the workflow which will handle everything else
|
||||
await c.env.CONTENT_WORKFLOW.create({
|
||||
params: {
|
||||
userId: user.id,
|
||||
content: body.content,
|
||||
spaces: body.spaces,
|
||||
type: type.value,
|
||||
uuid: contentId,
|
||||
uuid: documentId,
|
||||
url: indexedUrl,
|
||||
prefetched: body.prefetched,
|
||||
},
|
||||
id: contentId,
|
||||
id: documentId,
|
||||
});
|
||||
|
||||
return c.json({
|
||||
message: "Content added successfully",
|
||||
id: contentId,
|
||||
message:
|
||||
existingDoc.length > 0
|
||||
? "Content update started"
|
||||
: "Content processing started",
|
||||
id: documentId,
|
||||
type: type.value,
|
||||
updated: existingDoc.length > 0,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Add Content Error]", error);
|
||||
|
|
@ -1054,6 +1072,57 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
|
|||
? content
|
||||
: `https://supermemory.ai/content/${contentId}`;
|
||||
|
||||
// Check for existing document with same URL
|
||||
const existingDoc = await db
|
||||
.select()
|
||||
.from(documents)
|
||||
.where(
|
||||
and(
|
||||
eq(documents.userId, user.id),
|
||||
eq(documents.url, url),
|
||||
sql`${documents.url} IS NOT NULL`
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
if (existingDoc.length > 0) {
|
||||
// Update existing document
|
||||
await db
|
||||
.update(documents)
|
||||
.set({
|
||||
title,
|
||||
contentHash: documentHash,
|
||||
raw: content + "\n\n" + spaces?.join(" "),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(documents.id, existingDoc[0].id));
|
||||
|
||||
// Create workflow for updating
|
||||
await c.env.CONTENT_WORKFLOW.create({
|
||||
params: {
|
||||
userId: user.id,
|
||||
content,
|
||||
spaces,
|
||||
type: type.value,
|
||||
uuid: existingDoc[0].uuid,
|
||||
url,
|
||||
},
|
||||
id: existingDoc[0].uuid,
|
||||
});
|
||||
|
||||
succeeded++;
|
||||
sendMessage({
|
||||
progress: Math.round((processed / total) * 100),
|
||||
status: "updated",
|
||||
title: typeof item === "string" ? item : item.title,
|
||||
processed,
|
||||
total,
|
||||
succeeded,
|
||||
failed,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Insert into documents table
|
||||
await db.insert(documents).values({
|
||||
uuid: contentId,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { Env } from "../types";
|
|||
|
||||
export const extractPageContent = async (content: string, env: Env) => {
|
||||
console.log("content", content);
|
||||
const resp = await fetch(`https://md.dhr.wtf?url=${content}`);
|
||||
const resp = await fetch(`https://md.dhr.wtf?url=${content}?nocache`);
|
||||
|
||||
if (!resp.ok) {
|
||||
throw new Error(
|
||||
|
|
@ -10,7 +10,7 @@ export const extractPageContent = async (content: string, env: Env) => {
|
|||
);
|
||||
}
|
||||
|
||||
const metadataResp = await fetch(`https://md.dhr.wtf/metadata?url=${content}`);
|
||||
const metadataResp = await fetch(`https://md.dhr.wtf/metadata?url=${content}?nocache`);
|
||||
|
||||
if (!metadataResp.ok) {
|
||||
throw new Error(
|
||||
|
|
|
|||
|
|
@ -6,17 +6,106 @@ import {
|
|||
import { Env, WorkflowParams } from "../types";
|
||||
import { fetchContent } from "../utils/fetchers";
|
||||
import chunkText from "../utils/chunkers";
|
||||
import { database, eq, inArray } from "@supermemory/db";
|
||||
import { database, eq, inArray, and, or, sql } from "@supermemory/db";
|
||||
import {
|
||||
ChunkInsert,
|
||||
contentToSpace,
|
||||
documents,
|
||||
spaces,
|
||||
chunk,
|
||||
Document,
|
||||
} from "@supermemory/db/schema";
|
||||
import { embedMany } from "ai";
|
||||
import { openai } from "../providers";
|
||||
import { chunk } from "@supermemory/db/schema";
|
||||
import { NonRetryableError } from "cloudflare:workflows";
|
||||
import { createHash } from "crypto";
|
||||
|
||||
// Helper function to generate content hash
|
||||
const generateHash = (content: string) => {
|
||||
return createHash("sha256").update(content).digest("hex");
|
||||
};
|
||||
|
||||
interface ChunkUpdate {
|
||||
oldChunk?: typeof chunk.$inferSelect;
|
||||
newContent?: string;
|
||||
orderInDocument: number;
|
||||
needsUpdate: boolean;
|
||||
}
|
||||
|
||||
// Helper function to determine which chunks need updates
|
||||
const analyzeContentChanges = async (
|
||||
oldContent: string,
|
||||
newContent: string,
|
||||
existingChunks: (typeof chunk.$inferSelect)[],
|
||||
chunkSize: number = 768
|
||||
): Promise<ChunkUpdate[]> => {
|
||||
// First, chunk the new content with size limits
|
||||
const newChunks = chunkText(newContent, chunkSize);
|
||||
const updates: ChunkUpdate[] = [];
|
||||
|
||||
// Map existing chunks for quick lookup
|
||||
const existingChunksMap = new Map(
|
||||
existingChunks.map((c) => [c.orderInDocument, c])
|
||||
);
|
||||
|
||||
// Track which old chunks have been processed
|
||||
const processedOldChunks = new Set<number>();
|
||||
|
||||
// Process new chunks and match with old ones
|
||||
let currentOrder = 0;
|
||||
for (const newChunkText of newChunks) {
|
||||
const oldChunk = existingChunksMap.get(currentOrder);
|
||||
const newChunkHash = generateHash(newChunkText);
|
||||
|
||||
if (oldChunk) {
|
||||
processedOldChunks.add(currentOrder);
|
||||
}
|
||||
|
||||
// If the new chunk is too large, we need to split it
|
||||
if (newChunkText.length > chunkSize) {
|
||||
// Re-chunk this specific piece to ensure it fits
|
||||
const subChunks = chunkText(newChunkText, chunkSize);
|
||||
|
||||
// Add each sub-chunk as a separate update
|
||||
for (let i = 0; i < subChunks.length; i++) {
|
||||
const subChunk = subChunks[i];
|
||||
const subChunkHash = generateHash(subChunk);
|
||||
|
||||
updates.push({
|
||||
oldChunk: i === 0 ? oldChunk : undefined, // Only use the old chunk for the first sub-chunk
|
||||
newContent: subChunk,
|
||||
orderInDocument: currentOrder + i,
|
||||
needsUpdate: true, // Always need to update since we split the chunk
|
||||
});
|
||||
}
|
||||
|
||||
currentOrder += subChunks.length;
|
||||
} else {
|
||||
// Normal case - chunk fits within size limit
|
||||
updates.push({
|
||||
oldChunk,
|
||||
newContent: newChunkText,
|
||||
orderInDocument: currentOrder,
|
||||
needsUpdate: !oldChunk || oldChunk.contentHash !== newChunkHash,
|
||||
});
|
||||
currentOrder++;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle any remaining old chunks that weren't processed
|
||||
for (const [order, oldChunk] of existingChunksMap) {
|
||||
if (!processedOldChunks.has(order)) {
|
||||
updates.push({
|
||||
oldChunk,
|
||||
orderInDocument: order,
|
||||
needsUpdate: true, // Mark for deletion since it wasn't used in new content
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sort updates by order to ensure proper sequence
|
||||
return updates.sort((a, b) => a.orderInDocument - b.orderInDocument);
|
||||
};
|
||||
|
||||
// TODO: handle errors properly here.
|
||||
|
||||
|
|
@ -24,16 +113,17 @@ export class ContentWorkflow extends WorkflowEntrypoint<Env, WorkflowParams> {
|
|||
async run(event: WorkflowEvent<WorkflowParams>, step: WorkflowStep) {
|
||||
// Step 0: Check if user has reached memory limit
|
||||
await step.do("check memory limit", async () => {
|
||||
const existingMemories = await database(this.env.HYPERDRIVE.connectionString)
|
||||
const existingMemories = await database(
|
||||
this.env.HYPERDRIVE.connectionString
|
||||
)
|
||||
.select()
|
||||
.from(documents)
|
||||
.where(eq(documents.userId, event.payload.userId));
|
||||
|
||||
if (existingMemories.length >= 2000) {
|
||||
await database(this.env.HYPERDRIVE.connectionString)
|
||||
.delete(documents)
|
||||
.where(eq(documents.uuid, event.payload.uuid));
|
||||
throw new NonRetryableError("You have reached the maximum limit of 2000 memories");
|
||||
throw new NonRetryableError(
|
||||
"You have reached the maximum limit of 2000 memories"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -53,29 +143,59 @@ export class ContentWorkflow extends WorkflowEntrypoint<Env, WorkflowParams> {
|
|||
throw new NonRetryableError("The content is too big (maximum 20 pages)");
|
||||
}
|
||||
|
||||
const chunked = await step.do("chunk content", async () =>
|
||||
chunkText(rawContent.contentToVectorize, 768)
|
||||
);
|
||||
// Generate content hash
|
||||
const contentHash = generateHash(rawContent.contentToVectorize);
|
||||
|
||||
// Step 2: Create the document in the database.
|
||||
const document = await step.do("create document", async () => {
|
||||
try {
|
||||
// First check if document exists
|
||||
const existingDoc = await database(this.env.HYPERDRIVE.connectionString)
|
||||
// Step 2: Check for existing document by URL
|
||||
const existingDocument = await step.do(
|
||||
"check existing document",
|
||||
async () => {
|
||||
if (!event.payload.url) return null;
|
||||
|
||||
console.log(
|
||||
"[Workflow] Checking for existing document with URL:",
|
||||
event.payload.url
|
||||
);
|
||||
const docs = await database(this.env.HYPERDRIVE.connectionString)
|
||||
.select()
|
||||
.from(documents)
|
||||
.where(eq(documents.uuid, event.payload.uuid))
|
||||
.where(
|
||||
and(
|
||||
eq(documents.userId, event.payload.userId),
|
||||
eq(documents.url, event.payload.url),
|
||||
sql`${documents.url} IS NOT NULL`
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
return await database(this.env.HYPERDRIVE.connectionString)
|
||||
.insert(documents)
|
||||
.values({
|
||||
userId: event.payload.userId,
|
||||
type: event.payload.type,
|
||||
uuid: event.payload.uuid,
|
||||
...(event.payload.url && { url: event.payload.url }),
|
||||
if (docs[0]) {
|
||||
console.log("[Workflow] Found existing document:", {
|
||||
id: docs[0].id,
|
||||
uuid: docs[0].uuid,
|
||||
url: docs[0].url,
|
||||
});
|
||||
} else {
|
||||
console.log("[Workflow] No existing document found for URL");
|
||||
}
|
||||
|
||||
return docs[0] || null;
|
||||
}
|
||||
);
|
||||
|
||||
// Step 3: Update or create document
|
||||
const document = await step.do("update or create document", async () => {
|
||||
const db = database(this.env.HYPERDRIVE.connectionString);
|
||||
|
||||
if (existingDocument) {
|
||||
console.log("[Workflow] Updating existing document:", {
|
||||
id: existingDocument.id,
|
||||
uuid: existingDocument.uuid,
|
||||
});
|
||||
// Update existing document
|
||||
await db
|
||||
.update(documents)
|
||||
.set({
|
||||
title: rawContent.title,
|
||||
content: rawContent.contentToSave,
|
||||
description:
|
||||
"description" in rawContent
|
||||
? (rawContent.description ?? "")
|
||||
|
|
@ -85,62 +205,56 @@ export class ContentWorkflow extends WorkflowEntrypoint<Env, WorkflowParams> {
|
|||
? (rawContent.image ?? "")
|
||||
: (event.payload.prefetched?.ogImage ?? undefined),
|
||||
raw: rawContent.contentToVectorize,
|
||||
content: rawContent.contentToSave,
|
||||
contentHash,
|
||||
isSuccessfullyProcessed: false,
|
||||
updatedAt: new Date(),
|
||||
...(event.payload.createdAt && {
|
||||
createdAt: new Date(event.payload.createdAt),
|
||||
}),
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: documents.uuid,
|
||||
set: {
|
||||
title: rawContent.title,
|
||||
content: rawContent.contentToSave,
|
||||
description:
|
||||
"description" in rawContent
|
||||
? (rawContent.description ?? "")
|
||||
: (event.payload.prefetched?.description ?? undefined),
|
||||
ogImage:
|
||||
"image" in rawContent
|
||||
? (rawContent.image ?? "")
|
||||
: (event.payload.prefetched?.ogImage ?? undefined),
|
||||
raw: rawContent.contentToVectorize,
|
||||
isSuccessfullyProcessed: false,
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
})
|
||||
.returning();
|
||||
} catch (error) {
|
||||
console.log("here's the error", error);
|
||||
// Check if error is a unique constraint violation
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes("document_url_user_id_idx")
|
||||
) {
|
||||
// Document already exists for this user, stop workflow
|
||||
await database(this.env.HYPERDRIVE.connectionString)
|
||||
.delete(documents)
|
||||
.where(eq(documents.uuid, event.payload.uuid));
|
||||
throw new NonRetryableError("Document already exists for this user");
|
||||
}
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes("document_raw_user_idx")
|
||||
) {
|
||||
await database(this.env.HYPERDRIVE.connectionString)
|
||||
.delete(documents)
|
||||
.where(eq(documents.uuid, event.payload.uuid));
|
||||
throw new NonRetryableError("The exact same document already exists");
|
||||
}
|
||||
throw error; // Re-throw other errors
|
||||
.where(eq(documents.id, existingDocument.id));
|
||||
console.log("[Workflow] Document updated successfully");
|
||||
|
||||
return [existingDocument];
|
||||
}
|
||||
|
||||
console.log(
|
||||
"[Workflow] Updating document with UUID:",
|
||||
event.payload.uuid
|
||||
);
|
||||
// Create new document
|
||||
const updated = await db
|
||||
.update(documents)
|
||||
.set({
|
||||
title: rawContent.title,
|
||||
description:
|
||||
"description" in rawContent
|
||||
? (rawContent.description ?? "")
|
||||
: (event.payload.prefetched?.description ?? undefined),
|
||||
ogImage:
|
||||
"image" in rawContent
|
||||
? (rawContent.image ?? "")
|
||||
: (event.payload.prefetched?.ogImage ?? undefined),
|
||||
content: rawContent.contentToSave,
|
||||
contentHash,
|
||||
isSuccessfullyProcessed: false,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(documents.uuid, event.payload.uuid))
|
||||
.returning();
|
||||
console.log("[Workflow] Document update result:", {
|
||||
updatedId: updated[0]?.id,
|
||||
updatedUuid: updated[0]?.uuid,
|
||||
});
|
||||
return updated;
|
||||
});
|
||||
|
||||
if (!document || document.length === 0) {
|
||||
throw new Error(
|
||||
"Failed to create/update document - no document returned"
|
||||
);
|
||||
}
|
||||
// Step 4: Process content
|
||||
console.log("[Workflow] Processing content for document:", {
|
||||
id: document[0].id,
|
||||
uuid: document[0].uuid,
|
||||
});
|
||||
const chunked = await step.do("chunk content", async () =>
|
||||
chunkText(rawContent.contentToVectorize, 768)
|
||||
);
|
||||
|
||||
const model = openai(this.env, this.env.OPEN_AI_API_KEY).embedding(
|
||||
"text-embedding-3-large",
|
||||
|
|
@ -149,7 +263,7 @@ export class ContentWorkflow extends WorkflowEntrypoint<Env, WorkflowParams> {
|
|||
}
|
||||
);
|
||||
|
||||
// Step 3: Create chunks from the content.
|
||||
// Create embeddings for chunks
|
||||
const embeddings = await step.do(
|
||||
"create embeddings",
|
||||
{
|
||||
|
|
@ -167,52 +281,60 @@ export class ContentWorkflow extends WorkflowEntrypoint<Env, WorkflowParams> {
|
|||
values: chunked,
|
||||
}
|
||||
);
|
||||
|
||||
return embeddings;
|
||||
}
|
||||
);
|
||||
|
||||
// Step 4: Prepare chunk data
|
||||
const chunkInsertData: ChunkInsert[] = await step.do(
|
||||
"prepare chunk data",
|
||||
async () =>
|
||||
chunked.map((chunk, index) => ({
|
||||
// Step 5: Update chunks
|
||||
await step.do("update chunks", async () => {
|
||||
const db = database(this.env.HYPERDRIVE.connectionString);
|
||||
|
||||
// Delete existing chunks if any
|
||||
await db.delete(chunk).where(eq(chunk.documentId, document[0].id));
|
||||
|
||||
// Insert new chunks
|
||||
const chunkInsertData: ChunkInsert[] = chunked.map(
|
||||
(chunkText, index) => ({
|
||||
documentId: document[0].id,
|
||||
textContent: chunk,
|
||||
textContent: chunkText,
|
||||
contentHash: generateHash(chunkText),
|
||||
orderInDocument: index,
|
||||
embeddings: embeddings[index],
|
||||
}))
|
||||
);
|
||||
|
||||
console.log(chunkInsertData);
|
||||
|
||||
// Step 5: Insert chunks
|
||||
if (chunkInsertData.length > 0) {
|
||||
await step.do("insert chunks", async () =>
|
||||
database(this.env.HYPERDRIVE.connectionString).transaction(
|
||||
async (trx) => {
|
||||
await trx.insert(chunk).values(chunkInsertData);
|
||||
}
|
||||
)
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// step 6: add content to spaces
|
||||
if (chunkInsertData.length > 0) {
|
||||
await db.transaction(async (trx) => {
|
||||
for (const chunkData of chunkInsertData) {
|
||||
await trx
|
||||
.insert(chunk)
|
||||
.values(chunkData)
|
||||
.onConflictDoNothing({ target: chunk.contentHash });
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Step 6: Mark document as processed
|
||||
await step.do("mark document as processed", async () => {
|
||||
await database(this.env.HYPERDRIVE.connectionString)
|
||||
.update(documents)
|
||||
.set({ isSuccessfullyProcessed: true })
|
||||
.where(eq(documents.id, document[0].id));
|
||||
});
|
||||
|
||||
// Step 7: Add content to spaces if specified
|
||||
if (event.payload.spaces) {
|
||||
await step.do("add content to spaces", async () => {
|
||||
await database(this.env.HYPERDRIVE.connectionString).transaction(
|
||||
async (trx) => {
|
||||
// First get the space IDs from the UUIDs
|
||||
const spaceIds = await trx
|
||||
.select({ id: spaces.id })
|
||||
.from(spaces)
|
||||
.where(inArray(spaces.uuid, event.payload.spaces ?? []));
|
||||
|
||||
if (spaceIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
if (spaceIds.length === 0) return;
|
||||
|
||||
// Then insert the content-space mappings using the actual space IDs
|
||||
await trx.insert(contentToSpace).values(
|
||||
spaceIds.map((space) => ({
|
||||
contentId: document[0].id,
|
||||
|
|
@ -223,15 +345,5 @@ export class ContentWorkflow extends WorkflowEntrypoint<Env, WorkflowParams> {
|
|||
);
|
||||
});
|
||||
}
|
||||
|
||||
// Step 7: Mark the document as successfully processed
|
||||
await step.do("mark document as successfully processed", async () => {
|
||||
await database(this.env.HYPERDRIVE.connectionString)
|
||||
.update(documents)
|
||||
.set({
|
||||
isSuccessfullyProcessed: true,
|
||||
})
|
||||
.where(eq(documents.id, document[0].id));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
247
apps/backend/test/routes/memories.test.ts
Normal file
247
apps/backend/test/routes/memories.test.ts
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
import { describe, it, expect, beforeAll, afterAll } from "vitest";
|
||||
import { app } from "../../src";
|
||||
import type { Document } from "@supermemory/db/schema";
|
||||
|
||||
import { testClient } from "hono/testing";
|
||||
import { env, SELF } from "cloudflare:test";
|
||||
|
||||
const API_KEY =
|
||||
"sm_4y1M2QIpRtKJMMfXWCqAhD_NVXOqLqOzOOuIZ1qhL1Gj0BFWFPuRlng5TPvZ1OIu1Zn1G8_xuuz59M4o4l-sAkSieZgaW09COg";
|
||||
|
||||
interface MemoryResponse {
|
||||
items: Array<Document>;
|
||||
total: number;
|
||||
}
|
||||
|
||||
interface SuccessResponse {
|
||||
success: boolean;
|
||||
}
|
||||
|
||||
describe("Memories Routes", () => {
|
||||
const client = testClient(app, env);
|
||||
|
||||
describe("GET /api/memories", () => {
|
||||
it("should list memories with default pagination", async () => {
|
||||
const res = await client.api.memories.$get(
|
||||
{ query: {} },
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const data = (await res.json()) as MemoryResponse;
|
||||
expect(data).toHaveProperty("items");
|
||||
expect(data).toHaveProperty("total");
|
||||
expect(Array.isArray(data.items)).toBe(true);
|
||||
});
|
||||
|
||||
it("should respect pagination parameters", async () => {
|
||||
const res = await client.api.memories.$get(
|
||||
{
|
||||
query: {
|
||||
start: "1",
|
||||
count: "5",
|
||||
},
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const data = (await res.json()) as MemoryResponse;
|
||||
expect(data.items.length).toBeLessThanOrEqual(5);
|
||||
});
|
||||
|
||||
it("should filter by space ID", async () => {
|
||||
const res = await client.api.memories.$get(
|
||||
{
|
||||
query: {
|
||||
spaceId: "test-space-uuid",
|
||||
},
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const data = (await res.json()) as MemoryResponse;
|
||||
expect(data).toHaveProperty("items");
|
||||
expect(data).toHaveProperty("total");
|
||||
});
|
||||
|
||||
it("should handle unauthorized space access", async () => {
|
||||
const res = await client.api.memories.$get(
|
||||
{
|
||||
query: {
|
||||
spaceId: "private-space-uuid",
|
||||
},
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it("should reject requests without API key", async () => {
|
||||
const res = await client.api.memories.$get(
|
||||
{ query: {} },
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it("should respect ETag caching", async () => {
|
||||
// First request to get ETag
|
||||
const res1 = await client.api.memories.$get(
|
||||
{ query: {} },
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
const etag = res1.headers.get("ETag");
|
||||
expect(etag).toBeDefined();
|
||||
|
||||
// Second request with ETag
|
||||
const res2 = await client.api.memories.$get(
|
||||
{ query: {} },
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
"If-None-Match": etag!,
|
||||
},
|
||||
}
|
||||
);
|
||||
expect(res2.status).toBe(304);
|
||||
});
|
||||
});
|
||||
|
||||
describe("GET /api/memories/:id", () => {
|
||||
let testMemoryId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
const res = await client.api.memories.$get(
|
||||
{ query: {} },
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
const data = (await res.json()) as MemoryResponse;
|
||||
if (data.items.length > 0) {
|
||||
testMemoryId = data.items[0].uuid;
|
||||
}
|
||||
});
|
||||
|
||||
it("should retrieve a specific memory", async () => {
|
||||
if (!testMemoryId) {
|
||||
console.warn("No test memory available, skipping test");
|
||||
return;
|
||||
}
|
||||
|
||||
const res = await client.api.memories[":id"].$get(
|
||||
{
|
||||
param: {
|
||||
id: testMemoryId,
|
||||
},
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const memory = await res.json();
|
||||
expect(memory).toHaveProperty("uuid", testMemoryId);
|
||||
});
|
||||
|
||||
it("should reject unauthorized access", async () => {
|
||||
const res = await client.api.memories[":id"].$get(
|
||||
{
|
||||
param: {
|
||||
id: testMemoryId,
|
||||
},
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
});
|
||||
|
||||
describe("DELETE /api/memories/:id", () => {
|
||||
let memoryId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
const res = await client.api.memories.$get(
|
||||
{ query: {} },
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
},
|
||||
}
|
||||
);
|
||||
const data = (await res.json()) as MemoryResponse;
|
||||
if (data.items.length > 0) {
|
||||
memoryId = data.items[0].uuid;
|
||||
}
|
||||
});
|
||||
|
||||
it("should delete a memory", async () => {
|
||||
const res = await client.api.memories[":id"].$delete(
|
||||
{
|
||||
param: { id: memoryId },
|
||||
},
|
||||
{
|
||||
headers: { Authorization: `Bearer ${API_KEY}` },
|
||||
}
|
||||
);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(await res.json()).toEqual({ success: true });
|
||||
|
||||
// Verify deletion
|
||||
const getRes = await client.api.memories[":id"].$get(
|
||||
{
|
||||
param: { id: memoryId },
|
||||
},
|
||||
{
|
||||
headers: { Authorization: `Bearer ${API_KEY}` },
|
||||
}
|
||||
);
|
||||
expect(getRes.status).toBe(404);
|
||||
});
|
||||
|
||||
it("should reject unauthorized deletion", async () => {
|
||||
const res = await client.api.memories[":id"].$delete(
|
||||
{
|
||||
param: { id: memoryId },
|
||||
},
|
||||
{
|
||||
headers: { Authorization: `Bearer ${API_KEY}` },
|
||||
}
|
||||
);
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
});
|
||||
});
|
||||
32
apps/backend/test/setup.ts
Normal file
32
apps/backend/test/setup.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
import { Hono } from "hono";
|
||||
import type { Env } from "../src/types";
|
||||
import { describe, it, expect, beforeAll, afterAll } from "vitest";
|
||||
|
||||
// Helper to generate random test content
|
||||
export function generateTestContent(length: number = 1000): string {
|
||||
return Array(length)
|
||||
.fill(0)
|
||||
.map(() => Math.random().toString(36).charAt(2))
|
||||
.join("");
|
||||
}
|
||||
|
||||
// Helper to measure response time
|
||||
export async function measureResponseTime(
|
||||
fn: () => Promise<Response>
|
||||
): Promise<number> {
|
||||
const start = performance.now();
|
||||
await fn();
|
||||
return performance.now() - start;
|
||||
}
|
||||
|
||||
// Common test expectations
|
||||
export const expect200 = (response: Response) =>
|
||||
expect(response.status).toBe(200);
|
||||
export const expect401 = (response: Response) =>
|
||||
expect(response.status).toBe(401);
|
||||
export const expect403 = (response: Response) =>
|
||||
expect(response.status).toBe(403);
|
||||
export const expect404 = (response: Response) =>
|
||||
expect(response.status).toBe(404);
|
||||
export const expect429 = (response: Response) =>
|
||||
expect(response.status).toBe(429);
|
||||
|
|
@ -8,7 +8,8 @@
|
|||
"lib": ["ESNext"],
|
||||
"types": [
|
||||
"@cloudflare/workers-types/experimental",
|
||||
"@cloudflare/workers-types"
|
||||
"@cloudflare/workers-types",
|
||||
"@cloudflare/vitest-pool-workers"
|
||||
],
|
||||
"jsx": "react-jsx",
|
||||
"jsxImportSource": "hono/jsx"
|
||||
|
|
|
|||
11
apps/backend/vitest.config.ts
Normal file
11
apps/backend/vitest.config.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
import { defineWorkersConfig } from "@cloudflare/vitest-pool-workers/config";
|
||||
|
||||
export default defineWorkersConfig({
|
||||
test: {
|
||||
poolOptions: {
|
||||
workers: {
|
||||
wrangler: { configPath: "./wrangler.toml" },
|
||||
},
|
||||
}
|
||||
},
|
||||
});
|
||||
|
|
@ -19,7 +19,6 @@ name = "content-workflow-supermemory"
|
|||
binding = "CONTENT_WORKFLOW"
|
||||
class_name = "ContentWorkflow"
|
||||
|
||||
|
||||
[[kv_namespaces]]
|
||||
binding= "MD_CACHE"
|
||||
id = "3186489f943d409a9b772d876a58a73e"
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ import {
|
|||
AlertCircle,
|
||||
BookIcon,
|
||||
CheckCircle,
|
||||
Clipboard,
|
||||
ClipboardCheckIcon,
|
||||
FileUpIcon,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
|
|
@ -220,50 +218,66 @@ function Integrations() {
|
|||
</p>
|
||||
</div>
|
||||
|
||||
<div className="max-w-full bg-neutral-50 dark:bg-neutral-900 p-4 rounded-lg border border-neutral-200 dark:border-neutral-800 mb-8">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-sm font-medium text-neutral-600 dark:text-neutral-400">
|
||||
Your API Key
|
||||
</span>
|
||||
{apiKey ? (
|
||||
<button
|
||||
onClick={() => {
|
||||
navigator.clipboard.writeText(apiKey);
|
||||
setCopied(true);
|
||||
toast.success("API key copied to clipboard!");
|
||||
}}
|
||||
className="flex-1 flex items-center gap-2 font-mono text-sm bg-white dark:bg-neutral-800 px-3 py-1.5 rounded group hover:bg-neutral-100 dark:hover:bg-neutral-700 transition-all overflow-hidden"
|
||||
<div className="flex flex-col gap-2 mb-4">
|
||||
<div className="flex justify-end">
|
||||
<a
|
||||
href="https://docs.supermemory.ai"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="inline-flex items-center gap-1.5 px-2 py-1 text-xs font-medium text-neutral-600 dark:text-neutral-400 hover:text-neutral-900 dark:hover:text-neutral-200 transition-colors"
|
||||
>
|
||||
<span className="flex-shrink-0 text-neutral-400 dark:text-neutral-500 group-hover:text-neutral-600 dark:group-hover:text-neutral-300 transition-all">
|
||||
{copied ? (
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
strokeWidth={2}
|
||||
d="M5 13l4 4L19 7"
|
||||
/>
|
||||
</svg>
|
||||
) : (
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
strokeWidth={2}
|
||||
d="M8 16H6a2 2 0 01-2-2V6a2 2 0 012-2h8a2 2 0 012 2v2m-6 12h8a2 2 0 002-2v-8a2 2 0 00-2-2h-8a2 2 0 00-2 2v8a2 2 0 002 2z"
|
||||
/>
|
||||
</svg>
|
||||
)}
|
||||
</span>
|
||||
<span className="blur-sm group-hover:blur-none transition-all truncate max-w-[500px]">
|
||||
{apiKey}
|
||||
</span>
|
||||
</button>
|
||||
) : (
|
||||
<div className="flex-1 text-sm text-neutral-600 dark:text-neutral-400">Loading...</div>
|
||||
)}
|
||||
<BookIcon className="w-3 h-3" />
|
||||
API Documentation
|
||||
</a>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-sm font-medium text-neutral-600 dark:text-neutral-400">
|
||||
Your API Key
|
||||
</span>
|
||||
{apiKey ? (
|
||||
<button
|
||||
onClick={() => {
|
||||
navigator.clipboard.writeText(apiKey);
|
||||
setCopied(true);
|
||||
toast.success("API key copied to clipboard!");
|
||||
}}
|
||||
className="flex-1 flex items-center gap-2 font-mono text-sm bg-white dark:bg-neutral-800 px-3 py-1.5 rounded group hover:bg-neutral-100 dark:hover:bg-neutral-700 transition-all overflow-hidden"
|
||||
>
|
||||
<span className="flex-shrink-0 text-neutral-400 dark:text-neutral-500 group-hover:text-neutral-600 dark:group-hover:text-neutral-300 transition-all">
|
||||
{copied ? (
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
strokeWidth={2}
|
||||
d="M5 13l4 4L19 7"
|
||||
/>
|
||||
</svg>
|
||||
) : (
|
||||
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
strokeWidth={2}
|
||||
d="M8 16H6a2 2 0 01-2-2V6a2 2 0 012-2h8a2 2 0 012 2v2m-6 12h8a2 2 0 002-2v-8a2 2 0 00-2-2h-8a2 2 0 00-2 2v8a2 2 0 002 2z"
|
||||
/>
|
||||
</svg>
|
||||
)}
|
||||
</span>
|
||||
<div className="relative flex-1">
|
||||
<span className="absolute inset-0 flex items-center justify-center text-neutral-400 dark:text-neutral-500 opacity-0 group-hover:opacity-0 transition-opacity">
|
||||
Hover to reveal
|
||||
</span>
|
||||
<span className="blur-sm group-hover:blur-none transition-all truncate max-w-[500px]">
|
||||
{apiKey}
|
||||
</span>
|
||||
</div>
|
||||
</button>
|
||||
) : (
|
||||
<div className="flex-1 text-sm text-neutral-600 dark:text-neutral-400">Loading...</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-wrap gap-4 overflow-x-auto">
|
||||
<Card
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
],
|
||||
"compilerOptions": {
|
||||
"lib": ["DOM", "DOM.Iterable", "ES2022"],
|
||||
"types": ["@remix-run/cloudflare", "vite/client", "@cloudflare/workers-types/2023-07-01", "chrome"],
|
||||
"types": ["@remix-run/cloudflare", "vite/client", "@cloudflare/workers-types/2023-07-01", "chrome" ],
|
||||
"isolatedModules": true,
|
||||
"esModuleInterop": true,
|
||||
"jsx": "react-jsx",
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@
|
|||
"resend": "^4.0.1",
|
||||
"shiki": "^1.22.1",
|
||||
"sonner": "^1.7.0",
|
||||
"vitest": "^3.0.4",
|
||||
"web": "^0.0.2",
|
||||
"wrangler": "latest"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -146,6 +146,30 @@ export const documentType = pgTable("document_type", {
|
|||
type: text("type").primaryKey(),
|
||||
});
|
||||
|
||||
export const processedContent = pgTable(
|
||||
"processed_content",
|
||||
{
|
||||
id: serial("id").primaryKey(),
|
||||
contentHash: text("content_hash").notNull().unique(),
|
||||
content: text("content").notNull(),
|
||||
isSuccessfullyProcessed: boolean("is_successfully_processed").default(
|
||||
false
|
||||
),
|
||||
errorMessage: text("error_message"),
|
||||
createdAt: timestamp("created_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(),
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(),
|
||||
},
|
||||
(table) => ({
|
||||
contentHashIdx: uniqueIndex("processed_content_hash_idx").on(
|
||||
table.contentHash
|
||||
),
|
||||
})
|
||||
);
|
||||
|
||||
export const documents = pgTable(
|
||||
"documents",
|
||||
{
|
||||
|
|
@ -167,11 +191,11 @@ export const documents = pgTable(
|
|||
.notNull()
|
||||
.references(() => users.id, { onDelete: "cascade" }),
|
||||
content: text("content"),
|
||||
contentHash: text("content_hash"),
|
||||
isSuccessfullyProcessed: boolean("is_successfully_processed").default(
|
||||
false
|
||||
),
|
||||
errorMessage: text("error_message"),
|
||||
contentHash: text("content_hash"),
|
||||
},
|
||||
(document) => ({
|
||||
documentsIdIdx: uniqueIndex("document_id_idx").on(document.id),
|
||||
|
|
@ -181,6 +205,13 @@ export const documents = pgTable(
|
|||
document.raw,
|
||||
document.userId
|
||||
),
|
||||
documentContentHashIdx: index("document_content_hash_idx").on(
|
||||
document.contentHash
|
||||
),
|
||||
documentUrlUserIdx: uniqueIndex("document_url_user_idx").on(
|
||||
document.url,
|
||||
document.userId
|
||||
),
|
||||
})
|
||||
);
|
||||
|
||||
|
|
@ -213,7 +244,8 @@ export const chunk = pgTable(
|
|||
documentId: integer("document_id")
|
||||
.references(() => documents.id, { onDelete: "cascade" })
|
||||
.notNull(),
|
||||
textContent: text("text_content"),
|
||||
textContent: text("text_content").notNull(),
|
||||
contentHash: text("content_hash"),
|
||||
orderInDocument: integer("order_in_document").notNull(),
|
||||
embeddings: vector("embeddings", { dimensions: 1536 }),
|
||||
metadata: jsonb("metadata").$type<Metadata>(),
|
||||
|
|
@ -222,11 +254,12 @@ export const chunk = pgTable(
|
|||
.defaultNow(),
|
||||
updated_at: timestamp("updated_at", { withTimezone: true })
|
||||
.notNull()
|
||||
.defaultNow(), // handle deletion on application layer
|
||||
.defaultNow(),
|
||||
},
|
||||
(chunk) => ({
|
||||
chunkIdIdx: uniqueIndex("chunk_id_idx").on(chunk.id),
|
||||
chunkDocumentIdIdx: index("chunk_document_id_idx").on(chunk.documentId),
|
||||
chunkContentHashIdx: index("chunk_content_hash_idx").on(chunk.contentHash),
|
||||
embeddingIndex: index("embeddingIndex").using(
|
||||
"hnsw",
|
||||
chunk.embeddings.op("vector_cosine_ops")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue