diff --git a/.gitignore b/.gitignore index 3371c5cf..d8534601 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. bun.lockb +packages/scripts/* # Dependencies node_modules/ /.pnp diff --git a/apps/backend/drizzle/0015_perpetual_mauler.sql b/apps/backend/drizzle/0015_perpetual_mauler.sql new file mode 100644 index 00000000..ec419523 --- /dev/null +++ b/apps/backend/drizzle/0015_perpetual_mauler.sql @@ -0,0 +1 @@ +ALTER TABLE "chunks" ALTER COLUMN "embeddings" SET DATA TYPE vector(768); \ No newline at end of file diff --git a/apps/backend/drizzle/meta/0015_snapshot.json b/apps/backend/drizzle/meta/0015_snapshot.json new file mode 100644 index 00000000..d19dbfbb --- /dev/null +++ b/apps/backend/drizzle/meta/0015_snapshot.json @@ -0,0 +1,1207 @@ +{ + "id": "8529db1b-2d33-49e0-a413-f517eae7e4e4", + "prevId": "f5cef50a-50ec-4b49-99c2-4da3a0f6a098", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true + }, + "firstMessage": { + "name": "firstMessage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "messages": { + "name": "messages", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_threads_user_idx": { + "name": "chat_threads_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_user_id_users_id_fk": { + "name": "chat_threads_user_id_users_id_fk", + "tableFrom": "chat_threads", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "chat_threads_uuid_unique": { + "name": "chat_threads_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "public.chunks": { + "name": "chunks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "text_content": { + "name": "text_content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "order_in_document": { + "name": "order_in_document", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embeddings": { + "name": "embeddings", + "type": "vector(7)", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chunk_id_idx": { + "name": "chunk_id_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "chunk_document_id_idx": { + "name": "chunk_document_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "embeddingIndex": { + "name": "embeddingIndex", + "columns": [ + { + "expression": "embeddings", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + } + }, + "foreignKeys": { + "chunks_document_id_documents_id_fk": { + "name": "chunks_document_id_documents_id_fk", + "tableFrom": "chunks", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.content_to_space": { + "name": "content_to_space", + "schema": "", + "columns": { + "content_id": { + "name": "content_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "space_id": { + "name": "space_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "content_id_space_id_unique": { + "name": "content_id_space_id_unique", + "columns": [ + { + "expression": "content_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "space_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "content_to_space_content_id_documents_id_fk": { + "name": "content_to_space_content_id_documents_id_fk", + "tableFrom": "content_to_space", + "tableTo": "documents", + "columnsFrom": [ + "content_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "content_to_space_space_id_spaces_id_fk": { + "name": "content_to_space_space_id_spaces_id_fk", + "tableFrom": "content_to_space", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.document_type": { + "name": "document_type", + "schema": "", + "columns": { + "type": { + "name": "type", + "type": "text", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.documents": { + "name": "documents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "og_image": { + "name": "og_image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "raw": { + "name": "raw", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_successfully_processed": { + "name": "is_successfully_processed", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "document_id_idx": { + "name": "document_id_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_uuid_idx": { + "name": "document_uuid_idx", + "columns": [ + { + "expression": "uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_type_idx": { + "name": "document_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_raw_user_idx": { + "name": "document_raw_user_idx", + "columns": [ + { + "expression": "raw", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "documents_type_document_type_type_fk": { + "name": "documents_type_document_type_type_fk", + "tableFrom": "documents", + "tableTo": "document_type", + "columnsFrom": [ + "type" + ], + "columnsTo": [ + "type" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "documents_user_id_users_id_fk": { + "name": "documents_user_id_users_id_fk", + "tableFrom": "documents", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "documents_uuid_unique": { + "name": "documents_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "public.job": { + "name": "job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "lastAttemptAt": { + "name": "lastAttemptAt", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "user_id_url_idx": { + "name": "user_id_url_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "job_user_id_users_id_fk": { + "name": "job_user_id_users_id_fk", + "tableFrom": "job", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.saved_spaces": { + "name": "saved_spaces", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "space_id": { + "name": "space_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "saved_at": { + "name": "saved_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "saved_spaces_user_space_idx": { + "name": "saved_spaces_user_space_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "space_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_spaces_user_id_users_id_fk": { + "name": "saved_spaces_user_id_users_id_fk", + "tableFrom": "saved_spaces", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "saved_spaces_space_id_spaces_id_fk": { + "name": "saved_spaces_space_id_spaces_id_fk", + "tableFrom": "saved_spaces", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.space_access": { + "name": "space_access", + "schema": "", + "columns": { + "space_id": { + "name": "space_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "user_email": { + "name": "user_email", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_type": { + "name": "access_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'read'" + } + }, + "indexes": { + "space_id_user_email_idx": { + "name": "space_id_user_email_idx", + "columns": [ + { + "expression": "space_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "space_access_space_id_spaces_id_fk": { + "name": "space_access_space_id_spaces_id_fk", + "tableFrom": "space_access", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "space_access_status_space_access_status_status_fk": { + "name": "space_access_status_space_access_status_status_fk", + "tableFrom": "space_access", + "tableTo": "space_access_status", + "columnsFrom": [ + "status" + ], + "columnsTo": [ + "status" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.space_access_status": { + "name": "space_access_status", + "schema": "", + "columns": { + "status": { + "name": "status", + "type": "text", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.space_members": { + "name": "space_members", + "schema": "", + "columns": { + "spaceId": { + "name": "spaceId", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "space_members_space_user_idx": { + "name": "space_members_space_user_idx", + "columns": [ + { + "expression": "spaceId", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "space_members_spaceId_users_id_fk": { + "name": "space_members_spaceId_users_id_fk", + "tableFrom": "space_members", + "tableTo": "users", + "columnsFrom": [ + "spaceId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "space_members_user_id_users_id_fk": { + "name": "space_members_user_id_users_id_fk", + "tableFrom": "space_members", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.spaces": { + "name": "spaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ownerId": { + "name": "ownerId", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_public": { + "name": "is_public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "spaces_id_idx": { + "name": "spaces_id_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "spaces_owner_id_idx": { + "name": "spaces_owner_id_idx", + "columns": [ + { + "expression": "ownerId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "spaces_name_idx": { + "name": "spaces_name_idx", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "spaces_uuid_unique": { + "name": "spaces_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "first_name": { + "name": "first_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_name": { + "name": "last_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "profile_picture_url": { + "name": "profile_picture_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "telegram_id": { + "name": "telegram_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "has_onboarded": { + "name": "has_onboarded", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_api_key_generated_at": { + "name": "last_api_key_generated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tier": { + "name": "tier", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'free'" + } + }, + "indexes": { + "users_id_idx": { + "name": "users_id_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "users_uuid_idx": { + "name": "users_uuid_idx", + "columns": [ + { + "expression": "uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "users_email_idx": { + "name": "users_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "users_name_idx": { + "name": "users_name_idx", + "columns": [ + { + "expression": "first_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "users_created_at_idx": { + "name": "users_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "users_telegram_id_idx": { + "name": "users_telegram_id_idx", + "columns": [ + { + "expression": "telegram_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_uuid_unique": { + "name": "users_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + }, + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + } + }, + "public.waitlist": { + "name": "waitlist", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "varchar(512)", + "primaryKey": true, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/backend/drizzle/meta/_journal.json b/apps/backend/drizzle/meta/_journal.json index 2ef7aaa9..c8cea61a 100644 --- a/apps/backend/drizzle/meta/_journal.json +++ b/apps/backend/drizzle/meta/_journal.json @@ -106,6 +106,13 @@ "when": 1736852938881, "tag": "0014_mighty_the_captain", "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1737920848112, + "tag": "0015_perpetual_mauler", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/backend/package.json b/apps/backend/package.json index 5d5b1092..98066200 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -5,6 +5,7 @@ "deploy": "bunx wrangler deploy --minify", "generate-migration": "dotenv -- npx drizzle-kit generate", "migrate:local": "bun run ./scripts/migrate.ts", + "migrate:prod": "NODE_ENV=production bun run ./scripts/migrate.ts", "tail": "bunx wrangler tail" }, "dependencies": { diff --git a/apps/backend/scripts/migrate.ts b/apps/backend/scripts/migrate.ts index 4f8db70d..bb8bdcf8 100644 --- a/apps/backend/scripts/migrate.ts +++ b/apps/backend/scripts/migrate.ts @@ -6,11 +6,12 @@ import postgres from "postgres"; config(); -if (!process.env.DATABASE_URL) { - throw new Error("DATABASE_URL is not set"); -} +const isProd = process.env.NODE_ENV === "production"; +const connectionString = isProd ? process.env.PROD_DATABASE_URL : process.env.DATABASE_URL; -const connectionString = process.env.DATABASE_URL!; +if (!connectionString) { + throw new Error(`${isProd ? "PROD_DATABASE_URL" : "DATABASE_URL"} is not set`); +} console.log("Connecting to:", connectionString.replace(/:[^:@]+@/, ":****@")); // Log sanitized connection string diff --git a/apps/backend/src/routes/actions.ts b/apps/backend/src/routes/actions.ts index b9e6a4c2..1ac02fdd 100644 --- a/apps/backend/src/routes/actions.ts +++ b/apps/backend/src/routes/actions.ts @@ -54,8 +54,6 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() const { messages, threadId } = await c.req.valid("json"); - // TODO: add rate limiting - const unfilteredCoreMessages = convertToCoreMessages( (messages as Message[]) .filter((m) => m.content.length > 0) @@ -67,125 +65,66 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() ? `${JSON.stringify(m.annotations)}` : ""), experimental_attachments: - m.experimental_attachments && - m.experimental_attachments.length > 0 + m.experimental_attachments?.length && + m.experimental_attachments?.length > 0 ? m.experimental_attachments : (m.data as { files: [] })?.files, })) ); - // make sure that there is no empty messages. if there is, remove it. const coreMessages = unfilteredCoreMessages.filter( (message) => message.content.length > 0 ); - // .map(async (c) => { - // if ( - // Array.isArray(c.content) && - // c.content.some((c) => c.type !== "text") - // ) { - // // convert attachments (IMAGE and files) to base64 by fetching them - // const attachments = c.content.filter((c) => c.type !== "text"); - // const base64Attachments = await Promise.all( - // attachments.map(async (a) => { - // const type = (a as ImagePart | FilePart).type; - // if (type === "image") { - // const response = await fetch((a as ImagePart).image.toString()); - // return response.arrayBuffer(); - // } else if (type === "file") { - // const response = await fetch((a as FilePart).data.toString()); - // return response.arrayBuffer(); - // } - // }) - // ); - // } - // }); - - console.log("Core messages", JSON.stringify(coreMessages, null, 2)); - - let threadUuid = threadId; + const db = database(c.env.HYPERDRIVE.connectionString); const { initLogger, wrapAISDKModel } = await import("braintrust"); + // Initialize clients and loggers const logger = initLogger({ projectName: "supermemory", apiKey: c.env.BRAINTRUST_API_KEY, }); - // const gemini = createOpenAI({ - // apiKey: c.env.GEMINI_API_KEY, - // baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/", - // }); - const openaiClient = openai(c.env); + const googleClient = wrapAISDKModel(openai(c.env).chat("gpt-4o")); - const googleClient = wrapAISDKModel( - // google(c.env.GEMINI_API_KEY).chat("gemini-exp-1206") - openai(c.env).chat("gpt-4o") - ); - - // Create new thread if none exists - if (!threadUuid) { - const uuid = randomId(); - const newThread = await database(c.env.HYPERDRIVE.connectionString) - .insert(chatThreads) - .values({ - firstMessage: messages[0].content, - userId: user.id, - uuid: uuid, - messages: coreMessages, - }) - .returning(); - - threadUuid = newThread[0].uuid; - } - const openAi = openai(c.env); - - - - let lastUserMessage = coreMessages - .reverse() - .find((i) => i.role === "user"); - - // get the text of lastUserMEssage + // Get last user message and generate embedding in parallel with thread creation + let lastUserMessage = coreMessages.findLast((i) => i.role === "user"); const queryText = typeof lastUserMessage?.content === "string" ? lastUserMessage.content : lastUserMessage?.content.map((c) => (c as TextPart).text).join(""); - console.log("querytext", queryText); - - if (!queryText ||queryText.length === 0) { - return c.json({ error: "Failed to generate embedding for query" }, 500); + if (!queryText || queryText.length === 0) { + return c.json({ error: "Empty query" }, 400); } - const embedStart = performance.now(); - const { data: embedding } = await c.env.AI.run("@cf/baai/bge-base-en-v1.5", { - text: queryText, - }); - const embedEnd = performance.now(); - console.log(`Embedding generation took ${embedEnd - embedStart}ms`); + // Run embedding generation and thread creation in parallel + const [{ data: embedding }, thread] = await Promise.all([ + c.env.AI.run("@cf/baai/bge-base-en-v1.5", { text: queryText }), + !threadId + ? db + .insert(chatThreads) + .values({ + firstMessage: messages[0].content, + userId: user.id, + uuid: randomId(), + messages: coreMessages, + }) + .returning() + : null, + ]); + + const threadUuid = threadId || thread?.[0].uuid; if (!embedding) { - return c.json({ error: "Failed to generate embedding for query" }, 500); + return c.json({ error: "Failed to generate embedding" }, 500); } - // Perform semantic search using cosine similarity - // Log the query text to debug what we're searching for - console.log("Searching for:", queryText); - console.log("user id", user.id); + // Perform semantic search + const similarity = sql`1 - (${cosineDistance(chunk.embeddings, embedding[0])})`; - const similarity = sql`1 - (${cosineDistance( - chunk.embeddings, - embedding[0] - )})`; - - // Find similar chunks using cosine similarity - // Join with documents table to get chunks only from documents the user has access to - // First get all results to normalize - // Get top 20 results first to avoid processing entire dataset - const dbQueryStart = performance.now(); - const topResults = await database(c.env.HYPERDRIVE.connectionString) + const finalResults = await db .select({ - similarity, id: documents.id, content: documents.content, type: documents.type, @@ -200,62 +139,10 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() .from(chunk) .innerJoin(documents, eq(chunk.documentId, documents.id)) .where(and(eq(documents.userId, user.id), sql`${similarity} > 0.4`)) - .orderBy(desc(similarity)); + .orderBy(desc(similarity)) + .limit(5); - // Get unique documents with their highest similarity chunks - const uniqueDocuments = Object.values( - topResults.reduce( - (acc, curr) => { - if ( - !acc[curr.id] || - acc[curr.id].content === curr.content || - acc[curr.id].url === curr.url - ) { - acc[curr.id] = curr; - } - return acc; - }, - {} as Record - ) - ).slice(0, 5); - - const dbQueryEnd = performance.now(); - console.log(`Database query took ${dbQueryEnd - dbQueryStart}ms`); - - // Calculate min/max once for the subset - const processingStart = performance.now(); - const minSimilarity = Math.min( - ...uniqueDocuments.map((r) => r.similarity) - ); - const maxSimilarity = Math.max( - ...uniqueDocuments.map((r) => r.similarity) - ); - const range = maxSimilarity - minSimilarity; - - // Normalize the results - const normalizedResults = uniqueDocuments.map((result) => ({ - ...result, - normalizedSimilarity: - range === 0 ? 1 : (result.similarity - minSimilarity) / range, - })); - - // Get either all results above 0.6 threshold, or at least top 3 results - const results = normalizedResults - .sort((a, b) => b.normalizedSimilarity - a.normalizedSimilarity) - .slice( - 0, - Math.max( - 3, - normalizedResults.filter((r) => r.normalizedSimilarity > 0.6).length - ) - ); - - const processingEnd = performance.now(); - console.log( - `Results processing took ${processingEnd - processingStart}ms` - ); - - const cleanDocumentsForContext = results.map((d) => ({ + const cleanDocumentsForContext = finalResults.map((d) => ({ title: d.title, description: d.description, url: d.url, @@ -263,8 +150,6 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() content: d.content, })); - // Update lastUserMessage with search results - const messageUpdateStart = performance.now(); if (lastUserMessage) { lastUserMessage.content = typeof lastUserMessage.content === "string" @@ -277,26 +162,30 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() text: `${JSON.stringify(cleanDocumentsForContext)}`, }, ]; - } - - // edit the last coreusermessage in the array - if (lastUserMessage) { coreMessages[coreMessages.length - 1] = lastUserMessage; } - const messageUpdateEnd = performance.now(); - console.log( - `Message update took ${messageUpdateEnd - messageUpdateStart}ms` - ); try { - const streamStart = performance.now(); + const data = new StreamData(); + data.appendMessageAnnotation( + finalResults.map((r) => ({ + id: r.id, + content: r.content, + type: r.type, + url: r.url, + title: r.title, + description: r.description, + ogImage: r.ogImage, + userId: r.userId, + createdAt: r.createdAt.toISOString(), + updatedAt: r.updatedAt?.toISOString() || null, + })) + ); + const result = await streamText({ model: googleClient, experimental_providerMetadata: { - metadata: { - userId: user.id, - chatThreadId: threadUuid, - }, + metadata: { userId: user.id, chatThreadId: threadUuid ?? "" }, }, experimental_transform: smoothStream(), messages: [ @@ -323,12 +212,11 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() ], async onFinish(completion) { try { - // remove context from lastUserMessage if (lastUserMessage) { lastUserMessage.content = typeof lastUserMessage.content === "string" ? lastUserMessage.content.replace( - /([\s\S]*?)<\/context>/g, + /[\s\S]*?<\/context>/g, "" ) : lastUserMessage.content.filter( @@ -338,60 +226,34 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() part.text.startsWith("") ) ); - coreMessages[coreMessages.length - 1] = lastUserMessage; } - console.log("results", results); - const newMessages = [ ...coreMessages, { role: "assistant", content: completion.text + - `[${JSON.stringify(results)}]`, + `[${JSON.stringify(finalResults)}]`, }, ]; - await data.close(); if (threadUuid) { - await database(c.env.HYPERDRIVE.connectionString) + await db .update(chatThreads) .set({ messages: newMessages }) .where(eq(chatThreads.uuid, threadUuid)); } } catch (error) { console.error("Failed to update thread:", error); - // Continue execution - the message was delivered even if saving failed } }, }); - const streamEnd = performance.now(); - console.log(`Stream response took ${streamEnd - streamStart}ms`); - - const data = new StreamData(); - - const context = results.map((r) => ({ - similarity: r.similarity, - id: r.id, - content: r.content, - type: r.type, - url: r.url, - title: r.title, - description: r.description, - ogImage: r.ogImage, - userId: r.userId, - createdAt: r.createdAt.toISOString(), - updatedAt: r.updatedAt?.toISOString() || null, - })); - // Full context objects in the data - data.appendMessageAnnotation(context); - return result.toDataStreamResponse({ headers: { - "Supermemory-Thread-Uuid": threadUuid, + "Supermemory-Thread-Uuid": threadUuid ?? "", "Content-Type": "text/x-unknown", "content-encoding": "identity", "transfer-encoding": "chunked", @@ -408,7 +270,6 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() ); } - // Handle database connection errors if ((error as AISDKError).cause === "ECONNREFUSED") { return c.json({ error: "Database connection failed" }, 503); } diff --git a/apps/backend/src/utils/fetchers.ts b/apps/backend/src/utils/fetchers.ts index fbca3d4b..2329f48a 100644 --- a/apps/backend/src/utils/fetchers.ts +++ b/apps/backend/src/utils/fetchers.ts @@ -35,11 +35,6 @@ export const fetchContent = async ( tweetUrl.search = ""; // Remove all search params const tweetId = tweetUrl.pathname.split("/").pop(); - const unrolledTweetContent = await step.do( - "get unrolled tweet content", - async () => await unrollTweets(tweetUrl.toString()) - ); - const rawBaseTweetContent = await step.do( "extract tweet content", async () => { @@ -72,8 +67,10 @@ export const fetchContent = async ( }; raw: string; }; - - if (!unrolledTweetContent || isErr(unrolledTweetContent)) { + const unrolledTweetContent = { + value: [rawBaseTweetContent], + }; + if (true) { console.error("Can't get thread, reverting back to single tweet"); tweetContent = { text: rawBaseTweetContent.text, diff --git a/apps/web/app/components/Reminders.tsx b/apps/web/app/components/Reminders.tsx index 96be74bb..1a3fca49 100644 --- a/apps/web/app/components/Reminders.tsx +++ b/apps/web/app/components/Reminders.tsx @@ -3,7 +3,6 @@ import Markdown from "react-markdown"; import { useNavigate } from "@remix-run/react"; -import { client } from "../lib/utils/api"; import image from "./gradients/gradient1.png"; import { AddMemoryModal } from "./memories/AddMemory"; import { Button } from "./ui/button"; diff --git a/apps/web/app/root.tsx b/apps/web/app/root.tsx index 75485ce6..afedd49b 100644 --- a/apps/web/app/root.tsx +++ b/apps/web/app/root.tsx @@ -29,6 +29,7 @@ import "@fontsource/geist-sans/900.css"; import { QueryClient, QueryClientProvider, useQuery } from "@tanstack/react-query"; import { Toaster } from "sonner"; import posthog from "posthog-js"; +import { PostHogProvider, usePostHog} from 'posthog-js/react' const queryClient = new QueryClient(); @@ -130,10 +131,10 @@ export const loader = async ({ request, context }: LoaderFunctionArgs) => { }); }; -// Use React.memo to memoize the App component const App = React.memo(function App() { const data = useLoaderData(); const [theme] = useTheme(); + const posthog = usePostHog(); useEffect(() => { if (data.user) { @@ -147,16 +148,16 @@ const App = React.memo(function App() { lastName: data.user.lastName, }); } - }, []); + }, [data.user]); return ( - + - + @@ -174,23 +175,17 @@ const App = React.memo(function App() { }); export default function AppWithProviders() { - const data = useLoaderData(); - - const specifiedTheme = useMemo(() => data.theme, [data.theme]); - - const MemoizedApp = React.memo(App); - const MemoizedThemeProvider = useMemo( - () => - - , - [specifiedTheme] - ); + const data = useLoaderData() return ( - - - {MemoizedThemeProvider} - - + + + + + + + + + ); } diff --git a/apps/web/app/routes/pitch.index.tsx b/apps/web/app/routes/pitch.index.tsx new file mode 100644 index 00000000..0e1e10f6 --- /dev/null +++ b/apps/web/app/routes/pitch.index.tsx @@ -0,0 +1,93 @@ +import React, { useEffect } from "react"; + +import posthog from "posthog-js"; +import { Logo } from "~/components/icons/Logo"; +import { Theme, useTheme } from "~/lib/theme-provider"; + +function PitchPage1() { + return ( +
+ +
+

+ super + memory +

+

+ The second brain platform for everyone.
+ dhravya shah draft +

+
+
+ ); +} + +function PitchPage2() { + return ( +
+

current problems

+
+
+ Messy desk with papers +
+ current knowledgebase + + + +
+
+ +
+

Knowledge Management

+

Information overload is real. Notes scattered everywhere, bookmarks lost in endless folders, important details buried in email threads. We're building this for Chakshu who's drowning in digital chaos.

+
+
+

Trust & Privacy

+

Developer friends like Kshunya and Siddharth need their data private and secure

+
+
+

Enterprise Search

+

Cloudflare's internal knowledge is a maze of confusion

+
+
+

Digital Chaos

+

Notes in Notion, bookmarks in Chrome, knowledge in Slack, wisdom in emails... it's everywhere and nowhere

+
+
+

Information Anxiety

+

Brent spends hours searching through old emails

+
+
+

Developer Cost

+

Memory APIs are a costly maze of complexity

+
+
+
+ ); +} + +function Pitch() { + const [theme, setTheme] = useTheme(); + useEffect(() => { + posthog.capture("pitch_viewed"); + setTheme(Theme.LIGHT); + }, []); + + return ( +
+
+ +
+
+ +
+ {/* Add more pages here with snap-start class */} +
+ ); +} + +export default Pitch; diff --git a/packages/db/schema.ts b/packages/db/schema.ts index a2f24b06..969e93fd 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -215,7 +215,7 @@ export const chunk = pgTable( .notNull(), textContent: text("text_content"), orderInDocument: integer("order_in_document").notNull(), - embeddings: vector("embeddings", { dimensions: 1536 }), + embeddings: vector("embeddings", { dimensions: 768 }), metadata: jsonb("metadata").$type(), createdAt: timestamp("created_at", { withTimezone: true }) .notNull()