diff --git a/apps/backend/package.json b/apps/backend/package.json index 7ac9abc2..7b961b8d 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -24,8 +24,7 @@ "zod": "^3.23.8" }, "devDependencies": { - "@cloudflare/workers-types": "^4.20240925.0", - "wrangler": "3.99.0" + "@cloudflare/workers-types": "^4.20240925.0" }, "overrides": { "iron-webcrypto": "^1.2.1" diff --git a/apps/backend/src/index.tsx b/apps/backend/src/index.tsx index 981c45b7..cb63e53e 100644 --- a/apps/backend/src/index.tsx +++ b/apps/backend/src/index.tsx @@ -81,6 +81,7 @@ export const app = new Hono<{ Variables: Variables; Bindings: Env }>() default: { windowMs: 60 * 1000, // 1 minute limit: 100, // 100 requests per minute + }, common: { diff --git a/apps/backend/src/routes/actions.ts b/apps/backend/src/routes/actions.ts index 51a01293..4de1d339 100644 --- a/apps/backend/src/routes/actions.ts +++ b/apps/backend/src/routes/actions.ts @@ -18,7 +18,7 @@ import { chatThreads, documents, chunk, - spaces, + spaces as spaceInDb, spaceAccess, type Space, } from "@supermemory/db/schema"; @@ -757,8 +757,8 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() body.spaces.map(async (spaceId) => { const space = await db .select() - .from(spaces) - .where(eq(spaces.uuid, spaceId)) + .from(spaceInDb) + .where(eq(spaceInDb.uuid, spaceId)) .limit(1); if (!space[0]) { @@ -871,10 +871,26 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() "/batch-add", zValidator( "json", - z.object({ - urls: z.array(z.string()).min(1, "At least one URL is required"), - spaces: z.array(z.string()).max(5).optional(), - }) + z + .object({ + urls: z + .array(z.string()) + .min(1, "At least one URL is required") + .optional(), + contents: z + .array( + z.object({ + content: z.string(), + title: z.string(), + type: z.string(), + }) + ) + .optional(), + spaces: z.array(z.string()).max(5).optional(), + }) + .refine((data) => data.urls || data.contents, { + message: "Either urls or contents must be provided", + }) ), async (c) => { const user = c.get("user"); @@ -882,7 +898,7 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() return c.json({ error: "Unauthorized" }, 401); } - const { urls, spaces } = await c.req.valid("json"); + const { urls, contents, spaces } = await c.req.valid("json"); // Check space permissions if spaces are specified if (spaces && spaces.length > 0) { @@ -891,8 +907,8 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() spaces.map(async (spaceId) => { const space = await db .select() - .from(spaces) - .where(eq(spaces.uuid, spaceId)) + .from(spaceInDb) + .where(eq(spaceInDb.uuid, spaceId)) .limit(1); if (!space[0]) { @@ -953,21 +969,49 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() } // Create a new ReadableStream for progress updates + const encoder = new TextEncoder(); const stream = new ReadableStream({ async start(controller) { const db = database(c.env.HYPERDRIVE.connectionString); - const total = urls.length; + const items = urls || contents || []; + const total = items.length; let processed = 0; let failed = 0; let succeeded = 0; - for (const url of urls) { + const sendMessage = (data: any) => { + const message = encoder.encode(`data: ${JSON.stringify(data)}\n\n`); + controller.enqueue(message); + }; + + for (const item of items) { try { processed++; - // Calculate document hash for duplicate detection + // Handle both URL and markdown content + const content = typeof item === "string" ? item : item.content; + const title = typeof item === "string" ? null : item.title; + const type = + typeof item === "string" ? typeDecider(item) : Ok(item.type); + + if (isErr(type)) { + failed++; + sendMessage({ + progress: Math.round((processed / total) * 100), + status: "error", + url: typeof item === "string" ? item : item.title, + error: type.error.message, + processed, + total, + succeeded, + failed, + }); + continue; + } + + // Calculate document hash const encoder = new TextEncoder(); - const data = encoder.encode(url); + const data = encoder.encode(content); const hashBuffer = await crypto.subtle.digest("SHA-256", data); const hashArray = Array.from(new Uint8Array(hashBuffer)); const documentHash = hashArray @@ -983,46 +1027,32 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() eq(documents.userId, user.id), or( eq(documents.contentHash, documentHash), - eq(documents.url, url) + eq(documents.raw, content) ) ) ); if (existingDocs.length > 0) { failed++; - controller.enqueue( - `data: ${JSON.stringify({ - progress: Math.round((processed / total) * 100), - status: "duplicate", - url, - processed, - total, - succeeded, - failed, - })}\n\n` - ); + sendMessage({ + progress: Math.round((processed / total) * 100), + status: "duplicate", + title: typeof item === "string" ? item : item.title, + processed, + total, + succeeded, + failed, + }); continue; } const contentId = `add-${user.id}-${randomId()}`; - const type = typeDecider(url); - - if (isErr(type)) { - failed++; - controller.enqueue( - `data: ${JSON.stringify({ - progress: Math.round((processed / total) * 100), - status: "error", - url, - error: type.error.message, - processed, - total, - succeeded, - failed, - })}\n\n` - ); - continue; - } + const isExternalContent = + typeof item === "string" && + ["page", "tweet", "document", "notion"].includes(type.value); + const url = isExternalContent + ? content + : `https://supermemory.ai/content/${contentId}`; // Insert into documents table await db.insert(documents).values({ @@ -1030,15 +1060,16 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() userId: user.id, type: type.value, url, + title, contentHash: documentHash, - raw: url + "\n\n" + spaces?.join(" "), + raw: content + "\n\n" + spaces?.join(" "), }); // Create workflow for processing await c.env.CONTENT_WORKFLOW.create({ params: { userId: user.id, - content: url, + content, spaces, type: type.value, uuid: contentId, @@ -1048,45 +1079,41 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>() }); succeeded++; - controller.enqueue( - `data: ${JSON.stringify({ - progress: Math.round((processed / total) * 100), - status: "success", - url, - processed, - total, - succeeded, - failed, - })}\n\n` - ); + sendMessage({ + progress: Math.round((processed / total) * 100), + status: "success", + title: typeof item === "string" ? item : item.title, + processed, + total, + succeeded, + failed, + }); + + // Add a small delay between requests + await new Promise((resolve) => setTimeout(resolve, 100)); } catch (error) { failed++; - controller.enqueue( - `data: ${JSON.stringify({ - progress: Math.round((processed / total) * 100), - status: "error", - url, - error: - error instanceof Error ? error.message : "Unknown error", - processed, - total, - succeeded, - failed, - })}\n\n` - ); + sendMessage({ + progress: Math.round((processed / total) * 100), + status: "error", + title: typeof item === "string" ? item : item.title, + error: error instanceof Error ? error.message : "Unknown error", + processed, + total, + succeeded, + failed, + }); } } - controller.enqueue( - `data: ${JSON.stringify({ - progress: 100, - status: "complete", - processed, - total, - succeeded, - failed, - })}\n\n` - ); + sendMessage({ + progress: 100, + status: "complete", + processed, + total, + succeeded, + failed, + }); controller.close(); }, }); diff --git a/apps/web/app/components/memories/CSVUploadModal.tsx b/apps/web/app/components/memories/CSVUploadModal.tsx new file mode 100644 index 00000000..88c65eed --- /dev/null +++ b/apps/web/app/components/memories/CSVUploadModal.tsx @@ -0,0 +1,267 @@ +import { useEffect, useState } from "react"; + +import { Button } from "../ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogTitle } from "../ui/dialog"; +import SpacesSelector from "./SpacesSelector"; + +import { motion } from "framer-motion"; +import { AlertCircle, CheckCircle, Upload } from "lucide-react"; +import Papa from "papaparse"; +import { toast } from "sonner"; + +interface CSVUploadModalProps { + isOpen: boolean; + onClose: () => void; +} + +export function CSVUploadModal({ isOpen, onClose }: CSVUploadModalProps) { + const [file, setFile] = useState(null); + const [urls, setUrls] = useState([]); + const [isUploading, setIsUploading] = useState(false); + const [progress, setProgress] = useState<{ + progress: number; + processed: number; + total: number; + succeeded: number; + failed: number; + status: string; + } | null>(null); + const [selectedSpaces, setSelectedSpaces] = useState([]); + + const handleFileChange = (e: React.ChangeEvent) => { + const selectedFile = e.target.files?.[0]; + if (selectedFile) { + if (!selectedFile.name.endsWith(".csv")) { + toast.error("Please upload a CSV file"); + return; + } + setFile(selectedFile); + + // Parse CSV file + Papa.parse(selectedFile, { + complete: (results) => { + // Find column containing URLs by checking first row + const firstRow = results.data[0]; + let urlColumnIndex = -1; + + // Look for a column containing URLs in the header row + // @ts-expect-error - firstRow is of type unknown + firstRow.forEach((cell: string, index: number) => { + if ( + cell?.toLowerCase().includes("url") || + (cell && typeof cell === "string" && cell.trim().startsWith("http")) + ) { + urlColumnIndex = index; + } + }); + + // If no URL column found in header, check first data row + if (urlColumnIndex === -1 && results.data[1]) { + // @ts-expect-error - results.data[1] is of type unknown + results.data[1].forEach((cell: string, index: number) => { + if (cell && typeof cell === "string" && cell.trim().startsWith("http")) { + urlColumnIndex = index; + } + }); + } + + if (urlColumnIndex === -1) { + toast.error("Could not find a column containing URLs"); + setFile(null); + return; + } + + // Extract URLs from the identified column + const validUrls = results.data + .slice(1) // Skip header row + .map((row: any) => row[urlColumnIndex]) + .filter((url: string) => url && url.trim() && url.startsWith("http")); + + if (validUrls.length === 0) { + toast.error("No valid URLs found in the CSV file"); + setFile(null); + return; + } + + setUrls(validUrls); + toast.success(`Found ${validUrls.length} valid URLs in column ${urlColumnIndex + 1}`); + }, + error: (error) => { + toast.error("Error parsing CSV file: " + error.message); + setFile(null); + }, + }); + } + }; + + const handleUpload = async () => { + if (!file || urls.length === 0) return; + + setIsUploading(true); + try { + const response = await fetch("/backend/api/batch-add", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + urls, + spaces: selectedSpaces, + }), + }); + + if (!response.ok) { + throw new Error("Failed to start batch upload"); + } + + const reader = response.body?.getReader(); + if (!reader) throw new Error("No reader available"); + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + const text = new TextDecoder().decode(value); + const lines = text.split("\n"); + + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(5)); + setProgress(data); + + if (data.status === "complete") { + toast.success( + `Batch upload complete! ${data.succeeded} succeeded, ${data.failed} failed`, + ); + setTimeout(() => { + onClose(); + setFile(null); + setUrls([]); + setProgress(null); + }, 2000); + } + } catch (e) { + console.error("Error parsing SSE data:", e); + } + } + } + } + } catch (error) { + toast.error("Upload failed: " + (error instanceof Error ? error.message : "Unknown error")); + } finally { + setIsUploading(false); + } + }; + + return ( + !isUploading && onClose()}> + + {isUploading ? ( +
+ {progress ? ( + <> +
+
+ + {progress.progress}% + +
+ + + + +
+ + Processing URLs ({progress.processed}/{progress.total}) + + + {progress.succeeded} succeeded, {progress.failed} failed + + + ) : ( + + + + )} +
+ ) : ( +
+ Upload CSV File + + Upload a CSV file containing URLs to add to your memories. The URLs should be in the + first column. + + +
+ +
+ + {file && ( + <> +
+ + + {file.name} ({urls.length} URLs found) + +
+ + + +
+ + +
+ + )} +
+ )} + + + ); +} diff --git a/apps/web/app/components/memories/Integrations.tsx b/apps/web/app/components/memories/Integrations.tsx index 3825980c..9b01796b 100644 --- a/apps/web/app/components/memories/Integrations.tsx +++ b/apps/web/app/components/memories/Integrations.tsx @@ -5,15 +5,24 @@ import { useNavigate, useRouteLoaderData } from "@remix-run/react"; import { Button } from "../ui/button"; import { Card } from "../ui/card"; import { Dialog, DialogContent, DialogDescription, DialogTitle } from "../ui/dialog"; +import { CSVUploadModal } from "./CSVUploadModal"; +import { MarkdownUploadModal } from "./MarkdownUploadModal"; import { motion } from "framer-motion"; -import { AlertCircle, CheckCircle, Clipboard, ClipboardCheckIcon, X, FileUpIcon } from "lucide-react"; +import { + AlertCircle, + BookIcon, + CheckCircle, + Clipboard, + ClipboardCheckIcon, + FileUpIcon, + X, +} from "lucide-react"; import { toast } from "sonner"; import { type IntegrationConfig, getIntegrations } from "~/config/integrations"; import { getChromeExtensionId } from "~/config/util"; import { cn } from "~/lib/utils"; import { loader } from "~/root"; -import { CSVUploadModal } from "./CSVUploadModal"; function IntegrationButton({ integration, @@ -152,6 +161,7 @@ function Integrations() { const [apiKey, setApiKey] = useState(null); const [copied, setCopied] = useState(false); const [isCSVModalOpen, setIsCSVModalOpen] = useState(false); + const [isMarkdownModalOpen, setIsMarkdownModalOpen] = useState(false); const handleIntegrationClick = (integration: IntegrationConfig) => { setLoadingIntegration(integration); @@ -256,6 +266,169 @@ function Integrations() {
+ setIsMarkdownModalOpen(true)} + > +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+

Obsidian

+

+ Import notes from your Obsidian vault +

+
+
+ + setIsCSVModalOpen(true)} @@ -291,6 +464,10 @@ function Integrations() { )} setIsCSVModalOpen(false)} /> + setIsMarkdownModalOpen(false)} + />

diff --git a/apps/web/app/components/memories/MarkdownUploadModal.tsx b/apps/web/app/components/memories/MarkdownUploadModal.tsx new file mode 100644 index 00000000..48ac196f --- /dev/null +++ b/apps/web/app/components/memories/MarkdownUploadModal.tsx @@ -0,0 +1,294 @@ +import { useEffect, useState } from "react"; + +import { Button } from "../ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogTitle } from "../ui/dialog"; +import SpacesSelector from "./SpacesSelector"; + +import { motion } from "framer-motion"; +import { AlertCircle, CheckCircle, Upload } from "lucide-react"; +import { toast } from "sonner"; + +interface MarkdownUploadModalProps { + isOpen: boolean; + onClose: () => void; +} + +export function MarkdownUploadModal({ isOpen, onClose }: MarkdownUploadModalProps) { + const [files, setFiles] = useState([]); + const [isUploading, setIsUploading] = useState(false); + const [progress, setProgress] = useState<{ + progress: number; + processed: number; + total: number; + succeeded: number; + failed: number; + status: string; + } | null>(null); + const [selectedSpaces, setSelectedSpaces] = useState([]); + + const handleFileChange = async (e: React.ChangeEvent) => { + const selectedFiles = e.target.files; + if (!selectedFiles) return; + + const mdFiles: File[] = []; + const processFile = async (file: File) => { + if (file.name.endsWith(".md")) { + mdFiles.push(file); + } + }; + + // Handle both individual files and directory + const files = Array.from(selectedFiles); + await Promise.all(files.map(processFile)); + + if (mdFiles.length === 0) { + toast.error("No markdown files found"); + return; + } + + setFiles(mdFiles); + toast.success(`Found ${mdFiles.length} markdown files`); + }; + + const handleUpload = async () => { + if (files.length === 0) return; + + setIsUploading(true); + const progressToastId = toast.loading("Starting markdown import..."); + let lastToastTime = Date.now(); + + try { + // Convert markdown files to content + const contents = await Promise.all( + files.map(async (file) => { + const content = await file.text(); + return { + content, + title: file.name.replace(".md", ""), + type: "note", + }; + }), + ); + + // Send to batch endpoint + const response = await fetch("/backend/api/batch-add", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + contents, + spaces: selectedSpaces, + }), + }); + + if (!response.ok) { + throw new Error("Failed to start batch upload"); + } + + const reader = response.body?.getReader(); + if (!reader) throw new Error("No reader available"); + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + const text = new TextDecoder().decode(value); + const lines = text.split("\n"); + + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(5)); + setProgress(data); + + // Update the main progress toast every 500ms + const now = Date.now(); + if (now - lastToastTime > 500) { + toast.loading( + `Processing files: ${data.processed}/${data.total} (${data.progress}%)`, + { id: progressToastId }, + ); + lastToastTime = now; + } + + // Show individual file status toasts with a limit + if (data.status === "success" && data.processed % 5 === 0) { + toast.success(`Successfully imported ${data.processed} files so far`); + } else if (data.status === "error") { + toast.error(`Failed to import: ${data.title} - ${data.error}`); + } + + if (data.status === "complete") { + toast.success( + `Import complete! ${data.succeeded} succeeded, ${data.failed} failed`, + { id: progressToastId, duration: 5000 }, + ); + + // Wait for 2 seconds before closing + await new Promise((resolve) => setTimeout(resolve, 2000)); + onClose(); + setFiles([]); + setProgress(null); + break; + } + } catch (e) { + console.error("Error parsing SSE data:", e); + } + } + } + } + } catch (error) { + toast.error("Upload failed: " + (error instanceof Error ? error.message : "Unknown error")); + } finally { + setIsUploading(false); + } + }; + + return ( +

!isUploading && onClose()}> + + {isUploading ? ( +
+ {progress ? ( + <> +
+
+ + {progress.progress}% + +
+ + + + +
+ + Processing Files ({progress.processed}/{progress.total}) + + + {progress.succeeded} succeeded, {progress.failed} failed + + + ) : ( + + + + )} +
+ ) : ( +
+ Import from Obsidian + + Upload markdown files from your Obsidian vault. You can select multiple files or drop + a folder. + + +
+ {/* Individual Files Selection */} +
+ +
+ +
OR
+ + {/* Folder Selection */} +
+ +
+
+ + {files.length > 0 && ( + <> +
+ + + {files.length} markdown files selected + +
+ + + +
+ + +
+ + )} +
+ )} + + + ); +} diff --git a/apps/web/package.json b/apps/web/package.json index 1635235d..202d795f 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -6,7 +6,7 @@ "engineStrict": true, "packageManager": "bun@1.1.29", "scripts": { - "build": "bun i remix && remix vite:build", + "build": "remix vite:build", "cf-typegen": "wrangler types", "deploy": "cross-env NODE_ENV=production IS_DEPLOYING=true dotenv -- bun run build && wrangler pages deploy", "dev": "dotenv -- wrangler -v && remix vite:dev --port 3000", @@ -168,8 +168,7 @@ "tailwindcss": "^3.4.13", "typescript": "^5.6.2", "vite": "^5.4.8", - "vite-tsconfig-paths": "^5.0.1", - "wrangler": "3.99.0" + "vite-tsconfig-paths": "^5.0.1" }, "engines": { "node": ">=20.0.0" diff --git a/package.json b/package.json index 5c627306..5031ce4d 100644 --- a/package.json +++ b/package.json @@ -44,10 +44,10 @@ "@tanstack/react-query-devtools": "^5.60.6", "@types/chrome": "^0.0.287", "@types/node": "^22.10.1", + "@types/papaparse": "^5.3.15", "@types/postlight__mercury-parser": "^2.2.7", "@types/showdown": "^2.0.6", "@types/turndown": "^5.0.5", - "@upstash/ratelimit": "^2.0.5", "ai": "4.0.18", "autoevals": "^0.0.106", "aws4fetch": "^1.0.20", @@ -63,6 +63,7 @@ "mammoth": "^1.8.0", "nanoid": "^5.0.7", "notion-client": "^7.1.5", + "papaparse": "^5.5.1", "pdfjs-serverless": "^0.6.0", "postgres": "^3.4.4", "posthog-js": "^1.188.0", @@ -78,7 +79,7 @@ "shiki": "^1.22.1", "sonner": "^1.7.0", "web": "^0.0.2", - "wrangler": "3.99.0" + "wrangler": "latest" }, "overrides": { "iron-webcrypto": "^1.2.1",