import tools: CSV and markdown (obsidian)

This commit is contained in:
Dhravya Shah 2025-01-22 23:10:19 -07:00
parent 0c0ea871c3
commit 75fb461501
8 changed files with 854 additions and 89 deletions

View file

@ -24,8 +24,7 @@
"zod": "^3.23.8"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20240925.0",
"wrangler": "3.99.0"
"@cloudflare/workers-types": "^4.20240925.0"
},
"overrides": {
"iron-webcrypto": "^1.2.1"

View file

@ -81,6 +81,7 @@ export const app = new Hono<{ Variables: Variables; Bindings: Env }>()
default: {
windowMs: 60 * 1000, // 1 minute
limit: 100, // 100 requests per minute
},
common: {

View file

@ -18,7 +18,7 @@ import {
chatThreads,
documents,
chunk,
spaces,
spaces as spaceInDb,
spaceAccess,
type Space,
} from "@supermemory/db/schema";
@ -757,8 +757,8 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
body.spaces.map(async (spaceId) => {
const space = await db
.select()
.from(spaces)
.where(eq(spaces.uuid, spaceId))
.from(spaceInDb)
.where(eq(spaceInDb.uuid, spaceId))
.limit(1);
if (!space[0]) {
@ -871,10 +871,26 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
"/batch-add",
zValidator(
"json",
z.object({
urls: z.array(z.string()).min(1, "At least one URL is required"),
spaces: z.array(z.string()).max(5).optional(),
})
z
.object({
urls: z
.array(z.string())
.min(1, "At least one URL is required")
.optional(),
contents: z
.array(
z.object({
content: z.string(),
title: z.string(),
type: z.string(),
})
)
.optional(),
spaces: z.array(z.string()).max(5).optional(),
})
.refine((data) => data.urls || data.contents, {
message: "Either urls or contents must be provided",
})
),
async (c) => {
const user = c.get("user");
@ -882,7 +898,7 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
return c.json({ error: "Unauthorized" }, 401);
}
const { urls, spaces } = await c.req.valid("json");
const { urls, contents, spaces } = await c.req.valid("json");
// Check space permissions if spaces are specified
if (spaces && spaces.length > 0) {
@ -891,8 +907,8 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
spaces.map(async (spaceId) => {
const space = await db
.select()
.from(spaces)
.where(eq(spaces.uuid, spaceId))
.from(spaceInDb)
.where(eq(spaceInDb.uuid, spaceId))
.limit(1);
if (!space[0]) {
@ -953,21 +969,49 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
}
// Create a new ReadableStream for progress updates
const encoder = new TextEncoder();
const stream = new ReadableStream({
async start(controller) {
const db = database(c.env.HYPERDRIVE.connectionString);
const total = urls.length;
const items = urls || contents || [];
const total = items.length;
let processed = 0;
let failed = 0;
let succeeded = 0;
for (const url of urls) {
const sendMessage = (data: any) => {
const message = encoder.encode(`data: ${JSON.stringify(data)}\n\n`);
controller.enqueue(message);
};
for (const item of items) {
try {
processed++;
// Calculate document hash for duplicate detection
// Handle both URL and markdown content
const content = typeof item === "string" ? item : item.content;
const title = typeof item === "string" ? null : item.title;
const type =
typeof item === "string" ? typeDecider(item) : Ok(item.type);
if (isErr(type)) {
failed++;
sendMessage({
progress: Math.round((processed / total) * 100),
status: "error",
url: typeof item === "string" ? item : item.title,
error: type.error.message,
processed,
total,
succeeded,
failed,
});
continue;
}
// Calculate document hash
const encoder = new TextEncoder();
const data = encoder.encode(url);
const data = encoder.encode(content);
const hashBuffer = await crypto.subtle.digest("SHA-256", data);
const hashArray = Array.from(new Uint8Array(hashBuffer));
const documentHash = hashArray
@ -983,46 +1027,32 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
eq(documents.userId, user.id),
or(
eq(documents.contentHash, documentHash),
eq(documents.url, url)
eq(documents.raw, content)
)
)
);
if (existingDocs.length > 0) {
failed++;
controller.enqueue(
`data: ${JSON.stringify({
progress: Math.round((processed / total) * 100),
status: "duplicate",
url,
processed,
total,
succeeded,
failed,
})}\n\n`
);
sendMessage({
progress: Math.round((processed / total) * 100),
status: "duplicate",
title: typeof item === "string" ? item : item.title,
processed,
total,
succeeded,
failed,
});
continue;
}
const contentId = `add-${user.id}-${randomId()}`;
const type = typeDecider(url);
if (isErr(type)) {
failed++;
controller.enqueue(
`data: ${JSON.stringify({
progress: Math.round((processed / total) * 100),
status: "error",
url,
error: type.error.message,
processed,
total,
succeeded,
failed,
})}\n\n`
);
continue;
}
const isExternalContent =
typeof item === "string" &&
["page", "tweet", "document", "notion"].includes(type.value);
const url = isExternalContent
? content
: `https://supermemory.ai/content/${contentId}`;
// Insert into documents table
await db.insert(documents).values({
@ -1030,15 +1060,16 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
userId: user.id,
type: type.value,
url,
title,
contentHash: documentHash,
raw: url + "\n\n" + spaces?.join(" "),
raw: content + "\n\n" + spaces?.join(" "),
});
// Create workflow for processing
await c.env.CONTENT_WORKFLOW.create({
params: {
userId: user.id,
content: url,
content,
spaces,
type: type.value,
uuid: contentId,
@ -1048,45 +1079,41 @@ const actions = new Hono<{ Variables: Variables; Bindings: Env }>()
});
succeeded++;
controller.enqueue(
`data: ${JSON.stringify({
progress: Math.round((processed / total) * 100),
status: "success",
url,
processed,
total,
succeeded,
failed,
})}\n\n`
);
sendMessage({
progress: Math.round((processed / total) * 100),
status: "success",
title: typeof item === "string" ? item : item.title,
processed,
total,
succeeded,
failed,
});
// Add a small delay between requests
await new Promise((resolve) => setTimeout(resolve, 100));
} catch (error) {
failed++;
controller.enqueue(
`data: ${JSON.stringify({
progress: Math.round((processed / total) * 100),
status: "error",
url,
error:
error instanceof Error ? error.message : "Unknown error",
processed,
total,
succeeded,
failed,
})}\n\n`
);
sendMessage({
progress: Math.round((processed / total) * 100),
status: "error",
title: typeof item === "string" ? item : item.title,
error: error instanceof Error ? error.message : "Unknown error",
processed,
total,
succeeded,
failed,
});
}
}
controller.enqueue(
`data: ${JSON.stringify({
progress: 100,
status: "complete",
processed,
total,
succeeded,
failed,
})}\n\n`
);
sendMessage({
progress: 100,
status: "complete",
processed,
total,
succeeded,
failed,
});
controller.close();
},
});

View file

@ -0,0 +1,267 @@
import { useEffect, useState } from "react";
import { Button } from "../ui/button";
import { Dialog, DialogContent, DialogDescription, DialogTitle } from "../ui/dialog";
import SpacesSelector from "./SpacesSelector";
import { motion } from "framer-motion";
import { AlertCircle, CheckCircle, Upload } from "lucide-react";
import Papa from "papaparse";
import { toast } from "sonner";
interface CSVUploadModalProps {
isOpen: boolean;
onClose: () => void;
}
export function CSVUploadModal({ isOpen, onClose }: CSVUploadModalProps) {
const [file, setFile] = useState<File | null>(null);
const [urls, setUrls] = useState<string[]>([]);
const [isUploading, setIsUploading] = useState(false);
const [progress, setProgress] = useState<{
progress: number;
processed: number;
total: number;
succeeded: number;
failed: number;
status: string;
} | null>(null);
const [selectedSpaces, setSelectedSpaces] = useState<string[]>([]);
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const selectedFile = e.target.files?.[0];
if (selectedFile) {
if (!selectedFile.name.endsWith(".csv")) {
toast.error("Please upload a CSV file");
return;
}
setFile(selectedFile);
// Parse CSV file
Papa.parse(selectedFile, {
complete: (results) => {
// Find column containing URLs by checking first row
const firstRow = results.data[0];
let urlColumnIndex = -1;
// Look for a column containing URLs in the header row
// @ts-expect-error - firstRow is of type unknown
firstRow.forEach((cell: string, index: number) => {
if (
cell?.toLowerCase().includes("url") ||
(cell && typeof cell === "string" && cell.trim().startsWith("http"))
) {
urlColumnIndex = index;
}
});
// If no URL column found in header, check first data row
if (urlColumnIndex === -1 && results.data[1]) {
// @ts-expect-error - results.data[1] is of type unknown
results.data[1].forEach((cell: string, index: number) => {
if (cell && typeof cell === "string" && cell.trim().startsWith("http")) {
urlColumnIndex = index;
}
});
}
if (urlColumnIndex === -1) {
toast.error("Could not find a column containing URLs");
setFile(null);
return;
}
// Extract URLs from the identified column
const validUrls = results.data
.slice(1) // Skip header row
.map((row: any) => row[urlColumnIndex])
.filter((url: string) => url && url.trim() && url.startsWith("http"));
if (validUrls.length === 0) {
toast.error("No valid URLs found in the CSV file");
setFile(null);
return;
}
setUrls(validUrls);
toast.success(`Found ${validUrls.length} valid URLs in column ${urlColumnIndex + 1}`);
},
error: (error) => {
toast.error("Error parsing CSV file: " + error.message);
setFile(null);
},
});
}
};
const handleUpload = async () => {
if (!file || urls.length === 0) return;
setIsUploading(true);
try {
const response = await fetch("/backend/api/batch-add", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
urls,
spaces: selectedSpaces,
}),
});
if (!response.ok) {
throw new Error("Failed to start batch upload");
}
const reader = response.body?.getReader();
if (!reader) throw new Error("No reader available");
while (true) {
const { done, value } = await reader.read();
if (done) break;
const text = new TextDecoder().decode(value);
const lines = text.split("\n");
for (const line of lines) {
if (line.startsWith("data: ")) {
try {
const data = JSON.parse(line.slice(5));
setProgress(data);
if (data.status === "complete") {
toast.success(
`Batch upload complete! ${data.succeeded} succeeded, ${data.failed} failed`,
);
setTimeout(() => {
onClose();
setFile(null);
setUrls([]);
setProgress(null);
}, 2000);
}
} catch (e) {
console.error("Error parsing SSE data:", e);
}
}
}
}
} catch (error) {
toast.error("Upload failed: " + (error instanceof Error ? error.message : "Unknown error"));
} finally {
setIsUploading(false);
}
};
return (
<Dialog open={isOpen} onOpenChange={() => !isUploading && onClose()}>
<DialogContent className="sm:max-w-md">
{isUploading ? (
<div className="text-center">
{progress ? (
<>
<div className="relative">
<div className="absolute inset-0 flex items-center justify-center">
<span className="text-lg font-semibold text-blue-500">
{progress.progress}%
</span>
</div>
<svg className="size-20 md:size-24 -rotate-90 transform">
<circle
className="text-neutral-200 dark:text-neutral-700"
strokeWidth="6"
stroke="currentColor"
fill="transparent"
r="45"
cx="48"
cy="48"
/>
<circle
className="text-blue-500 transition-all duration-300"
strokeWidth="6"
strokeDasharray={283}
strokeDashoffset={283 - (283 * progress.progress) / 100}
strokeLinecap="round"
stroke="currentColor"
fill="transparent"
r="45"
cx="48"
cy="48"
/>
</svg>
</div>
<DialogTitle className="mt-4">
Processing URLs ({progress.processed}/{progress.total})
</DialogTitle>
<DialogDescription className="mt-2">
{progress.succeeded} succeeded, {progress.failed} failed
</DialogDescription>
</>
) : (
<motion.div
animate={{ rotate: 360 }}
transition={{ duration: 2, repeat: Infinity, ease: "linear" }}
className="w-12 h-12 mx-auto"
>
<Upload className="h-full w-full text-neutral-900 dark:text-white" />
</motion.div>
)}
</div>
) : (
<div className="space-y-4">
<DialogTitle>Upload CSV File</DialogTitle>
<DialogDescription>
Upload a CSV file containing URLs to add to your memories. The URLs should be in the
first column.
</DialogDescription>
<div className="flex flex-col items-center justify-center w-full">
<label
htmlFor="csv-upload"
className="flex flex-col items-center justify-center w-full h-64 border-2 border-dashed rounded-lg cursor-pointer bg-gray-50 dark:bg-zinc-900 border-gray-300 dark:border-gray-600 hover:bg-gray-100 dark:hover:bg-zinc-800"
>
<div className="flex flex-col items-center justify-center pt-5 pb-6">
<Upload className="w-8 h-8 mb-4 text-gray-500 dark:text-gray-400" />
<p className="mb-2 text-sm text-gray-500 dark:text-gray-400">
<span className="font-semibold">Click to upload</span> or drag and drop
</p>
<p className="text-xs text-gray-500 dark:text-gray-400">CSV files only</p>
</div>
<input
id="csv-upload"
type="file"
className="hidden"
accept=".csv"
onChange={handleFileChange}
/>
</label>
</div>
{file && (
<>
<div className="flex items-center gap-2 p-2 bg-gray-50 dark:bg-zinc-900 rounded border border-gray-200 dark:border-gray-700">
<CheckCircle className="h-4 w-4 text-green-500" />
<span className="text-sm text-gray-700 dark:text-gray-300">
{file.name} ({urls.length} URLs found)
</span>
</div>
<SpacesSelector selectedSpaces={selectedSpaces} onChange={setSelectedSpaces} />
<div className="flex justify-end gap-2">
<Button variant="outline" onClick={onClose}>
Cancel
</Button>
<Button onClick={handleUpload} disabled={urls.length === 0}>
Upload
</Button>
</div>
</>
)}
</div>
)}
</DialogContent>
</Dialog>
);
}

View file

@ -5,15 +5,24 @@ import { useNavigate, useRouteLoaderData } from "@remix-run/react";
import { Button } from "../ui/button";
import { Card } from "../ui/card";
import { Dialog, DialogContent, DialogDescription, DialogTitle } from "../ui/dialog";
import { CSVUploadModal } from "./CSVUploadModal";
import { MarkdownUploadModal } from "./MarkdownUploadModal";
import { motion } from "framer-motion";
import { AlertCircle, CheckCircle, Clipboard, ClipboardCheckIcon, X, FileUpIcon } from "lucide-react";
import {
AlertCircle,
BookIcon,
CheckCircle,
Clipboard,
ClipboardCheckIcon,
FileUpIcon,
X,
} from "lucide-react";
import { toast } from "sonner";
import { type IntegrationConfig, getIntegrations } from "~/config/integrations";
import { getChromeExtensionId } from "~/config/util";
import { cn } from "~/lib/utils";
import { loader } from "~/root";
import { CSVUploadModal } from "./CSVUploadModal";
function IntegrationButton({
integration,
@ -152,6 +161,7 @@ function Integrations() {
const [apiKey, setApiKey] = useState<string | null>(null);
const [copied, setCopied] = useState(false);
const [isCSVModalOpen, setIsCSVModalOpen] = useState(false);
const [isMarkdownModalOpen, setIsMarkdownModalOpen] = useState(false);
const handleIntegrationClick = (integration: IntegrationConfig) => {
setLoadingIntegration(integration);
@ -256,6 +266,169 @@ function Integrations() {
</div>
<div className="flex flex-wrap gap-4 overflow-x-auto">
<Card
className="group relative overflow-hidden transition-all hover:shadow-lg flex-1 basis-[calc(33.333%-1rem)]"
onClick={() => setIsMarkdownModalOpen(true)}
>
<div className="absolute inset-0 opacity-0 transition-opacity group-hover:opacity-100 bg-gradient-to-r from-purple-500/10 to-purple-600/10" />
<div className="relative z-10 flex flex-col items-center gap-4 p-6">
<div className="rounded-full bg-white/10 p-3">
<svg
xmlns="http://www.w3.org/2000/svg"
className="w-6 h-6"
preserveAspectRatio="xMidYMid"
viewBox="0 0 256 332"
>
<defs>
<radialGradient
id="a"
cx="72.819%"
cy="96.934%"
r="163.793%"
fx="72.819%"
fy="96.934%"
gradientTransform="rotate(-104 11141.322 0)"
>
<stop offset="0%" stop-color="#FFF" stop-opacity=".4" />
<stop offset="100%" stop-opacity=".1" />
</radialGradient>
<radialGradient
id="b"
cx="52.917%"
cy="90.632%"
r="190.361%"
fx="52.917%"
fy="90.632%"
gradientTransform="rotate(-82 10746.75 0)"
>
<stop offset="0%" stop-color="#FFF" stop-opacity=".6" />
<stop offset="100%" stop-color="#FFF" stop-opacity=".1" />
</radialGradient>
<radialGradient
id="c"
cx="31.174%"
cy="97.138%"
r="178.714%"
fx="31.174%"
fy="97.138%"
gradientTransform="rotate(-77 10724.606 0)"
>
<stop offset="0%" stop-color="#FFF" stop-opacity=".8" />
<stop offset="100%" stop-color="#FFF" stop-opacity=".4" />
</radialGradient>
<radialGradient
id="d"
cx="71.813%"
cy="99.994%"
r="92.086%"
fx="71.813%"
fy="99.994%"
gradientTransform="translate(0 22251839.658) skewY(-90)"
>
<stop offset="0%" stop-color="#FFF" stop-opacity=".3" />
<stop offset="100%" stop-opacity=".3" />
</radialGradient>
<radialGradient
id="e"
cx="117.013%"
cy="34.769%"
r="328.729%"
fx="117.013%"
fy="34.769%"
gradientTransform="rotate(102 -1004.443 0)"
>
<stop offset="0%" stop-color="#FFF" stop-opacity="0" />
<stop offset="100%" stop-color="#FFF" stop-opacity=".2" />
</radialGradient>
<radialGradient
id="f"
cx="-9.431%"
cy="8.712%"
r="153.492%"
fx="-9.431%"
fy="8.712%"
gradientTransform="rotate(45 1674.397 0)"
>
<stop offset="0%" stop-color="#FFF" stop-opacity=".2" />
<stop offset="100%" stop-color="#FFF" stop-opacity=".4" />
</radialGradient>
<radialGradient
id="g"
cx="103.902%"
cy="-22.172%"
r="394.771%"
fx="103.902%"
fy="-22.172%"
gradientTransform="rotate(80 3757.522 0)"
>
<stop offset="0%" stop-color="#FFF" stop-opacity=".1" />
<stop offset="100%" stop-color="#FFF" stop-opacity=".3" />
</radialGradient>
<radialGradient
id="h"
cx="99.348%"
cy="89.193%"
r="203.824%"
fx="99.348%"
fy="89.193%"
gradientTransform="translate(0 -38783246.548) skewY(-90)"
>
<stop offset="0%" stop-color="#FFF" stop-opacity=".2" />
<stop offset="50%" stop-color="#FFF" stop-opacity=".2" />
<stop offset="100%" stop-color="#FFF" stop-opacity=".3" />
</radialGradient>
</defs>
<path
fill-opacity=".3"
d="M209.056 308.305c-2.043 14.93-16.738 26.638-31.432 22.552-20.823-5.658-44.946-14.616-66.634-16.266l-33.317-2.515a22.002 22.002 0 0 1-14.144-6.522L6.167 246.778a21.766 21.766 0 0 1-4.244-24.124s35.36-77.478 36.775-81.485c1.257-4.008 6.13-39.211 8.958-58.07a22.002 22.002 0 0 1 7.072-12.965L122.462 9.47a22.002 22.002 0 0 1 31.903 2.672l57.048 71.978a23.18 23.18 0 0 1 4.872 14.38c0 13.594 1.179 41.646 8.8 59.72a236.756 236.756 0 0 0 27.974 45.732 11.001 11.001 0 0 1 .786 12.258c-4.95 8.408-14.851 24.595-28.76 45.26a111.738 111.738 0 0 0-16.108 46.834h.079Z"
/>
<path
fill="#6C31E3"
d="M209.606 305.79c-2.043 15.009-16.737 26.717-31.432 22.71-20.744-5.737-44.79-14.695-66.555-16.345L78.38 309.64a21.923 21.923 0 0 1-14.144-6.6L6.874 244.106a21.923 21.923 0 0 1-4.243-24.36s35.438-77.792 36.774-81.878c1.336-4.007 6.13-39.289 8.958-58.305a22.002 22.002 0 0 1 7.072-13.044L123.17 5.621a22.002 22.002 0 0 1 31.902 2.75l56.97 72.292a23.338 23.338 0 0 1 4.871 14.38c0 13.673 1.18 41.804 8.723 59.955a238.092 238.092 0 0 0 27.974 45.969 11.001 11.001 0 0 1 .864 12.336c-5.03 8.487-14.851 24.674-28.838 45.497a112.603 112.603 0 0 0-16.03 46.99Z"
/>
<path
fill="url(#a)"
d="M70.365 307.44c26.638-53.983 25.93-92.722 14.537-120.225-10.372-25.459-29.781-41.489-45.025-51.468a19.233 19.233 0 0 1-1.415 4.243L2.631 219.747a21.923 21.923 0 0 0 4.321 24.36l57.284 58.933a23.762 23.762 0 0 0 6.129 4.4Z"
/>
<path
fill="url(#b)"
d="M142.814 197.902a86.025 86.025 0 0 1 21.06 4.793c21.844 8.172 41.724 26.56 58.147 61.999 1.179-2.043 2.357-4.008 3.615-5.894a960.226 960.226 0 0 0 28.838-45.497 11.001 11.001 0 0 0-.786-12.336 238.092 238.092 0 0 1-28.052-45.969c-7.544-18.073-8.644-46.282-8.723-59.955 0-5.186-1.65-10.294-4.871-14.38l-56.97-72.292-.943-1.178c4.165 13.75 3.93 24.752 1.336 34.731-2.357 9.272-6.757 17.68-11.394 26.56-1.571 2.986-3.143 6.05-4.636 9.193a110.01 110.01 0 0 0-12.415 45.576c-.786 19.016 3.064 42.825 15.716 74.65h.078Z"
/>
<path
fill="url(#c)"
d="M142.736 197.902c-12.652-31.824-16.502-55.633-15.716-74.65.786-18.858 6.286-33.002 12.415-45.575l4.715-9.193c4.558-8.88 8.88-17.288 11.315-26.56a61.684 61.684 0 0 0-1.336-34.731c-8.136-8.94-21.96-9.642-30.96-1.572L55.436 66.519a22.002 22.002 0 0 0-7.072 13.044l-8.25 54.69c0 .55-.158 1.022-.236 1.572 15.244 9.901 34.574 25.931 45.025 51.312 2.043 5.029 3.772 10.294 5.029 16.03a157.157 157.157 0 0 1 52.805-5.343v.078Z"
/>
<path
fill="url(#d)"
d="M178.253 328.5c14.616 4.007 29.31-7.701 31.353-22.789a120.225 120.225 0 0 1 12.494-41.017c-16.502-35.44-36.382-53.827-58.148-61.999-23.18-8.643-48.404-5.736-74.021.472 5.736 26.01 2.357 60.034-19.487 104.273 2.436 1.257 5.186 1.965 7.936 2.2l34.496 2.593c18.701 1.336 46.597 11.001 65.377 16.266Z"
/>
<path
fill="url(#e)"
d="M127.177 122.074c-.864 18.859 1.493 40.39 14.144 72.135l-3.929-.393c-11.394-33.081-13.908-50.054-13.044-69.149.786-19.094 6.994-33.789 13.123-46.361 1.571-3.143 5.186-9.037 6.758-12.023 4.557-8.879 7.622-13.515 10.215-21.609 3.772-11.315 2.986-16.658 2.514-22.001 2.908 19.251-8.172 35.988-16.501 53.04a113.939 113.939 0 0 0-13.358 46.361h.078Z"
/>
<path
fill="url(#f)"
d="M88.674 188.551c1.571 3.458 2.907 6.287 3.85 10.608l-3.379.786c-1.336-5.029-2.357-8.643-4.322-12.965-11.472-26.953-29.86-40.861-44.79-51.076 18.074 9.744 36.697 25.066 48.64 52.647Z"
/>
<path
fill="url(#g)"
d="M92.681 202.617c6.286 29.467-.786 66.948-21.609 103.409 17.445-36.146 25.931-70.8 18.859-102.938l2.75-.55v.079Z"
/>
<path
fill="url(#h)"
d="M164.659 199.867c34.181 12.808 47.383 40.86 57.205 64.355-12.18-24.516-29.074-51.626-58.462-61.684-22.317-7.7-41.175-6.758-73.471.55l-.707-3.143c34.26-7.858 52.176-8.8 75.435 0v-.078Z"
/>
</svg>
</div>
<div className="text-center">
<h3 className="font-semibold text-neutral-900 dark:text-white">Obsidian</h3>
<p className="mt-1 text-sm text-neutral-500 dark:text-neutral-400 hidden md:block">
Import notes from your Obsidian vault
</p>
</div>
</div>
</Card>
<Card
className="group relative overflow-hidden transition-all hover:shadow-lg flex-1 basis-[calc(33.333%-1rem)]"
onClick={() => setIsCSVModalOpen(true)}
@ -291,6 +464,10 @@ function Integrations() {
)}
<CSVUploadModal isOpen={isCSVModalOpen} onClose={() => setIsCSVModalOpen(false)} />
<MarkdownUploadModal
isOpen={isMarkdownModalOpen}
onClose={() => setIsMarkdownModalOpen(false)}
/>
<div className="mt-8 md:mt-12 text-center">
<p className="text-sm text-neutral-600 dark:text-neutral-400">

View file

@ -0,0 +1,294 @@
import { useEffect, useState } from "react";
import { Button } from "../ui/button";
import { Dialog, DialogContent, DialogDescription, DialogTitle } from "../ui/dialog";
import SpacesSelector from "./SpacesSelector";
import { motion } from "framer-motion";
import { AlertCircle, CheckCircle, Upload } from "lucide-react";
import { toast } from "sonner";
interface MarkdownUploadModalProps {
isOpen: boolean;
onClose: () => void;
}
export function MarkdownUploadModal({ isOpen, onClose }: MarkdownUploadModalProps) {
const [files, setFiles] = useState<File[]>([]);
const [isUploading, setIsUploading] = useState(false);
const [progress, setProgress] = useState<{
progress: number;
processed: number;
total: number;
succeeded: number;
failed: number;
status: string;
} | null>(null);
const [selectedSpaces, setSelectedSpaces] = useState<string[]>([]);
const handleFileChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
const selectedFiles = e.target.files;
if (!selectedFiles) return;
const mdFiles: File[] = [];
const processFile = async (file: File) => {
if (file.name.endsWith(".md")) {
mdFiles.push(file);
}
};
// Handle both individual files and directory
const files = Array.from(selectedFiles);
await Promise.all(files.map(processFile));
if (mdFiles.length === 0) {
toast.error("No markdown files found");
return;
}
setFiles(mdFiles);
toast.success(`Found ${mdFiles.length} markdown files`);
};
const handleUpload = async () => {
if (files.length === 0) return;
setIsUploading(true);
const progressToastId = toast.loading("Starting markdown import...");
let lastToastTime = Date.now();
try {
// Convert markdown files to content
const contents = await Promise.all(
files.map(async (file) => {
const content = await file.text();
return {
content,
title: file.name.replace(".md", ""),
type: "note",
};
}),
);
// Send to batch endpoint
const response = await fetch("/backend/api/batch-add", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
contents,
spaces: selectedSpaces,
}),
});
if (!response.ok) {
throw new Error("Failed to start batch upload");
}
const reader = response.body?.getReader();
if (!reader) throw new Error("No reader available");
while (true) {
const { done, value } = await reader.read();
if (done) break;
const text = new TextDecoder().decode(value);
const lines = text.split("\n");
for (const line of lines) {
if (line.startsWith("data: ")) {
try {
const data = JSON.parse(line.slice(5));
setProgress(data);
// Update the main progress toast every 500ms
const now = Date.now();
if (now - lastToastTime > 500) {
toast.loading(
`Processing files: ${data.processed}/${data.total} (${data.progress}%)`,
{ id: progressToastId },
);
lastToastTime = now;
}
// Show individual file status toasts with a limit
if (data.status === "success" && data.processed % 5 === 0) {
toast.success(`Successfully imported ${data.processed} files so far`);
} else if (data.status === "error") {
toast.error(`Failed to import: ${data.title} - ${data.error}`);
}
if (data.status === "complete") {
toast.success(
`Import complete! ${data.succeeded} succeeded, ${data.failed} failed`,
{ id: progressToastId, duration: 5000 },
);
// Wait for 2 seconds before closing
await new Promise((resolve) => setTimeout(resolve, 2000));
onClose();
setFiles([]);
setProgress(null);
break;
}
} catch (e) {
console.error("Error parsing SSE data:", e);
}
}
}
}
} catch (error) {
toast.error("Upload failed: " + (error instanceof Error ? error.message : "Unknown error"));
} finally {
setIsUploading(false);
}
};
return (
<Dialog open={isOpen} onOpenChange={() => !isUploading && onClose()}>
<DialogContent className="sm:max-w-md">
{isUploading ? (
<div className="text-center">
{progress ? (
<>
<div className="relative">
<div className="absolute inset-0 flex items-center justify-center">
<span className="text-lg font-semibold text-blue-500">
{progress.progress}%
</span>
</div>
<svg className="size-20 md:size-24 -rotate-90 transform">
<circle
className="text-neutral-200 dark:text-neutral-700"
strokeWidth="6"
stroke="currentColor"
fill="transparent"
r="45"
cx="48"
cy="48"
/>
<circle
className="text-blue-500 transition-all duration-300"
strokeWidth="6"
strokeDasharray={283}
strokeDashoffset={283 - (283 * progress.progress) / 100}
strokeLinecap="round"
stroke="currentColor"
fill="transparent"
r="45"
cx="48"
cy="48"
/>
</svg>
</div>
<DialogTitle className="mt-4">
Processing Files ({progress.processed}/{progress.total})
</DialogTitle>
<DialogDescription className="mt-2">
{progress.succeeded} succeeded, {progress.failed} failed
</DialogDescription>
</>
) : (
<motion.div
animate={{ rotate: 360 }}
transition={{ duration: 2, repeat: Infinity, ease: "linear" }}
className="w-12 h-12 mx-auto"
>
<Upload className="h-full w-full text-neutral-900 dark:text-white" />
</motion.div>
)}
</div>
) : (
<div className="space-y-4">
<DialogTitle>Import from Obsidian</DialogTitle>
<DialogDescription>
Upload markdown files from your Obsidian vault. You can select multiple files or drop
a folder.
</DialogDescription>
<div className="flex flex-col gap-4">
{/* Individual Files Selection */}
<div className="flex flex-col items-center justify-center w-full">
<label
htmlFor="markdown-files-upload"
className="flex flex-col items-center justify-center w-full h-32 border-2 border-dashed rounded-lg cursor-pointer bg-gray-50 dark:bg-zinc-900 border-gray-300 dark:border-gray-600 hover:bg-gray-100 dark:hover:bg-zinc-800"
>
<div className="flex flex-col items-center justify-center pt-5 pb-6">
<Upload className="w-6 h-6 mb-2 text-gray-500 dark:text-gray-400" />
<p className="text-sm text-gray-500 dark:text-gray-400">
<span className="font-semibold">Select Files</span>
</p>
<p className="text-xs text-gray-500 dark:text-gray-400">
Choose individual markdown files
</p>
</div>
<input
id="markdown-files-upload"
type="file"
className="hidden"
accept=".md"
multiple
onChange={handleFileChange}
/>
</label>
</div>
<div className="text-center text-sm text-gray-500 dark:text-gray-400">OR</div>
{/* Folder Selection */}
<div className="flex flex-col items-center justify-center w-full">
<label
htmlFor="markdown-folder-upload"
className="flex flex-col items-center justify-center w-full h-32 border-2 border-dashed rounded-lg cursor-pointer bg-gray-50 dark:bg-zinc-900 border-gray-300 dark:border-gray-600 hover:bg-gray-100 dark:hover:bg-zinc-800"
>
<div className="flex flex-col items-center justify-center pt-5 pb-6">
<Upload className="w-6 h-6 mb-2 text-gray-500 dark:text-gray-400" />
<p className="text-sm text-gray-500 dark:text-gray-400">
<span className="font-semibold">Select Folder</span>
</p>
<p className="text-xs text-gray-500 dark:text-gray-400">
Choose an entire folder
</p>
</div>
<input
id="markdown-folder-upload"
type="file"
className="hidden"
accept=".md"
multiple
// @ts-ignore - webkitdirectory is a non-standard attribute
webkitdirectory=""
// @ts-ignore - directory is a non-standard attribute
directory=""
onChange={handleFileChange}
/>
</label>
</div>
</div>
{files.length > 0 && (
<>
<div className="flex items-center gap-2 p-2 bg-gray-50 dark:bg-zinc-900 rounded border border-gray-200 dark:border-gray-700">
<CheckCircle className="h-4 w-4 text-green-500" />
<span className="text-sm text-gray-700 dark:text-gray-300">
{files.length} markdown files selected
</span>
</div>
<SpacesSelector selectedSpaces={selectedSpaces} onChange={setSelectedSpaces} />
<div className="flex justify-end gap-2">
<Button variant="outline" onClick={onClose}>
Cancel
</Button>
<Button onClick={handleUpload}>Import</Button>
</div>
</>
)}
</div>
)}
</DialogContent>
</Dialog>
);
}

View file

@ -6,7 +6,7 @@
"engineStrict": true,
"packageManager": "bun@1.1.29",
"scripts": {
"build": "bun i remix && remix vite:build",
"build": "remix vite:build",
"cf-typegen": "wrangler types",
"deploy": "cross-env NODE_ENV=production IS_DEPLOYING=true dotenv -- bun run build && wrangler pages deploy",
"dev": "dotenv -- wrangler -v && remix vite:dev --port 3000",
@ -168,8 +168,7 @@
"tailwindcss": "^3.4.13",
"typescript": "^5.6.2",
"vite": "^5.4.8",
"vite-tsconfig-paths": "^5.0.1",
"wrangler": "3.99.0"
"vite-tsconfig-paths": "^5.0.1"
},
"engines": {
"node": ">=20.0.0"

View file

@ -44,10 +44,10 @@
"@tanstack/react-query-devtools": "^5.60.6",
"@types/chrome": "^0.0.287",
"@types/node": "^22.10.1",
"@types/papaparse": "^5.3.15",
"@types/postlight__mercury-parser": "^2.2.7",
"@types/showdown": "^2.0.6",
"@types/turndown": "^5.0.5",
"@upstash/ratelimit": "^2.0.5",
"ai": "4.0.18",
"autoevals": "^0.0.106",
"aws4fetch": "^1.0.20",
@ -63,6 +63,7 @@
"mammoth": "^1.8.0",
"nanoid": "^5.0.7",
"notion-client": "^7.1.5",
"papaparse": "^5.5.1",
"pdfjs-serverless": "^0.6.0",
"postgres": "^3.4.4",
"posthog-js": "^1.188.0",
@ -78,7 +79,7 @@
"shiki": "^1.22.1",
"sonner": "^1.7.0",
"web": "^0.0.2",
"wrangler": "3.99.0"
"wrangler": "latest"
},
"overrides": {
"iron-webcrypto": "^1.2.1",