From 0568c9466e493cda90685fd86ac031d65cac6028 Mon Sep 17 00:00:00 2001 From: Mahesh Sanikommmu Date: Thu, 23 Oct 2025 20:11:19 -0700 Subject: [PATCH] added document add and fetch feature --- .../web/app/onboarding/setup/chat-sidebar.tsx | 355 +++++++++++++++--- packages/lib/api.ts | 4 + packages/lib/posthog.tsx | 2 +- 3 files changed, 309 insertions(+), 52 deletions(-) diff --git a/apps/web/app/onboarding/setup/chat-sidebar.tsx b/apps/web/app/onboarding/setup/chat-sidebar.tsx index 1203eb96..21aa0863 100644 --- a/apps/web/app/onboarding/setup/chat-sidebar.tsx +++ b/apps/web/app/onboarding/setup/chat-sidebar.tsx @@ -1,11 +1,12 @@ "use client" -import { useState, useEffect } from "react" +import { useState, useEffect, useCallback, useRef } from "react" import { motion, AnimatePresence } from "motion/react" import NovaOrb from "@/components/nova/nova-orb" import { Button } from "@ui/components/button" import { PanelRightCloseIcon, SendIcon } from "lucide-react" import { collectValidUrls } from "@/utils/url-helpers" +import { $fetch } from "@lib/api" interface ChatSidebarProps { formData: { @@ -25,9 +26,11 @@ export function ChatSidebar({ formData }: ChatSidebarProps) { title?: string description: string fullContent: string + type?: "formData" | "exa" | "memory" | "waiting" }[] >([]) const [isLoading, setIsLoading] = useState(false) + const displayedMemoriesRef = useRef>(new Set()) const handleSend = () => { console.log("Message:", message) @@ -45,52 +48,277 @@ export function ChatSidebar({ formData }: ChatSidebarProps) { setIsChatOpen(!isChatOpen) } - // Helper function to truncate text to 2 lines - const truncateText = (text: string, maxLength = 120) => { - if (text.length <= maxLength) return text - return text.slice(0, maxLength).trim() + "..." - } + const pollForMemories = useCallback( + async (documentIds: string[]) => { + const maxAttempts = 30 // 30 attempts * 3 seconds = 90 seconds max + const pollInterval = 3000 // 3 seconds + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + try { + const response = await $fetch("@get/documents/:id", { + params: { id: documentIds[0] ?? "" }, + disableValidation: true, + }) + + console.log("response", response) + + if (response.data) { + const document = response.data + + if (document.memoryEntries && document.memoryEntries.length > 0) { + const newMemories: typeof messages = [] + + document.memoryEntries.forEach((memory) => { + if (!displayedMemoriesRef.current.has(memory.memory)) { + displayedMemoriesRef.current.add(memory.memory) + newMemories.push({ + url: document.url || "", + title: memory.title || document.title || "Memory", + description: memory.memory || "", + fullContent: memory.memory || "", + type: "memory" as const, + }) + } + }) + + if (newMemories.length > 0) { + setMessages((prev) => [...prev, ...newMemories]) + } + } + + if (document.status === "done") { + const waitingMessage = { + url: "", + title: "", + description: "Waiting for your input", + fullContent: "Waiting for your input", + type: "waiting" as const, + } + + setMessages((prev) => { + const withoutWaiting = prev.filter( + (msg) => msg.type !== "waiting", + ) + return [...withoutWaiting, waitingMessage] + }) + break + } + } + + await new Promise((resolve) => setTimeout(resolve, pollInterval)) + } catch (error) { + console.warn("Error polling for memories:", error) + await new Promise((resolve) => setTimeout(resolve, pollInterval)) + } + } + + const waitingMessage = { + url: "", + title: "", + description: "Waiting for your input", + fullContent: "Waiting for your input", + type: "waiting" as const, + } + + setMessages((prev) => { + const withoutWaiting = prev.filter((msg) => msg.type !== "waiting") + return [...withoutWaiting, waitingMessage] + }) + }, + [], + ) - // Fetch content when formData is available useEffect(() => { if (!formData) return - const fetchContent = async () => { - setIsLoading(true) - const urls = collectValidUrls(formData.linkedin, formData.otherLinks) + const urls = collectValidUrls(formData.linkedin, formData.otherLinks) + + console.log("urls", urls) + + if (urls.length > 0) { + const processContent = async () => { + setIsLoading(true) - if (urls.length > 0) { try { + // Step 1: Fetch content from Exa const response = await fetch("/api/exa/fetch-content", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ urls }), }) const { results } = await response.json() + console.log("results", results) - const newMessages = results.map( - (result: { - url: string - title?: string - text?: string - description?: string - }) => ({ - url: result.url, - title: result.title, - description: result.text || result.description || "", - fullContent: result.text || result.description || "", - }), - ) - setMessages(newMessages) + // Step 2: Create documents from Exa results + const documentIds: string[] = [] + for (const result of results) { + try { + const docResponse = await $fetch("@post/documents", { + body: { + content: result.text || result.description || "", + containerTags: ["sm_project_default"], + metadata: { + sm_source: "consumer", + exa_url: result.url, + exa_title: result.title, + }, + }, + }) + + if (docResponse.data?.id) { + documentIds.push(docResponse.data.id) + } + } catch (error) { + console.warn("Error creating document:", error) + } + } + + // Step 2.5: Create document from description if it exists + if (formData.description?.trim()) { + try { + const descDocResponse = await $fetch("@post/documents", { + body: { + content: formData.description, + containerTags: ["sm_project_default"], + metadata: { + sm_source: "consumer", + description_source: "user_input", + }, + }, + }) + + if (descDocResponse.data?.id) { + documentIds.push(descDocResponse.data.id) + } + } catch (error) { + console.warn("Error creating description document:", error) + } + } + + // Step 3: Poll for memories + if (documentIds.length > 0) { + await pollForMemories(documentIds) + } else { + // No documents created, show waiting + const waitingMessage = { + url: "", + title: "", + description: "Waiting for your input", + fullContent: "Waiting for your input", + type: "waiting" as const, + } + setMessages([waitingMessage]) + } } catch (error) { - console.warn("Error fetching content:", error) - } - } - setIsLoading(false) - } + console.warn("Error processing content:", error) - fetchContent() - }, [formData]) + const waitingMessage = { + url: "", + title: "", + description: "Waiting for your input", + fullContent: "Waiting for your input", + type: "waiting" as const, + } + + setMessages([waitingMessage]) + } + setIsLoading(false) + } + + processContent() + } else { + console.log("description", formData.description) + if (formData.description?.trim()) { + const processDescription = async () => { + setIsLoading(true) + try { + const descDocResponse = await $fetch("@post/documents", { + body: { + content: formData.description, + containerTags: ["sm_project_default"], + metadata: { + sm_source: "consumer", + description_source: "user_input", + }, + }, + }) + + if (descDocResponse.data?.id) { + console.log("descDocResponse.data.id", descDocResponse.data.id) + await pollForMemories([descDocResponse.data.id]) + } else { + const waitingMessage = { + url: "", + title: "", + description: "Waiting for your input", + fullContent: "Waiting for your input", + type: "waiting" as const, + } + setMessages([waitingMessage]) + } + } catch (error) { + console.warn("Error processing description:", error) + + const waitingMessage = { + url: "", + title: "", + description: "Waiting for your input", + fullContent: "Waiting for your input", + type: "waiting" as const, + } + setMessages([waitingMessage]) + } + setIsLoading(false) + } + + processDescription() + } else { + const formDataMessages = [] + + if (formData.twitter) { + formDataMessages.push({ + url: formData.twitter, + title: "Twitter Profile", + description: `Twitter: ${formData.twitter}`, + fullContent: formData.twitter, + type: "formData" as const, + }) + } + + if (formData.linkedin) { + formDataMessages.push({ + url: formData.linkedin, + title: "LinkedIn Profile", + description: `LinkedIn: ${formData.linkedin}`, + fullContent: formData.linkedin, + type: "formData" as const, + }) + } + + if (formData.otherLinks.length > 0) { + formData.otherLinks.forEach((link) => { + formDataMessages.push({ + url: link, + title: "Other Link", + description: `Link: ${link}`, + fullContent: link, + type: "formData" as const, + }) + }) + } + + const waitingMessage = { + url: "", + title: "", + description: "Waiting for your input", + fullContent: "Waiting for your input", + type: "waiting" as const, + } + + setMessages([...formDataMessages, waitingMessage]) + } + } + }, [formData, pollForMemories]) return ( @@ -130,10 +358,10 @@ export function ChatSidebar({ formData }: ChatSidebarProps) { Close chat -
- {messages.length === 0 && !isLoading && ( +
+ {messages.length === 0 && !isLoading && !formData && (
- + Waiting for your input
)} @@ -145,24 +373,49 @@ export function ChatSidebar({ formData }: ChatSidebarProps) { {messages.map((msg, i) => (
- {msg.title && ( -

- {msg.title} -

+ {msg.type === "waiting" ? ( +
+ + {msg.description} +
+ ) : ( + <> +
+
+ {msg.title && ( +

+ {msg.title} +

+ )} + {msg.url && ( + + {msg.url} + + )} + {msg.description && msg.type === "memory" && ( +

+ {msg.description} +

+ )} +
+ )} - - {msg.url} - -

- {truncateText(msg.description)} -

))}
diff --git a/packages/lib/api.ts b/packages/lib/api.ts index bfdc3ac0..657067a1 100644 --- a/packages/lib/api.ts +++ b/packages/lib/api.ts @@ -159,6 +159,10 @@ export const apiSchema = createSchema({ output: MigrateMCPResponseSchema, }, + "@get/documents/:id": { + output: z.any(), + }, + // Delete a memory "@delete/documents/:id": { output: z.any(), // 204 No-Content diff --git a/packages/lib/posthog.tsx b/packages/lib/posthog.tsx index 0c436e1e..438cef64 100644 --- a/packages/lib/posthog.tsx +++ b/packages/lib/posthog.tsx @@ -73,7 +73,7 @@ export function PostHogProvider({ children }: { children: React.ReactNode }) { return ( <> - + {process.env.NODE_ENV === "production" && } {children}