From 3b26fc393ee19ac03f3e90237d3596fb5ac4a3cf Mon Sep 17 00:00:00 2001 From: abhigyanpatwari Date: Tue, 6 Jan 2026 12:32:09 +0530 Subject: [PATCH] grep working, embedings autostart also working --- src/App.tsx | 28 +++- src/components/Header.tsx | 15 +- src/components/RightPanel.tsx | 27 ++-- src/components/ToolCallCard.tsx | 169 ++++++++++++++++++++++ src/core/llm/agent.ts | 140 ++++++++++++------- src/core/llm/tools.ts | 240 +++++++++++++++++++++++++++++--- src/core/llm/types.ts | 23 ++- src/hooks/useAppState.tsx | 136 +++++++++++++++--- src/workers/ingestion.worker.ts | 18 ++- 9 files changed, 686 insertions(+), 110 deletions(-) create mode 100644 src/components/ToolCallCard.tsx diff --git a/src/App.tsx b/src/App.tsx index b88a0fec0..61cb2f352 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -26,6 +26,8 @@ const AppContent = () => { setSettingsPanelOpen, refreshLLMSettings, initializeAgent, + startEmbeddings, + embeddingStatus, } = useAppState(); const graphCanvasRef = useRef(null); @@ -43,6 +45,17 @@ const AppContent = () => { setGraph(result.graph); setFileContents(result.fileContents); setViewMode('exploring'); + + // Auto-start embeddings pipeline in background + // Uses WebGPU if available, falls back to WASM + startEmbeddings().catch((err) => { + // WebGPU not available - try WASM fallback silently + if (err?.name === 'WebGPUNotAvailableError' || err?.message?.includes('WebGPU')) { + startEmbeddings('wasm').catch(console.warn); + } else { + console.warn('Embeddings auto-start failed:', err); + } + }); } catch (error) { console.error('Pipeline error:', error); setProgress({ @@ -56,7 +69,7 @@ const AppContent = () => { setProgress(null); }, 3000); } - }, [setViewMode, setGraph, setFileContents, setProgress, setProjectName, runPipeline]); + }, [setViewMode, setGraph, setFileContents, setProgress, setProjectName, runPipeline, startEmbeddings]); const handleGitClone = useCallback(async (files: FileEntry[]) => { // Extract project name from first file path (e.g., "owner-repo-123/src/..." -> "owner-repo") @@ -74,6 +87,17 @@ const AppContent = () => { setGraph(result.graph); setFileContents(result.fileContents); setViewMode('exploring'); + + // Auto-start embeddings pipeline in background + // Uses WebGPU if available, falls back to WASM + startEmbeddings().catch((err) => { + // WebGPU not available - try WASM fallback silently + if (err?.name === 'WebGPUNotAvailableError' || err?.message?.includes('WebGPU')) { + startEmbeddings('wasm').catch(console.warn); + } else { + console.warn('Embeddings auto-start failed:', err); + } + }); } catch (error) { console.error('Pipeline error:', error); setProgress({ @@ -87,7 +111,7 @@ const AppContent = () => { setProgress(null); }, 3000); } - }, [setViewMode, setGraph, setFileContents, setProgress, setProjectName, runPipelineFromFiles]); + }, [setViewMode, setGraph, setFileContents, setProgress, setProjectName, runPipelineFromFiles, startEmbeddings]); const handleFocusNode = useCallback((nodeId: string) => { graphCanvasRef.current?.focusNode(nodeId); diff --git a/src/components/Header.tsx b/src/components/Header.tsx index 89ab6267f..12ab765c3 100644 --- a/src/components/Header.tsx +++ b/src/components/Header.tsx @@ -1,4 +1,4 @@ -import { Search, Settings, HelpCircle, Sparkles } from 'lucide-react'; +import { Search, Settings, HelpCircle, Sparkles, Github, Star } from 'lucide-react'; import { useAppState } from '../hooks/useAppState'; import { useState, useMemo, useRef, useEffect } from 'react'; import { GraphNode } from '../core/graph/types'; @@ -183,6 +183,19 @@ export const Header = ({ onFocusNode }: HeaderProps) => { {/* Right section */}
+ {/* GitHub Star Button */} + + + Star if cool + + ✨ + + {/* Stats */} {graph && (
diff --git a/src/components/RightPanel.tsx b/src/components/RightPanel.tsx index 4c0181319..e516fd916 100644 --- a/src/components/RightPanel.tsx +++ b/src/components/RightPanel.tsx @@ -8,6 +8,7 @@ import { vscDarkPlus } from 'react-syntax-highlighter/dist/esm/styles/prism'; import ReactMarkdown from 'react-markdown'; import { useAppState } from '../hooks/useAppState'; import { NODE_COLORS } from '../lib/constants'; +import { ToolCallCard } from './ToolCallCard'; // Custom syntax theme const customTheme = { @@ -307,18 +308,15 @@ export const RightPanel = () => {
)} - {/* Tool call indicator */} + {/* Active tool calls - shown at top during execution */} {currentToolCalls.length > 0 && ( -
+
+
+ + Working... +
{currentToolCalls.map(tc => ( - - - {tc.name} - ({tc.status}) - + ))}
)} @@ -398,14 +396,11 @@ export const RightPanel = () => { ) : ( message.content )} + {/* Tool calls shown as expandable cards */} {message.toolCalls && message.toolCalls.length > 0 && ( -
+
{message.toolCalls.map(tc => ( -
- - {tc.name} - {tc.result && → {tc.result}} -
+ ))}
)} diff --git a/src/components/ToolCallCard.tsx b/src/components/ToolCallCard.tsx new file mode 100644 index 000000000..ca98ec1bc --- /dev/null +++ b/src/components/ToolCallCard.tsx @@ -0,0 +1,169 @@ +/** + * ToolCallCard Component + * + * Displays a tool call with expand/collapse functionality. + * Shows the tool name, status, and when expanded, the query/args and result. + */ + +import { useState } from 'react'; +import { ChevronDown, ChevronRight, Sparkles, Check, Loader2, AlertCircle } from 'lucide-react'; +import type { ToolCallInfo } from '../core/llm/types'; + +interface ToolCallCardProps { + toolCall: ToolCallInfo; + /** Start expanded (useful for in-progress calls) */ + defaultExpanded?: boolean; +} + +/** + * Format tool arguments for display + */ +const formatArgs = (args: Record): string => { + if (!args || Object.keys(args).length === 0) { + return ''; + } + + // Special handling for Cypher queries + if ('query' in args && typeof args.query === 'string') { + return args.query; + } + if ('cypher' in args && typeof args.cypher === 'string') { + // For execute_vector_cypher, show both the natural language query and cypher + let result = ''; + if ('query' in args) { + result += `Search: "${args.query}"\n\n`; + } + result += args.cypher; + return result; + } + + // For other tools, show as formatted JSON + return JSON.stringify(args, null, 2); +}; + +/** + * Get status icon and color + */ +const getStatusDisplay = (status: ToolCallInfo['status']) => { + switch (status) { + case 'running': + return { + icon: , + color: 'text-amber-400', + bgColor: 'bg-amber-500/10', + borderColor: 'border-amber-500/30', + }; + case 'completed': + return { + icon: , + color: 'text-emerald-400', + bgColor: 'bg-emerald-500/10', + borderColor: 'border-emerald-500/30', + }; + case 'error': + return { + icon: , + color: 'text-rose-400', + bgColor: 'bg-rose-500/10', + borderColor: 'border-rose-500/30', + }; + default: + return { + icon: , + color: 'text-text-muted', + bgColor: 'bg-surface', + borderColor: 'border-border-subtle', + }; + } +}; + +/** + * Get a friendly display name for the tool + */ +const getToolDisplayName = (name: string): string => { + const names: Record = { + 'execute_cypher': '🔍 Cypher Query', + 'execute_vector_cypher': '🧠 Semantic + Graph Query', + 'semantic_search': '🔎 Semantic Search', + 'semantic_search_with_context': '🔎 Semantic Search + Context', + 'get_code_content': '📄 Read Code', + 'get_codebase_stats': '📊 Get Stats', + 'get_graph_schema': '📋 Get Schema', + }; + return names[name] || name; +}; + +export const ToolCallCard = ({ toolCall, defaultExpanded = false }: ToolCallCardProps) => { + const [isExpanded, setIsExpanded] = useState(defaultExpanded); + const status = getStatusDisplay(toolCall.status); + const formattedArgs = formatArgs(toolCall.args); + + return ( +
+ {/* Header - always visible */} + + + {/* Expanded content */} + {isExpanded && ( +
+ {/* Arguments/Query */} + {formattedArgs && ( +
+
+ {toolCall.name.includes('cypher') ? 'Query' : 'Input'} +
+ 
+                {formattedArgs}
+
+
+ )} + + {/* Result */} + {toolCall.result && ( +
+
+ Result +
+ 
+                {toolCall.result.length > 2000 
+                  ? toolCall.result.slice(0, 2000) + '\n\n... (truncated)'
+                  : toolCall.result
+                }
+
+
+ )} + + {/* Loading state for in-progress */} + {toolCall.status === 'running' && !toolCall.result && ( +
+ + Executing... +
+ )} +
+ )} +
+ ); +}; + +export default ToolCallCard; + diff --git a/src/core/llm/agent.ts b/src/core/llm/agent.ts index 78033d205..73ca5ec70 100644 --- a/src/core/llm/agent.ts +++ b/src/core/llm/agent.ts @@ -27,6 +27,8 @@ CAPABILITIES: - Execute Cypher queries to explore code structure (functions, classes, files, imports, call graphs) - Perform semantic search to find code by meaning (when embeddings are available) - Combine semantic search + graph traversal in a SINGLE Cypher query via the vector index (when embeddings are available) +- Search for exact text patterns using grep (regex) across all files +- Read full file contents directly - Trace dependencies and relationships between code elements - Explain code architecture and patterns @@ -34,21 +36,33 @@ APPROACH: 1. Start by understanding what the user wants to know 2. Choose the right tool(s) for the task: - Use 'get_codebase_stats' first if you need an overview - - Use 'semantic_search' for simple concept-based lookup (find relevant nodes) - - Use 'semantic_search_with_context' for simple semantic + neighborhood expansion (prebuilt 1-3 hop expansion) - - Use 'execute_vector_cypher' when you need semantic search + CUSTOM traversal/filters/returns in ONE query - - Your Cypher MUST include the placeholder {{QUERY_VECTOR}} where a FLOAT[384] vector belongs - - The vector index is on CodeEmbedding (code_embedding_idx). You must JOIN back to CodeNode via emb.nodeId - - Use 'execute_cypher' for pure structural queries (no vector search) - - Use 'get_code_content' to show actual source code + - Use 'grep_code' to find EXACT text patterns (strings, error messages, TODOs, variable names) + - Use 'read_file' to see the FULL content of any file + - Use 'semantic_search' for CONCEPT-based lookup (find code by meaning, not exact text) + - Use 'semantic_search_with_context' for semantic + neighborhood expansion + - Use 'execute_vector_cypher' when you need semantic search + CUSTOM traversal in ONE query + - Use 'execute_cypher' for pure structural queries (relationships, call graphs) + - Use 'get_code_content' to show source code for a specific node ID 3. Interpret results and explain them clearly 4. Suggest follow-up explorations when relevant +TOOL SELECTION GUIDE: +| Task | Best Tool | +|------|-----------| +| Find exact string "API_KEY" | grep_code | +| Find all TODO comments | grep_code | +| Find code related to "authentication" | semantic_search | +| What functions call X? | execute_cypher | +| Show me file utils.ts | read_file | +| Show code for a found node | get_code_content | +| Find auth code AND its callers | semantic_search_with_context | + IMPORTANT NOTES ABOUT THE DATABASE: - Nodes are stored in CodeNode(id, label, name, filePath, startLine, endLine, content) - Edges are stored in CodeRelation(FROM CodeNode TO CodeNode, type) where type ∈ {CALLS, IMPORTS, CONTAINS, DEFINES} - Embeddings are stored separately in CodeEmbedding(nodeId, embedding) for memory efficiency - Vector index: code_embedding_idx on CodeEmbedding.embedding (cosine distance; smaller distance = more similar) +- Full file contents are available via grep_code and read_file (not truncated) UNIFIED VECTOR + GRAPH QUERY PATTERN (ONE QUERY): 1) Vector search to get closest embeddings @@ -72,8 +86,7 @@ STYLE: - If a query fails, explain why and suggest alternatives LIMITATIONS: -- You can only see code that's been indexed in the knowledge graph -- Semantic search requires embeddings to be generated first +- Semantic search requires embeddings to be generated first (but grep_code always works) - Large codebases may require more specific queries When showing code or query results, format them nicely using markdown.`; @@ -140,14 +153,16 @@ export const createGraphRAGAgent = ( executeQuery: (cypher: string) => Promise, semanticSearch: (query: string, k?: number, maxDistance?: number) => Promise, semanticSearchWithContext: (query: string, k?: number, hops?: number) => Promise, - isEmbeddingReady: () => boolean + isEmbeddingReady: () => boolean, + fileContents: Map ) => { const model = createChatModel(config); const tools = createGraphRAGTools( executeQuery, semanticSearch, semanticSearchWithContext, - isEmbeddingReady + isEmbeddingReady, + fileContents ); const agent = createReactAgent({ @@ -169,7 +184,13 @@ export interface AgentMessage { /** * Stream a response from the agent - * Yields chunks for real-time UI updates + * Uses streamMode: "values" to get step-by-step updates including reasoning + * + * Each step shows: + * - AI reasoning/thinking (content before tool calls) + * - Tool calls with arguments + * - Tool results + * - Final answer */ export async function* streamAgentResponse( agent: ReturnType, @@ -181,53 +202,70 @@ export async function* streamAgentResponse( content: m.content, })); - // In LangGraph, we stream events to get granular updates - const stream = await agent.streamEvents( + // Use stream with "values" mode to get each step as a complete state + // This lets us see reasoning, tool calls, and results separately + const stream = await agent.stream( { messages: formattedMessages }, - { version: 'v2' } + { streamMode: 'values' } ); - for await (const event of stream) { - const { event: eventType, data } = event; - const dataAny = data as any; - - // Handle tool calls start - if (eventType === 'on_tool_start') { - yield { - type: 'tool_call', - toolCall: { - id: dataAny?.tool_call_id || dataAny?.toolCallId || Date.now().toString(), // fallback if ID missing - name: (event as any).name, - args: dataAny?.input ?? {}, - status: 'running', - }, - }; - } + let lastMessageCount = formattedMessages.length; + + for await (const step of stream) { + const stepMessages = step.messages || []; - // Handle tool output - if (eventType === 'on_tool_end') { - yield { - type: 'tool_result', - toolCall: { - id: dataAny?.tool_call_id || dataAny?.toolCallId || '', // we might need to match by name if ID missing - name: (event as any).name, - args: {}, - result: typeof dataAny?.output === 'string' ? dataAny.output : JSON.stringify(dataAny?.output), - status: 'completed', - }, - }; - } - - // Handle streamed LLM content - if (eventType === 'on_chat_model_stream') { - const content = dataAny?.chunk?.content; - if (content && typeof content === 'string') { + // Process only new messages since last step + for (let i = lastMessageCount; i < stepMessages.length; i++) { + const msg = stepMessages[i]; + const msgType = msg._getType?.() || msg.type || 'unknown'; + + // AI message with content (reasoning or final answer) + if (msgType === 'ai' || msgType === 'AIMessage') { + const content = msg.content; + const toolCalls = msg.tool_calls || msg.additional_kwargs?.tool_calls || []; + + // If has content, yield it (reasoning or answer) + if (content && typeof content === 'string' && content.trim()) { + yield { + type: toolCalls.length > 0 ? 'reasoning' : 'content', + reasoning: toolCalls.length > 0 ? content : undefined, + content: toolCalls.length === 0 ? content : undefined, + }; + } + + // If has tool calls, yield each one + for (const tc of toolCalls) { + yield { + type: 'tool_call', + toolCall: { + id: tc.id || `tool-${Date.now()}-${Math.random().toString(36).slice(2)}`, + name: tc.name || tc.function?.name || 'unknown', + args: tc.args || (tc.function?.arguments ? JSON.parse(tc.function.arguments) : {}), + status: 'running', + }, + }; + } + } + + // Tool message (result from a tool) + if (msgType === 'tool' || msgType === 'ToolMessage') { + const toolCallId = msg.tool_call_id || msg.additional_kwargs?.tool_call_id || ''; + const result = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content); + yield { - type: 'content', - content: content, + type: 'tool_result', + toolCall: { + id: toolCallId, + name: msg.name || 'tool', + args: {}, + result: result, + status: 'completed', + }, }; } } + + lastMessageCount = stepMessages.length; } yield { type: 'done' }; diff --git a/src/core/llm/tools.ts b/src/core/llm/tools.ts index 988b0331d..c94fc76ce 100644 --- a/src/core/llm/tools.ts +++ b/src/core/llm/tools.ts @@ -18,7 +18,8 @@ export const createGraphRAGTools = ( executeQuery: (cypher: string) => Promise, semanticSearch: (query: string, k?: number, maxDistance?: number) => Promise, semanticSearchWithContext: (query: string, k?: number, hops?: number) => Promise, - isEmbeddingReady: () => boolean + isEmbeddingReady: () => boolean, + fileContents: Map ) => { /** * Tool: Execute Cypher Query @@ -165,10 +166,10 @@ export const createGraphRAGTools = ( const formatted = results.map((r, i) => { const location = r.startLine ? ` (lines ${r.startLine}-${r.endLine})` : ''; - return `[${i + 1}] ${r.label}: ${r.name}\n File: ${r.filePath}${location}\n Relevance: ${(1 - r.distance).toFixed(2)}`; + return `[${i + 1}] ${r.label}: ${r.name}\n ID: ${r.nodeId}\n File: ${r.filePath}${location}\n Relevance: ${(1 - r.distance).toFixed(2)}`; }); - return `Found ${results.length} semantically similar code elements:\n\n${formatted.join('\n\n')}`; + return `Found ${results.length} semantically similar code elements (use ID with get_code_content to see source):\n\n${formatted.join('\n\n')}`; } catch (error) { const message = error instanceof Error ? error.message : String(error); return `Semantic search error: ${message}`; @@ -179,7 +180,7 @@ export const createGraphRAGTools = ( description: 'Search for code by meaning using semantic similarity. Good for finding code related to a concept even if exact terms are not used.', schema: z.object({ query: z.string().describe('Natural language description of what you are looking for'), - limit: z.number().optional().describe('Maximum number of results to return (default: 10)'), + limit: z.number().optional().nullable().describe('Maximum number of results to return (default: 10)'), }), } ); @@ -204,6 +205,7 @@ export const createGraphRAGTools = ( // Results are flattened: one row per (match → connected) pair // Group by match for cleaner output const grouped = new Map 15 ? `\n ... and ${g.connections.length - 15} more` : ''; - return `[${i + 1}] ${g.matchLabel}: ${g.matchName}\n File: ${g.matchPath}\n Relevance: ${(1 - g.distance).toFixed(2)}\n Connections:\n ${connectionsList}${more}`; + return `[${i + 1}] ${g.matchLabel}: ${g.matchName}\n ID: ${g.matchId}\n File: ${g.matchPath}\n Relevance: ${(1 - g.distance).toFixed(2)}\n Connections:\n ${connectionsList}${more}`; }); - return `Found ${grouped.size} code elements with ${results.length} total connections:\n\n${formatted.join('\n\n')}`; + return `Found ${grouped.size} code elements with ${results.length} total connections (use ID with get_code_content to see source):\n\n${formatted.join('\n\n')}`; } catch (error) { const message = error instanceof Error ? error.message : String(error); return `Search with context error: ${message}`; @@ -260,7 +263,7 @@ export const createGraphRAGTools = ( description: 'Search for code semantically AND show directly connected code elements with relationship types (CALLS, IMPORTS, DEFINES, CONTAINS). Shows what each match is connected to and how.', schema: z.object({ query: z.string().describe('Natural language description of what you are looking for'), - limit: z.number().optional().describe('Number of semantic matches to find (default: 5)'), + limit: z.number().optional().nullable().describe('Number of semantic matches to find (default: 5)'), }), } ); @@ -277,7 +280,7 @@ export const createGraphRAGTools = ( name: 'get_graph_schema', description: 'Get the graph database schema including node types, relationships, and Cypher query patterns. Call this before writing Cypher queries.', schema: z.object({ - includeExamples: z.boolean().optional().describe('Whether to include query examples (default: true)'), + includeExamples: z.boolean().optional().nullable().describe('Whether to include query examples (default: true)'), }), } ); @@ -285,14 +288,16 @@ export const createGraphRAGTools = ( /** * Tool: Get Code Content * Retrieve the source code for a specific node + * Uses fileContents Map for full content (not truncated DB content) */ const getCodeContentTool = tool( async ({ nodeId }: { nodeId: string }) => { try { + // Query graph for node metadata (fast, small data) const results = await executeQuery( `MATCH (n:CodeNode {id: '${nodeId.replace(/'/g, "''")}'}) RETURN n.name AS name, n.label AS label, n.filePath AS filePath, - n.content AS content, n.startLine AS startLine, n.endLine AS endLine` + n.startLine AS startLine, n.endLine AS endLine` ); if (results.length === 0) { @@ -303,15 +308,36 @@ export const createGraphRAGTools = ( const name = node.name ?? node[0]; const label = node.label ?? node[1]; const filePath = node.filePath ?? node[2]; - const content = node.content ?? node[3]; - const startLine = node.startLine ?? node[4]; - const endLine = node.endLine ?? node[5]; + const startLine = node.startLine ?? node[3]; + const endLine = node.endLine ?? node[4]; - if (!content) { - return `${label} "${name}" in ${filePath} (no source code available)`; + // Get FULL content from fileContents Map (not truncated DB) + const fileContent = fileContents.get(filePath); + + if (!fileContent) { + return `${label}: ${name}\nFile: ${filePath}\n(File content not available in memory)`; } - return `${label}: ${name}\nFile: ${filePath}\nLines: ${startLine}-${endLine}\n\n\`\`\`\n${content}\n\`\`\``; + // For File nodes, return full content (limited for very large files) + if (label === 'File' || label === 'Folder') { + const MAX_FILE_CONTENT = 30000; + if (fileContent.length > MAX_FILE_CONTENT) { + return `${label}: ${name}\nFile: ${filePath}\nTotal size: ${fileContent.length} characters\n\n\`\`\`\n${fileContent.slice(0, MAX_FILE_CONTENT)}\n\`\`\`\n\n... [truncated, use read_file for full content]`; + } + return `${label}: ${name}\nFile: ${filePath}\n\n\`\`\`\n${fileContent}\n\`\`\``; + } + + // For Function/Class/Method nodes, extract specific lines with context + const lines = fileContent.split('\n'); + const contextBefore = 3; + const contextAfter = 20; // Show more after to capture full function body + + const start = Math.max(0, (startLine ?? 0) - contextBefore); + const end = Math.min(lines.length - 1, (endLine ?? startLine ?? 0) + contextAfter); + + const snippet = lines.slice(start, end + 1).join('\n'); + + return `${label}: ${name}\nFile: ${filePath}\nLines: ${startLine + 1}-${endLine + 1}\n\n\`\`\`\n${snippet}\n\`\`\``; } catch (error) { const message = error instanceof Error ? error.message : String(error); return `Error retrieving code: ${message}`; @@ -361,7 +387,7 @@ export const createGraphRAGTools = ( ? 'Ready (semantic search available)' : 'Not generated (use execute_cypher for queries)'; - return `Codebase Statistics:\n\nNodes by type:\n${nodeStats}\n\nRelationships by type:\n${relStats}\n\nEmbeddings: ${embeddingStatus}`; + return `Codebase Statistics:\n\nNodes by type:\n${nodeStats}\n\nRelationships by type:\n${relStats}\n\nEmbeddings: ${embeddingStatus}\n\nFiles in memory: ${fileContents.size}`; } catch (error) { const message = error instanceof Error ? error.message : String(error); return `Error getting stats: ${message}`; @@ -371,7 +397,185 @@ export const createGraphRAGTools = ( name: 'get_codebase_stats', description: 'Get an overview of the codebase including counts of different element types (files, functions, classes) and relationship types.', schema: z.object({ - verbose: z.boolean().optional().describe('Include detailed breakdown (default: false)'), + verbose: z.boolean().optional().nullable().describe('Include detailed breakdown (default: false)'), + }), + } + ); + + /** + * Tool: Grep Code + * Search for patterns across all file contents using regex + */ + const grepCodeTool = tool( + async ({ pattern, filePattern, caseSensitive, maxResults }: { + pattern: string; + filePattern?: string; + caseSensitive?: boolean; + maxResults?: number; + }) => { + try { + const flags = caseSensitive ? 'g' : 'gi'; + let regex: RegExp; + try { + regex = new RegExp(pattern, flags); + } catch (e) { + return `Invalid regex pattern: ${pattern}. Error: ${e instanceof Error ? e.message : String(e)}`; + } + + const results: Array<{ + file: string; + line: number; + content: string; + }> = []; + + const limit = maxResults ?? 100; + + for (const [filePath, content] of fileContents.entries()) { + // Optional file pattern filter + if (filePattern && !filePath.toLowerCase().includes(filePattern.toLowerCase())) { + continue; + } + + const lines = content.split('\n'); + for (let i = 0; i < lines.length; i++) { + if (regex.test(lines[i])) { + results.push({ + file: filePath, + line: i + 1, + content: lines[i].trim().slice(0, 200), // Limit line length + }); + + if (results.length >= limit) break; + } + // Reset regex lastIndex for global flag + regex.lastIndex = 0; + } + + if (results.length >= limit) break; + } + + if (results.length === 0) { + return `No matches found for pattern: "${pattern}"${filePattern ? ` in files matching "${filePattern}"` : ''}`; + } + + const formatted = results.map(r => + `${r.file}:${r.line}: ${r.content}` + ).join('\n'); + + const truncatedMsg = results.length >= limit + ? `\n\n(Showing first ${limit} results. Use filePattern to narrow search.)` + : ''; + + return `Found ${results.length} matches for "${pattern}":\n\n${formatted}${truncatedMsg}`; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return `Grep error: ${message}`; + } + }, + { + name: 'grep_code', + description: 'Search for text patterns across all files in the codebase using regex. Use this to find exact strings, error messages, TODOs, specific variable names, or any text pattern. Returns file paths and line numbers of matches.', + schema: z.object({ + pattern: z.string().describe('Regex pattern to search for (e.g., "TODO", "console\\.log", "API_KEY")'), + filePattern: z.string().optional().nullable().describe('Optional filter - only search files whose path contains this string (e.g., ".ts", "src/api")'), + caseSensitive: z.boolean().optional().nullable().describe('Whether search is case-sensitive (default: false)'), + maxResults: z.number().optional().nullable().describe('Maximum number of results to return (default: 100)'), + }), + } + ); + + /** + * Tool: Read File + * Read the full content of a file by its path + */ + const readFileTool = tool( + async ({ filePath }: { filePath: string }) => { + // Normalize the requested path (handle Windows-style paths) + const normalizedRequest = filePath.replace(/\\/g, '/').toLowerCase(); + + // Try exact match first + let content = fileContents.get(filePath); + let actualPath = filePath; + + // If not found, try smarter matching + if (!content) { + // Score each file path by how well it matches the request + const candidates: Array<{ path: string; score: number }> = []; + + for (const [path] of fileContents.entries()) { + const normalizedPath = path.toLowerCase(); + + // Exact match (case-insensitive) + if (normalizedPath === normalizedRequest) { + candidates.push({ path, score: 1000 }); + continue; + } + + // Ends with the requested path (e.g., "README.md" matches "src/agent_service/README.md") + if (normalizedPath.endsWith(normalizedRequest)) { + // Score higher for shorter paths (more specific match) + candidates.push({ path, score: 100 + (200 - path.length) }); + continue; + } + + // Path contains all segments of the request in order + const requestSegments = normalizedRequest.split('/').filter(Boolean); + const pathSegments = normalizedPath.split('/'); + let matchScore = 0; + let lastMatchIdx = -1; + + for (const seg of requestSegments) { + const idx = pathSegments.findIndex((s, i) => i > lastMatchIdx && s.includes(seg)); + if (idx > lastMatchIdx) { + matchScore += 10; + lastMatchIdx = idx; + } + } + + // Only include if we matched more than half the segments + if (matchScore >= requestSegments.length * 5) { + candidates.push({ path, score: matchScore }); + } + } + + // Sort by score descending, pick best match + candidates.sort((a, b) => b.score - a.score); + + if (candidates.length > 0) { + actualPath = candidates[0].path; + content = fileContents.get(actualPath); + } + } + + if (!content) { + // List similar files to help the user + const fileName = filePath.split('/').pop()?.toLowerCase() || ''; + const similarFiles = Array.from(fileContents.keys()) + .filter(p => p.toLowerCase().includes(fileName)) + .slice(0, 5); + + if (similarFiles.length > 0) { + return `File not found: "${filePath}"\n\nDid you mean one of these?\n${similarFiles.map(f => ` - ${f}`).join('\n')}`; + } + return `File not found: "${filePath}"`; + } + + // For very large files, truncate with a warning + const MAX_CONTENT = 50000; // ~50KB + if (content.length > MAX_CONTENT) { + const truncated = content.slice(0, MAX_CONTENT); + const lines = content.split('\n').length; + return `File: ${actualPath}\nTotal lines: ${lines}\n\n(Showing first ${MAX_CONTENT} characters, file is ${content.length} characters total)\n\n${truncated}\n\n... [truncated]`; + } + + const lines = content.split('\n').length; + return `File: ${actualPath}\nLines: ${lines}\n\n${content}`; + }, + { + name: 'read_file', + description: 'Read the full content of a file by its path. Use this to see the complete source code of any file in the codebase.', + schema: z.object({ + filePath: z.string().describe('The file path to read (can be partial path like "src/utils.ts")'), }), } ); @@ -384,5 +588,7 @@ export const createGraphRAGTools = ( getSchemaTool, getCodeContentTool, getStatsTool, + grepCodeTool, + readFileTool, ]; }; diff --git a/src/core/llm/types.ts b/src/core/llm/types.ts index b4b14f719..282cab508 100644 --- a/src/core/llm/types.ts +++ b/src/core/llm/types.ts @@ -118,14 +118,35 @@ export interface ToolCallInfo { /** * Streaming chunk from agent + * Now supports step-based streaming where each step is a distinct message */ export interface AgentStreamChunk { - type: 'content' | 'tool_call' | 'tool_result' | 'error' | 'done'; + type: 'reasoning' | 'tool_call' | 'tool_result' | 'content' | 'error' | 'done'; + /** LLM's reasoning/thinking text (shown as a step) */ + reasoning?: string; + /** Final answer content (streamed token by token) */ content?: string; + /** Tool call information */ toolCall?: ToolCallInfo; + /** Error message */ error?: string; } +/** + * A single step in the agent's execution + * Used for displaying the agent's thought process + */ +export interface AgentStep { + id: string; + type: 'reasoning' | 'tool_call' | 'answer'; + /** For reasoning steps */ + content?: string; + /** For tool_call steps */ + toolCall?: ToolCallInfo; + /** Timestamp */ + timestamp: number; +} + /** * Graph schema information for LLM context */ diff --git a/src/hooks/useAppState.tsx b/src/hooks/useAppState.tsx index 5c546488f..88c63cb6e 100644 --- a/src/hooks/useAppState.tsx +++ b/src/hooks/useAppState.tsx @@ -376,29 +376,63 @@ export const AppStateProvider = ({ children }: { children: ReactNode }) => { const assistantMessageId = `assistant-${Date.now()}`; let assistantContent = ''; const toolCallsForMessage: ToolCallInfo[] = []; + let reasoningSteps: string[] = []; // Collect reasoning steps try { const onChunk = Comlink.proxy((chunk: AgentStreamChunk) => { switch (chunk.type) { - case 'content': - if (chunk.content) { - assistantContent += chunk.content; - // Update the assistant message in place + case 'reasoning': + // LLM's thinking/reasoning before tool calls + if (chunk.reasoning) { + reasoningSteps.push(chunk.reasoning); + // Update message to show reasoning setChatMessages(prev => { const existing = prev.find(m => m.id === assistantMessageId); + // Build content with reasoning steps + const reasoningText = reasoningSteps.join('\n\n'); if (existing) { return prev.map(m => m.id === assistantMessageId - ? { ...m, content: assistantContent } + ? { ...m, content: reasoningText, toolCalls: [...toolCallsForMessage] } : m ); } else { return [...prev, { id: assistantMessageId, role: 'assistant' as const, - content: assistantContent, + content: reasoningText, timestamp: Date.now(), - toolCalls: toolCallsForMessage, + toolCalls: [...toolCallsForMessage], + }]; + } + }); + } + break; + + case 'content': + // Final answer content + if (chunk.content) { + assistantContent = chunk.content; // Replace, don't append (it's a full message) + // Update the assistant message + setChatMessages(prev => { + const existing = prev.find(m => m.id === assistantMessageId); + // Combine reasoning + final answer + const fullContent = reasoningSteps.length > 0 + ? reasoningSteps.join('\n\n') + '\n\n---\n\n' + assistantContent + : assistantContent; + if (existing) { + return prev.map(m => + m.id === assistantMessageId + ? { ...m, content: fullContent, toolCalls: [...toolCallsForMessage] } + : m + ); + } else { + return [...prev, { + id: assistantMessageId, + role: 'assistant' as const, + content: fullContent, + timestamp: Date.now(), + toolCalls: [...toolCallsForMessage], }]; } }); @@ -410,20 +444,83 @@ export const AppStateProvider = ({ children }: { children: ReactNode }) => { const tc = chunk.toolCall; toolCallsForMessage.push(tc); setCurrentToolCalls(prev => [...prev, tc]); + // Update message to include this tool call + setChatMessages(prev => { + const existing = prev.find(m => m.id === assistantMessageId); + const currentContent = reasoningSteps.length > 0 + ? reasoningSteps.join('\n\n') + : assistantContent || ''; + if (existing) { + return prev.map(m => + m.id === assistantMessageId + ? { ...m, content: currentContent, toolCalls: [...toolCallsForMessage] } + : m + ); + } else { + return [...prev, { + id: assistantMessageId, + role: 'assistant' as const, + content: currentContent, + timestamp: Date.now(), + toolCalls: [...toolCallsForMessage], + }]; + } + }); } break; case 'tool_result': if (chunk.toolCall) { const tc = chunk.toolCall; - setCurrentToolCalls(prev => - prev.map(t => t.id === tc.id ? { ...t, result: tc.result, status: 'completed' } : t) - ); - // Update the tool call in the message too - const idx = toolCallsForMessage.findIndex(t => t.id === tc.id); - if (idx >= 0) { - toolCallsForMessage[idx] = { ...toolCallsForMessage[idx], result: tc.result, status: 'completed' }; + // Update the tool call status - try ID first, then find first running tool with same name + let idx = toolCallsForMessage.findIndex(t => t.id === tc.id); + if (idx < 0) { + // ID didn't match - find the first RUNNING tool with same name (not already completed) + idx = toolCallsForMessage.findIndex(t => t.name === tc.name && t.status === 'running'); } + if (idx < 0) { + // Still no match - find any tool with same name + idx = toolCallsForMessage.findIndex(t => t.name === tc.name && !t.result); + } + if (idx >= 0) { + toolCallsForMessage[idx] = { + ...toolCallsForMessage[idx], + result: tc.result, + status: 'completed' + }; + } + // Same logic for currentToolCalls + setCurrentToolCalls(prev => { + // Find by ID first + let targetIdx = prev.findIndex(t => t.id === tc.id); + if (targetIdx < 0) { + // Find first running tool with same name + targetIdx = prev.findIndex(t => t.name === tc.name && t.status === 'running'); + } + if (targetIdx < 0) { + // Find any incomplete tool with same name + targetIdx = prev.findIndex(t => t.name === tc.name && !t.result); + } + if (targetIdx >= 0) { + return prev.map((t, i) => i === targetIdx + ? { ...t, result: tc.result, status: 'completed' } + : t + ); + } + return prev; + }); + // Update message with completed tool call + setChatMessages(prev => { + const existing = prev.find(m => m.id === assistantMessageId); + if (existing) { + return prev.map(m => + m.id === assistantMessageId + ? { ...m, toolCalls: [...toolCallsForMessage] } + : m + ); + } + return prev; + }); } break; @@ -432,22 +529,23 @@ export const AppStateProvider = ({ children }: { children: ReactNode }) => { break; case 'done': - // Finalize the assistant message with all tool calls + // Finalize the assistant message setChatMessages(prev => { const existing = prev.find(m => m.id === assistantMessageId); + const finalContent = assistantContent || (reasoningSteps.length > 0 ? reasoningSteps.join('\n\n') : ''); if (existing) { return prev.map(m => m.id === assistantMessageId - ? { ...m, content: assistantContent, toolCalls: toolCallsForMessage } + ? { ...m, content: finalContent, toolCalls: [...toolCallsForMessage] } : m ); - } else if (assistantContent) { + } else if (finalContent || toolCallsForMessage.length > 0) { return [...prev, { id: assistantMessageId, role: 'assistant' as const, - content: assistantContent, + content: finalContent, timestamp: Date.now(), - toolCalls: toolCallsForMessage, + toolCalls: [...toolCallsForMessage], }]; } return prev; diff --git a/src/workers/ingestion.worker.ts b/src/workers/ingestion.worker.ts index e6823a5c5..f70a7b2e4 100644 --- a/src/workers/ingestion.worker.ts +++ b/src/workers/ingestion.worker.ts @@ -26,6 +26,9 @@ const getKuzuAdapter = async () => { let embeddingProgress: EmbeddingProgress | null = null; let isEmbeddingComplete = false; +// File contents state - stores full file contents for grep/read tools +let storedFileContents: Map = new Map(); + // Agent state let currentAgent: ReturnType | null = null; let currentProviderConfig: ProviderConfig | null = null; @@ -50,6 +53,9 @@ const workerApi = { // Run the actual pipeline const result = await runIngestionPipeline(file, onProgress); + // Store file contents for grep/read tools (full content, not truncated) + storedFileContents = result.fileContents; + // Load graph into KuzuDB for querying (optional - gracefully degrades) try { onProgress({ @@ -67,8 +73,9 @@ const workerApi = { await kuzu.loadGraphToKuzu(result.graph, result.fileContents); if (import.meta.env.DEV) { - const stats = await kuzu.getKuzuStats(); - console.log('KuzuDB loaded:', stats); + const stats = await kuzu.getKuzuStats(); + console.log('KuzuDB loaded:', stats); + console.log('📁 Stored', storedFileContents.size, 'files for grep/read tools'); } } catch { // KuzuDB is optional - silently continue without it @@ -136,6 +143,9 @@ const workerApi = { // Run the pipeline const result = await runPipelineFromFiles(files, onProgress); + // Store file contents for grep/read tools (full content, not truncated) + storedFileContents = result.fileContents; + // Load graph into KuzuDB for querying (optional - gracefully degrades) try { onProgress({ @@ -155,6 +165,7 @@ const workerApi = { if (import.meta.env.DEV) { const stats = await kuzu.getKuzuStats(); console.log('KuzuDB loaded:', stats); + console.log('📁 Stored', storedFileContents.size, 'files for grep/read tools'); } } catch { // KuzuDB is optional - silently continue without it @@ -328,7 +339,8 @@ const workerApi = { kuzu.executeQuery, semanticSearchWrapper, semanticSearchWithContextWrapper, - () => isEmbeddingComplete + () => isEmbeddingComplete, + storedFileContents ); currentProviderConfig = config;