diff --git a/gitnexus/src/core/ingestion/languages/python.ts b/gitnexus/src/core/ingestion/languages/python.ts index 43748a9103..2b37aab1b2 100644 --- a/gitnexus/src/core/ingestion/languages/python.ts +++ b/gitnexus/src/core/ingestion/languages/python.ts @@ -10,6 +10,7 @@ * - namedBindingExtractor: present (from X import Y) */ +import type { NodeLabel } from 'gitnexus-shared'; import { SupportedLanguages } from 'gitnexus-shared'; import { createClassExtractor } from '../class-extractors/generic.js'; import { pythonClassConfig } from '../class-extractors/configs/python.js'; @@ -29,6 +30,8 @@ import { pythonVariableConfig } from '../variable-extractors/configs/python.js'; import { createCallExtractor } from '../call-extractors/generic.js'; import { pythonCallConfig } from '../call-extractors/configs/python.js'; import { createHeritageExtractor } from '../heritage-extractors/generic.js'; +import type { CaptureMap } from '../language-provider.js'; +import type { SyntaxNode } from '../utils/ast-helpers.js'; import { emitPythonScopeCaptures, pythonFunctionDefinitionLabel, @@ -72,6 +75,34 @@ const BUILT_INS: ReadonlySet = new Set([ 'abs', ]); +function pythonDescriptionExtractor( + nodeLabel: NodeLabel, + _nodeName: string, + captureMap: CaptureMap, +): string | undefined { + if (nodeLabel !== 'Function' && nodeLabel !== 'Method') return undefined; + const functionNode = captureMap['definition.function'] ?? captureMap['definition.method']; + if (functionNode === undefined) return undefined; + return extractPythonDocstring(functionNode); +} + +function extractPythonDocstring(functionNode: SyntaxNode): string | undefined { + const body = functionNode.childForFieldName('body'); + const firstStatement = body?.namedChild(0); + if (firstStatement?.type !== 'expression_statement') return undefined; + + const literal = firstStatement.namedChild(0); + if (literal?.type !== 'string') return undefined; + return normalizePythonStringLiteral(literal.text); +} + +function normalizePythonStringLiteral(text: string): string | undefined { + const match = text.match(/^[rRuUbBfF]*("""|'''|"|')([\s\S]*)\1$/); + const raw = match?.[2]?.trim(); + if (!raw) return undefined; + return raw.replace(/\s+/g, ' '); +} + export const pythonProvider = defineLanguage({ id: SupportedLanguages.Python, extensions: ['.py'], @@ -88,6 +119,7 @@ export const pythonProvider = defineLanguage({ variableExtractor: createVariableExtractor(pythonVariableConfig), classExtractor: createClassExtractor(pythonClassConfig), heritageExtractor: createHeritageExtractor(SupportedLanguages.Python), + descriptionExtractor: pythonDescriptionExtractor, builtInNames: BUILT_INS, labelOverride: pythonFunctionDefinitionLabel, diff --git a/gitnexus/src/core/ingestion/pipeline-phases/processes.ts b/gitnexus/src/core/ingestion/pipeline-phases/processes.ts index a525443d19..209906cfba 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/processes.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/processes.ts @@ -112,12 +112,15 @@ export const processesPhase: PipelinePhase = { } list.push(url); } - const toolsByFile = new Map(); + const toolsByHandlerId = new Map(); + const toolsWithoutHandlerByFile = new Map(); for (const td of toolDefs) { - let list = toolsByFile.get(td.filePath); + const key = td.handlerNodeId ?? td.filePath; + const targetMap = td.handlerNodeId ? toolsByHandlerId : toolsWithoutHandlerByFile; + let list = targetMap.get(key); if (!list) { list = []; - toolsByFile.set(td.filePath, list); + targetMap.set(key, list); } list.push(td.name); } @@ -145,7 +148,9 @@ export const processesPhase: PipelinePhase = { linked++; } } - const toolNames = toolsByFile.get(entryFile); + const exactToolNames = toolsByHandlerId.get(proc.entryPointId); + const fallbackToolNames = toolsWithoutHandlerByFile.get(entryFile); + const toolNames = exactToolNames ?? fallbackToolNames; if (toolNames) { for (const toolName of toolNames) { const toolNodeId = generateId('Tool', toolName); diff --git a/gitnexus/src/core/ingestion/pipeline-phases/tools.ts b/gitnexus/src/core/ingestion/pipeline-phases/tools.ts index 5c6baf86d2..023c8a1aff 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/tools.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/tools.ts @@ -20,6 +20,7 @@ export interface ToolDef { name: string; filePath: string; description: string; + handlerNodeId?: string; } export interface ToolsOutput { @@ -42,7 +43,14 @@ export const toolsPhase: PipelinePhase = { for (const td of allToolDefs) { if (seenToolNames.has(td.toolName)) continue; seenToolNames.add(td.toolName); - toolDefs.push({ name: td.toolName, filePath: td.filePath, description: td.description }); + const handlerNodeId = + td.handlerNodeId && ctx.graph.getNode(td.handlerNodeId) ? td.handlerNodeId : undefined; + toolDefs.push({ + name: td.toolName, + filePath: td.filePath, + description: td.description, + ...(handlerNodeId !== undefined ? { handlerNodeId } : {}), + }); } // TS tool definition arrays — require inputSchema nearby @@ -84,10 +92,10 @@ export const toolsPhase: PipelinePhase = { properties: { name: td.name, filePath: td.filePath, description: td.description }, }); - const handlerFileId = generateId('File', td.filePath); + const handlerId = td.handlerNodeId ?? generateId('File', td.filePath); ctx.graph.addRelationship({ - id: generateId('HANDLES_TOOL', `${handlerFileId}->${toolNodeId}`), - sourceId: handlerFileId, + id: generateId('HANDLES_TOOL', `${handlerId}->${toolNodeId}`), + sourceId: handlerId, targetId: toolNodeId, type: 'HANDLES_TOOL', confidence: 1.0, diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 49cc997b28..c7f50fa455 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -214,6 +214,7 @@ export interface ExtractedToolDef { toolName: string; description: string; lineNumber: number; + handlerNodeId?: string; } export interface ExtractedORMQuery { @@ -2145,8 +2146,9 @@ const processFileGroup = ( result.toolDefs.push({ filePath: file.path, toolName: nodeName, - description: dec.arg || '', + description: (dec.arg || description || '').slice(0, 200), lineNumber: definitionNode.startPosition.row + lineOffset, + handlerNodeId: nodeId, }); } fileDecorators.delete(checkLine); diff --git a/gitnexus/test/fixtures/lang-resolution/python-mcp-tools/server.py b/gitnexus/test/fixtures/lang-resolution/python-mcp-tools/server.py index f63680042a..22abf8d640 100644 --- a/gitnexus/test/fixtures/lang-resolution/python-mcp-tools/server.py +++ b/gitnexus/test/fixtures/lang-resolution/python-mcp-tools/server.py @@ -1,11 +1,28 @@ from mcp import tool +def _format_weather(city: str) -> str: + return f"Weather in {city}: sunny" + +def _lookup_weather(city: str) -> str: + return _format_weather(city) + +def _rank_docs(query: str) -> list: + return [query] + +def _lookup_docs(query: str) -> list: + return _rank_docs(query) + @mcp.tool() def get_weather(city: str) -> str: """Get weather for a city.""" - return f"Weather in {city}: sunny" + return _lookup_weather(city) @mcp.tool() def search_docs(query: str) -> list: """Search documentation.""" - return [] + return _lookup_docs(query) + +@mcp.tool("Explicit description") +def explicit_tool() -> str: + """Docstring that should not be used.""" + return "explicit" diff --git a/gitnexus/test/fixtures/local-backend-seed.ts b/gitnexus/test/fixtures/local-backend-seed.ts index 9a4d4e0c13..3f299046e2 100644 --- a/gitnexus/test/fixtures/local-backend-seed.ts +++ b/gitnexus/test/fixtures/local-backend-seed.ts @@ -8,6 +8,10 @@ export const LOCAL_BACKEND_SEED_DATA = [ `CREATE (fn:Function {id: 'func:login', name: 'login', filePath: 'src/auth.ts', startLine: 1, endLine: 15, isExported: true, content: 'function login() {}', description: 'User login'})`, `CREATE (fn:Function {id: 'func:validate', name: 'validate', filePath: 'src/auth.ts', startLine: 17, endLine: 25, isExported: true, content: 'function validate() {}', description: 'Validate input'})`, `CREATE (fn:Function {id: 'func:hash', name: 'hash', filePath: 'src/utils.ts', startLine: 1, endLine: 8, isExported: true, content: 'function hash() {}', description: 'Hash utility'})`, + `CREATE (fn:Function {id: 'func:alpha', name: 'alpha', filePath: 'src/tools.py', startLine: 1, endLine: 8, isExported: true, content: 'def alpha(): pass', description: 'Alpha tool handler'})`, + `CREATE (fn:Function {id: 'func:beta', name: 'beta', filePath: 'src/tools.py', startLine: 10, endLine: 18, isExported: true, content: 'def beta(): pass', description: 'Beta tool handler'})`, + `CREATE (t:Tool {id: 'Tool:alpha', name: 'alpha', filePath: 'src/tools.py', description: 'Calls chain A.'})`, + `CREATE (t:Tool {id: 'Tool:beta', name: 'beta', filePath: 'src/tools.py', description: 'Calls chain B.'})`, // Class `CREATE (c:Class {id: 'class:AuthService', name: 'AuthService', filePath: 'src/auth.ts', startLine: 30, endLine: 60, isExported: true, content: 'class AuthService {}', description: 'Authentication service'})`, `CREATE (c:Class {id: 'class:BaseService', name: 'BaseService', filePath: 'src/base.ts', startLine: 1, endLine: 20, isExported: true, content: 'class BaseService {}', description: 'Base service class'})`, @@ -18,6 +22,8 @@ export const LOCAL_BACKEND_SEED_DATA = [ `CREATE (c:Community {id: 'comm:auth', label: 'Auth', heuristicLabel: 'Authentication', keywords: ['auth', 'login'], description: 'Auth module', enrichedBy: 'heuristic', cohesion: 0.8, symbolCount: 3})`, // Process `CREATE (p:Process {id: 'proc:login-flow', label: 'LoginFlow', heuristicLabel: 'User Login', processType: 'intra_community', stepCount: 2, communities: ['auth'], entryPointId: 'func:login', terminalId: 'func:validate'})`, + `CREATE (p:Process {id: 'proc:alpha-flow', label: 'AlphaFlow', heuristicLabel: 'Alpha Flow', processType: 'intra_community', stepCount: 3, communities: ['tools'], entryPointId: 'func:alpha', terminalId: 'func:hash'})`, + `CREATE (p:Process {id: 'proc:beta-flow', label: 'BetaFlow', heuristicLabel: 'Beta Flow', processType: 'intra_community', stepCount: 3, communities: ['tools'], entryPointId: 'func:beta', terminalId: 'func:validate'})`, // Relationships `MATCH (a:Function), (b:Function) WHERE a.id = 'func:login' AND b.id = 'func:validate' CREATE (a)-[:CodeRelation {type: 'CALLS', confidence: 1.0, reason: 'direct', step: 0}]->(b)`, @@ -29,6 +35,14 @@ export const LOCAL_BACKEND_SEED_DATA = [ CREATE (a)-[:CodeRelation {type: 'STEP_IN_PROCESS', confidence: 1.0, reason: '', step: 1}]->(p)`, `MATCH (a:Function), (p:Process) WHERE a.id = 'func:validate' AND p.id = 'proc:login-flow' CREATE (a)-[:CodeRelation {type: 'STEP_IN_PROCESS', confidence: 1.0, reason: '', step: 2}]->(p)`, + `MATCH (h:Function), (t:Tool) WHERE h.id = 'func:alpha' AND t.id = 'Tool:alpha' + CREATE (h)-[:CodeRelation {type: 'HANDLES_TOOL', confidence: 1.0, reason: 'tool-definition', step: 0}]->(t)`, + `MATCH (h:Function), (t:Tool) WHERE h.id = 'func:beta' AND t.id = 'Tool:beta' + CREATE (h)-[:CodeRelation {type: 'HANDLES_TOOL', confidence: 1.0, reason: 'tool-definition', step: 0}]->(t)`, + `MATCH (t:Tool), (p:Process) WHERE t.id = 'Tool:alpha' AND p.id = 'proc:alpha-flow' + CREATE (t)-[:CodeRelation {type: 'ENTRY_POINT_OF', confidence: 0.85, reason: 'tool-handler-entry-point', step: 0}]->(p)`, + `MATCH (t:Tool), (p:Process) WHERE t.id = 'Tool:beta' AND p.id = 'proc:beta-flow' + CREATE (t)-[:CodeRelation {type: 'ENTRY_POINT_OF', confidence: 0.85, reason: 'tool-handler-entry-point', step: 0}]->(p)`, // HAS_METHOD: AuthService -> authenticate `MATCH (c:Class), (m:Method) WHERE c.id = 'class:AuthService' AND m.id = 'method:AuthService.authenticate' CREATE (c)-[:CodeRelation {type: 'HAS_METHOD', confidence: 1.0, reason: 'class-method', step: 0}]->(m)`, diff --git a/gitnexus/test/integration/local-backend-calltool.test.ts b/gitnexus/test/integration/local-backend-calltool.test.ts index fa6a088582..81fd8f07b1 100644 --- a/gitnexus/test/integration/local-backend-calltool.test.ts +++ b/gitnexus/test/integration/local-backend-calltool.test.ts @@ -110,6 +110,17 @@ withTestLbugDB( expect(result.timing.bm25 ?? result.timing.vector).toBeGreaterThanOrEqual(0); }); + it('tool_map returns per-tool flows without cross-attributing same-file tools', async () => { + const result = await backend.callTool('tool_map', {}); + expect(result).not.toHaveProperty('error'); + + const tools = new Map(result.tools.map((tool: any) => [tool.name, tool])); + expect(tools.get('alpha')?.description).toBe('Calls chain A.'); + expect(tools.get('beta')?.description).toBe('Calls chain B.'); + expect(tools.get('alpha')?.flows).toEqual(['AlphaFlow']); + expect(tools.get('beta')?.flows).toEqual(['BetaFlow']); + }); + it('unknown tool throws', async () => { await expect(backend.callTool('nonexistent_tool', {})).rejects.toThrow(/unknown tool/i); }); diff --git a/gitnexus/test/integration/resolvers/python-mcp-tools.test.ts b/gitnexus/test/integration/resolvers/python-mcp-tools.test.ts index 5c29f6d265..d4f9229d59 100644 --- a/gitnexus/test/integration/resolvers/python-mcp-tools.test.ts +++ b/gitnexus/test/integration/resolvers/python-mcp-tools.test.ts @@ -4,6 +4,7 @@ import { FIXTURES, getRelationships, getNodesByLabel, + getNodesByLabelFull, runPipelineFromRepo, type PipelineResult, } from './helpers.js'; @@ -19,23 +20,62 @@ describe('Python @mcp.tool() detection', () => { const tools = getNodesByLabel(result, 'Tool'); expect(tools).toContain('get_weather'); expect(tools).toContain('search_docs'); + expect(tools).toContain('explicit_tool'); }); - it('creates HANDLES_TOOL edges from handler file to Tool nodes', () => { + it('uses handler functions for HANDLES_TOOL edges', () => { const edges = getRelationships(result, 'HANDLES_TOOL'); expect(edges.length).toBeGreaterThanOrEqual(2); const weatherEdge = edges.find((e) => e.target === 'get_weather'); expect(weatherEdge).toBeDefined(); - expect(weatherEdge!.sourceFilePath).toContain('server.py'); + expect(weatherEdge!.source).toBe('get_weather'); + expect(weatherEdge!.sourceLabel).toBe('Function'); const searchEdge = edges.find((e) => e.target === 'search_docs'); expect(searchEdge).toBeDefined(); - expect(searchEdge!.sourceFilePath).toContain('server.py'); + expect(searchEdge!.source).toBe('search_docs'); + expect(searchEdge!.sourceLabel).toBe('Function'); }); - it('detects exactly 2 tools from the fixture', () => { + it('detects exactly 3 tools from the fixture', () => { const tools = getNodesByLabel(result, 'Tool'); - expect(tools).toHaveLength(2); + expect(tools).toHaveLength(3); + }); + + it('uses Python handler docstrings as tool descriptions', () => { + const tools = getNodesByLabelFull(result, 'Tool'); + const descriptions = new Map(tools.map((tool) => [tool.name, tool.properties.description])); + + expect(descriptions.get('get_weather')).toBe('Get weather for a city.'); + expect(descriptions.get('search_docs')).toBe('Search documentation.'); + expect(descriptions.get('explicit_tool')).toBe('Explicit description'); + }); + + it('links each tool only to flows rooted at its handler', () => { + const edges = getRelationships(result, 'ENTRY_POINT_OF').filter( + (e) => e.sourceLabel === 'Tool' && e.targetLabel === 'Process', + ); + + const flowsByTool = new Map(); + for (const edge of edges) { + let flows = flowsByTool.get(edge.source); + if (flows === undefined) { + flows = []; + flowsByTool.set(edge.source, flows); + } + flows.push(edge.target); + } + + const weatherFlows = flowsByTool.get('get_weather') ?? []; + const searchFlows = flowsByTool.get('search_docs') ?? []; + + expect(weatherFlows).toHaveLength(1); + expect(weatherFlows[0]).toContain('Get_weather'); + expect(weatherFlows[0]).toContain('_format_weather'); + expect(searchFlows).toHaveLength(1); + expect(searchFlows[0]).toContain('Search_docs'); + expect(searchFlows[0]).toContain('_rank_docs'); + expect(weatherFlows).not.toEqual(searchFlows); }); }); diff --git a/gitnexus/test/unit/tool-process-linking.test.ts b/gitnexus/test/unit/tool-process-linking.test.ts new file mode 100644 index 0000000000..6c536bb38b --- /dev/null +++ b/gitnexus/test/unit/tool-process-linking.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it } from 'vitest'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { processesPhase } from '../../src/core/ingestion/pipeline-phases/processes.js'; +import { toolsPhase } from '../../src/core/ingestion/pipeline-phases/tools.js'; +import type { + PhaseResult, + PipelineContext, +} from '../../src/core/ingestion/pipeline-phases/types.js'; +import type { KnowledgeGraph } from '../../src/core/graph/types.js'; +import type { GraphNode, GraphRelationship, NodeLabel } from 'gitnexus-shared'; + +function makeCtx(graph: KnowledgeGraph, repoPath = 'D:/tmp/repo'): PipelineContext { + return { + repoPath, + graph, + onProgress: () => {}, + pipelineStart: 0, + }; +} + +function phaseResult(phaseName: string, output: T): PhaseResult { + return { phaseName, output, durationMs: 0 }; +} + +function addNode( + graph: KnowledgeGraph, + id: string, + label: NodeLabel, + name: string, + filePath: string, +) { + graph.addNode({ + id, + label, + properties: { + name, + filePath, + startLine: 1, + endLine: 1, + isExported: true, + content: '', + }, + } satisfies GraphNode); +} + +function addCall(graph: KnowledgeGraph, sourceId: string, targetId: string) { + graph.addRelationship({ + id: `${sourceId}->${targetId}`, + sourceId, + targetId, + type: 'CALLS', + confidence: 1, + reason: 'direct', + } satisfies GraphRelationship); +} + +describe('Tool handler and process linking phases', () => { + it('falls back to the file node when a parsed tool handler is missing from the graph', async () => { + const graph = createKnowledgeGraph(); + addNode(graph, 'File:src/tools.py', 'File', 'tools.py', 'src/tools.py'); + + const output = await toolsPhase.execute( + makeCtx(graph), + new Map([ + [ + 'parse', + phaseResult('parse', { + allToolDefs: [ + { + filePath: 'src/tools.py', + toolName: 'stale_tool', + description: 'Stale handler', + lineNumber: 1, + handlerNodeId: 'Function:src/tools.py:missing', + }, + ], + allPaths: [], + }), + ], + ]), + ); + + expect(output.toolDefs).toEqual([ + { name: 'stale_tool', filePath: 'src/tools.py', description: 'Stale handler' }, + ]); + + const edge = graph.relationships.find((rel) => rel.type === 'HANDLES_TOOL'); + expect(edge).toMatchObject({ + sourceId: 'File:src/tools.py', + targetId: 'Tool:stale_tool', + }); + }); + + it('does not attach file-level fallback tools to handler-specific processes', async () => { + const graph = createKnowledgeGraph(); + const filePath = 'src/tools.ts'; + const alpha = 'Function:src/tools.ts:alpha'; + const alphaHelper = 'Function:src/tools.ts:alphaHelper'; + const alphaLeaf = 'Function:src/tools.ts:alphaLeaf'; + const fileEntry = 'Function:src/tools.ts:fileEntry'; + const fileHelper = 'Function:src/tools.ts:fileHelper'; + const fileLeaf = 'Function:src/tools.ts:fileLeaf'; + + addNode(graph, 'File:src/tools.ts', 'File', 'tools.ts', filePath); + addNode(graph, alpha, 'Function', 'alpha', filePath); + addNode(graph, alphaHelper, 'Function', 'alphaHelper', filePath); + addNode(graph, alphaLeaf, 'Function', 'alphaLeaf', filePath); + addNode(graph, fileEntry, 'Function', 'fileEntry', filePath); + addNode(graph, fileHelper, 'Function', 'fileHelper', filePath); + addNode(graph, fileLeaf, 'Function', 'fileLeaf', filePath); + addNode(graph, 'Tool:alpha', 'Tool', 'alpha', filePath); + addNode(graph, 'Tool:fallback_tool', 'Tool', 'fallback_tool', filePath); + addCall(graph, alpha, alphaHelper); + addCall(graph, alphaHelper, alphaLeaf); + addCall(graph, fileEntry, fileHelper); + addCall(graph, fileHelper, fileLeaf); + + await processesPhase.execute( + makeCtx(graph), + new Map([ + ['structure', phaseResult('structure', { totalFiles: 1 })], + ['communities', phaseResult('communities', { communityResult: { memberships: [] } })], + ['routes', phaseResult('routes', { routeRegistry: new Map() })], + [ + 'tools', + phaseResult('tools', { + toolDefs: [ + { name: 'alpha', filePath, description: '', handlerNodeId: alpha }, + { name: 'fallback_tool', filePath, description: '' }, + ], + }), + ], + ]), + ); + + const processEntryById = new Map( + graph.nodes + .filter((node) => node.label === 'Process') + .map((node) => [node.id, node.properties.entryPointId]), + ); + const linkedEntriesByTool = new Map(); + for (const rel of graph.relationships.filter((edge) => edge.type === 'ENTRY_POINT_OF')) { + const list = linkedEntriesByTool.get(rel.sourceId) ?? []; + list.push(processEntryById.get(rel.targetId)); + linkedEntriesByTool.set(rel.sourceId, list); + } + + expect(linkedEntriesByTool.get('Tool:alpha')).toEqual([alpha]); + expect(linkedEntriesByTool.get('Tool:fallback_tool')).toEqual([fileEntry]); + }); +});