From ccf38129e67b59ce3b07258b98be48eeeaa32676 Mon Sep 17 00:00:00 2001 From: abhigyanpatwari Date: Tue, 17 Feb 2026 02:25:07 +0530 Subject: [PATCH] repowiki CLI command implemented --- .claude/settings.local.json | 35 +- .claude/worktrees/sweet-faraday | 1 + .gitignore | 3 + AGENTS.md | 2 +- ARCHITECTURE.md | 740 +++++----------- CLAUDE.md | 2 +- CLAUDE.md.bak | 61 -- README.md | 381 ++++---- eval/.env.example | 23 + eval/.gitignore | 16 + eval/README.md | 210 +++++ eval/__init__.py | 1 + eval/agents/__init__.py | 0 eval/agents/gitnexus_agent.py | 209 +++++ eval/analysis/__init__.py | 0 eval/analysis/analyze_results.py | 445 ++++++++++ eval/bridge/__init__.py | 0 eval/bridge/gitnexus_tools.sh | 155 ++++ eval/bridge/mcp_bridge.py | 336 +++++++ eval/configs/models/claude-haiku.yaml | 9 + eval/configs/models/claude-opus.yaml | 9 + eval/configs/models/claude-sonnet.yaml | 9 + eval/configs/models/glm-4.7.yaml | 7 + eval/configs/models/glm-5.yaml | 7 + eval/configs/models/minimax-2.5.yaml | 7 + eval/configs/modes/baseline.yaml | 9 + eval/configs/modes/native.yaml | 19 + eval/configs/modes/native_augment.yaml | 24 + eval/environments/__init__.py | 0 eval/environments/gitnexus_docker.py | 397 +++++++++ eval/prompts/instance_baseline.jinja | 80 ++ eval/prompts/instance_native.jinja | 102 +++ eval/prompts/instance_native_augment.jinja | 103 +++ eval/prompts/system_baseline.jinja | 15 + eval/prompts/system_native.jinja | 54 ++ eval/prompts/system_native_augment.jinja | 56 ++ eval/pyproject.toml | 35 + eval/run_eval.py | 508 +++++++++++ gitnexus/README.md | 113 ++- gitnexus/_hook_test.json | 1 - gitnexus/src/cli/analyze.ts | 2 +- gitnexus/src/cli/eval-server.ts | 430 +++++++++ gitnexus/src/cli/index.ts | 60 +- gitnexus/src/cli/mcp.ts | 10 + gitnexus/src/cli/serve.ts | 2 - gitnexus/src/cli/setup.ts | 57 +- gitnexus/src/cli/tool.ts | 120 +++ gitnexus/src/cli/wiki.ts | 234 +++++ .../src/core/embeddings/embedding-pipeline.ts | 19 +- gitnexus/src/core/embeddings/types.ts | 4 +- gitnexus/src/core/ingestion/call-processor.ts | 101 ++- .../src/core/ingestion/community-processor.ts | 73 +- .../src/core/ingestion/filesystem-walker.ts | 32 +- .../src/core/ingestion/heritage-processor.ts | 89 +- .../src/core/ingestion/import-processor.ts | 294 ++++++- .../src/core/ingestion/parsing-processor.ts | 176 ++-- .../core/ingestion/workers/parse-worker.ts | 535 ++++++++++++ .../src/core/ingestion/workers/worker-pool.ts | 89 ++ gitnexus/src/core/kuzu/kuzu-adapter.ts | 2 - gitnexus/src/core/kuzu/schema.ts | 3 + gitnexus/src/core/wiki/generator.ts | 819 ++++++++++++++++++ gitnexus/src/core/wiki/graph-queries.ts | 300 +++++++ gitnexus/src/core/wiki/llm-client.ts | 147 ++++ gitnexus/src/core/wiki/prompts.ts | 207 +++++ gitnexus/src/mcp/core/embedder.ts | 6 +- gitnexus/src/mcp/core/kuzu-adapter.ts | 53 +- gitnexus/src/mcp/local/local-backend.ts | 10 +- gitnexus/src/mcp/server.ts | 2 +- gitnexus/src/storage/git.ts | 58 -- gitnexus/src/storage/repo-manager.ts | 36 + todo_live_check_feature.md | 171 ---- 71 files changed, 7094 insertions(+), 1231 deletions(-) create mode 160000 .claude/worktrees/sweet-faraday delete mode 100644 CLAUDE.md.bak create mode 100644 eval/.env.example create mode 100644 eval/.gitignore create mode 100644 eval/README.md create mode 100644 eval/__init__.py create mode 100644 eval/agents/__init__.py create mode 100644 eval/agents/gitnexus_agent.py create mode 100644 eval/analysis/__init__.py create mode 100644 eval/analysis/analyze_results.py create mode 100644 eval/bridge/__init__.py create mode 100644 eval/bridge/gitnexus_tools.sh create mode 100644 eval/bridge/mcp_bridge.py create mode 100644 eval/configs/models/claude-haiku.yaml create mode 100644 eval/configs/models/claude-opus.yaml create mode 100644 eval/configs/models/claude-sonnet.yaml create mode 100644 eval/configs/models/glm-4.7.yaml create mode 100644 eval/configs/models/glm-5.yaml create mode 100644 eval/configs/models/minimax-2.5.yaml create mode 100644 eval/configs/modes/baseline.yaml create mode 100644 eval/configs/modes/native.yaml create mode 100644 eval/configs/modes/native_augment.yaml create mode 100644 eval/environments/__init__.py create mode 100644 eval/environments/gitnexus_docker.py create mode 100644 eval/prompts/instance_baseline.jinja create mode 100644 eval/prompts/instance_native.jinja create mode 100644 eval/prompts/instance_native_augment.jinja create mode 100644 eval/prompts/system_baseline.jinja create mode 100644 eval/prompts/system_native.jinja create mode 100644 eval/prompts/system_native_augment.jinja create mode 100644 eval/pyproject.toml create mode 100644 eval/run_eval.py delete mode 100644 gitnexus/_hook_test.json create mode 100644 gitnexus/src/cli/eval-server.ts create mode 100644 gitnexus/src/cli/tool.ts create mode 100644 gitnexus/src/cli/wiki.ts create mode 100644 gitnexus/src/core/ingestion/workers/parse-worker.ts create mode 100644 gitnexus/src/core/ingestion/workers/worker-pool.ts create mode 100644 gitnexus/src/core/wiki/generator.ts create mode 100644 gitnexus/src/core/wiki/graph-queries.ts create mode 100644 gitnexus/src/core/wiki/llm-client.ts create mode 100644 gitnexus/src/core/wiki/prompts.ts delete mode 100644 todo_live_check_feature.md diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 92cca123d7..a2d91f924e 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -32,7 +32,40 @@ "Bash(git add:*)", "Bash(Glob)", "Bash(Bash\"\\) per new Claude Code schema\n- Rename gitnexus-hook.js โ†’ gitnexus-hook.cjs for CommonJS compatibility\n- Fix setup.ts: correct hook filename and timeout \\(8000ms instead of 10ms\\)\n- Bump to v1.1.9 and publish to npm\n\nCo-Authored-By: Claude Opus 4.6 \nEOF\n\\)\")", - "Bash(git push:*)" + "Bash(git push:*)", + "WebFetch(domain:docs.kuzudb.com)", + "WebFetch(domain:github.com)", + "WebFetch(domain:raw.githubusercontent.com)", + "WebFetch(domain:read.engineerscodex.com)", + "WebFetch(domain:towardsdatascience.com)", + "WebFetch(domain:kilo.ai)", + "WebFetch(domain:deepwiki.com)", + "WebFetch(domain:turbopuffer.com)", + "WebFetch(domain:windsurf.com)", + "WebFetch(domain:modal.com)", + "WebFetch(domain:www.augmentcode.com)", + "WebFetch(domain:www.qodo.ai)", + "WebFetch(domain:arxiv.org)", + "WebFetch(domain:cognition.ai)", + "WebFetch(domain:microsoft.github.io)", + "WebFetch(domain:github.github.com)", + "WebFetch(domain:gist.github.com)", + "WebFetch(domain:fsoft-ai4code.github.io)", + "mcp__gitnexus__cypher", + "WebFetch(domain:repomix.com)", + "WebFetch(domain:www.humanlayer.dev)", + "WebFetch(domain:agents.md)", + "WebFetch(domain:eclipsesource.com)", + "WebFetch(domain:www.usefulfunctions.co.uk)", + "WebFetch(domain:developers.googleblog.com)", + "WebFetch(domain:www.anthropic.com)", + "WebFetch(domain:www.driver.ai)", + "WebFetch(domain:blog.sshh.io)", + "WebFetch(domain:docs.qodo.ai)", + "WebFetch(domain:smartlogic.io)", + "Bash(ls:*)", + "Bash(wc:*)", + "Bash(grep:*)" ] }, "enableAllProjectMcpServers": true, diff --git a/.claude/worktrees/sweet-faraday b/.claude/worktrees/sweet-faraday new file mode 160000 index 0000000000..44572ad0bd --- /dev/null +++ b/.claude/worktrees/sweet-faraday @@ -0,0 +1 @@ +Subproject commit 44572ad0bddea03d548b5f25ace720ba4f4c1548 diff --git a/.gitignore b/.gitignore index a92662bcbb..2f97f68d56 100644 --- a/.gitignore +++ b/.gitignore @@ -42,5 +42,8 @@ coverage/ .env*.local .gitnexus +# Assets (screenshots, images) +assets/ + # Generated files (should not be indexed) repomix-output* diff --git a/AGENTS.md b/AGENTS.md index 8bc8631769..290e07e9b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ # GitNexus MCP -This project is indexed by GitNexus as **GitnexusV2** (1021 symbols, 2497 relationships, 77 execution flows). +This project is indexed by GitNexus as **GitnexusV2** (1295 symbols, 3262 relationships, 99 execution flows). GitNexus provides a knowledge graph over this codebase โ€” call chains, blast radius, execution flows, and semantic search. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6b7f3144b6..6ad457e44d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,593 +1,247 @@ -# PyBaMM Architecture - End-to-End Analysis +# GitNexus Architecture -## Executive Summary +> Auto-generated from the GitNexus knowledge graph (1021 symbols, 2552 edges, 79 execution flows). -**PyBaMM** (Python Battery Mathematical Modelling) is a comprehensive, open-source framework for modeling and simulating battery behavior. The project contains **973 files**, **4,342 functions**, and **735 classes** organized in a layered architecture optimized for modularity, extensibility, and scientific computation. +## Overview ---- +GitNexus is a graph-powered code intelligence platform that indexes codebases into a knowledge graph and exposes them via MCP (Model Context Protocol) tools for AI agents. It consists of three packages: -## ๐Ÿ—๏ธ High-Level Architecture Layers +- **`gitnexus/`** โ€” Core CLI + MCP server (published to npm). Indexes repositories, stores graphs in KuzuDB, and serves queries. +- **`gitnexus-web/`** โ€” Browser-based frontend with WebAssembly tree-sitter and in-browser KuzuDB. +- **`gitnexus-claude-plugin/`** / **`gitnexus-cursor-integration/`** โ€” IDE integrations that augment AI agent tool calls with graph context. + +## Functional Areas + +The codebase is organized into 14 functional clusters detected by community analysis: + +| Module | Symbols | Cohesion | Responsibility | +|--------|---------|----------|----------------| +| **Ingestion** | 108 | 24% | Multi-phase pipeline: file walking, tree-sitter parsing, import resolution, call tracing, heritage extraction, community detection, process tracing | +| **Kuzu** | 50 | 23% | KuzuDB graph storage adapter, CSV generation, schema management, query execution | +| **Embeddings** | 46 | 35% | Embedding pipeline: text generation from symbols, ONNX model inference, vector storage | +| **Components** | 41 | 35% | Web UI React components (graph visualization, search, navigation) | +| **Local** | 38 | 15% | MCP backend: tool implementations (query, context, impact, rename), resource handlers, search (BM25 + semantic) | +| **Workers** | 38 | 27% | Web Workers for browser-side ingestion and tree-sitter parsing; Node.js worker threads for parallel parsing | +| **LLM** | 28 | 37% | LLM-based cluster enrichment, prompt building, provider abstraction | +| **CLI** | 24 | 35% | Command handlers (analyze, setup, serve, mcp), AI context file generation, IDE hook/skill installation | +| **Storage** | 22 | 32% | Repository registry, `.gitnexus/` directory management, staleness detection | +| **Services** | 14 | 31% | Shared services (config, ignore patterns, language support) | +| **Hooks** | 12 | 56% | Claude Code / Cursor hook scripts for augmenting search tools with graph context | +| **Search** | 11 | 36% | Hybrid search: BM25 keyword index + semantic vector search with reciprocal rank fusion | + +## Key Execution Flows + +### 1. CLI Analyze Pipeline + +The primary ingestion path when a user runs `npx gitnexus analyze`: ``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ USER INTERFACE & EXAMPLES โ”‚ -โ”‚ (Jupyter Notebooks, Scripts, Experiments) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ SIMULATION & EXPERIMENT ORCHESTRATION LAYER โ”‚ -โ”‚ โ€ข Simulation - High-level simulation runner โ”‚ -โ”‚ โ€ข Experiment - Define charging/discharging cycles โ”‚ -โ”‚ โ€ข BatchStudy - Multi-parameter studies โ”‚ -โ”‚ โ€ข Callbacks - Monitor simulation progress โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ MODEL LAYER (Hierarchical) โ”‚ -โ”œโ”€ BaseBatteryModel (Physical domain constraints) โ”‚ -โ”œโ”€ Full Models: โ”‚ -โ”‚ โ”œโ”€ Lithium-Ion (DFN, SPM, SPMe, MPM, MSMR, etc.) โ”‚ -โ”‚ โ”œโ”€ Lead-Acid (Full, LOQS models) โ”‚ -โ”‚ โ”œโ”€ Sodium-Ion (emerging battery chemistry) โ”‚ -โ”‚ โ””โ”€ Equivalent Circuit Models (ECM) โ”‚ -โ”œโ”€ Submodels (Pluggable domain-specific components): โ”‚ -โ”‚ โ”œโ”€ Particle Diffusion (kinetics in electrodes) โ”‚ -โ”‚ โ”œโ”€ Electrode Kinetics (Butler-Volmer, Marcus, etc.) โ”‚ -โ”‚ โ”œโ”€ Interface Chemistry (SEI growth, Li-plating, OCP) โ”‚ -โ”‚ โ”œโ”€ Thermal Management (lumped, distributed 1D-3D) โ”‚ -โ”‚ โ”œโ”€ Current Collector Physics โ”‚ -โ”‚ โ”œโ”€ Electrolyte Transport (conductivity, diffusion) โ”‚ -โ”‚ โ”œโ”€ Convection (internal circulation) โ”‚ -โ”‚ โ”œโ”€ Porosity & Tortuosity (pore network) โ”‚ -โ”‚ โ””โ”€ Active Material Loss (cycling degradation) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ EXPRESSION TREE (Symbolic Computation Layer) โ”‚ -โ”‚ Directed Acyclic Graph (DAG) of mathematical expressions โ”‚ -โ”œโ”€ Symbol - Base class for all nodes โ”‚ -โ”‚ โ”œโ”€ Variable - State vector entries โ”‚ -โ”‚ โ”œโ”€ Parameter - Model parameters โ”‚ -โ”‚ โ”œโ”€ Scalar/Array - Constants โ”‚ -โ”‚ โ”œโ”€ StateVector - Discretized spatial domain โ”‚ -โ”‚ โ””โ”€ InputParameter - Time-varying inputs โ”‚ -โ”œโ”€ Operators โ”‚ -โ”‚ โ”œโ”€ BinaryOperators - +, -, *, /, power, etc. โ”‚ -โ”‚ โ”œโ”€ UnaryOperators - exp, log, sin, cos, etc. โ”‚ -โ”‚ โ”œโ”€ Concatenations - Stack vectors โ”‚ -โ”‚ โ””โ”€ Broadcasts - Repeat/tile operations โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ DISCRETISATION LAYER (PDE โ†’ ODE/DAE Conversion) โ”‚ -โ”‚ Transforms continuous PDEs into discrete systems โ”‚ -โ”œโ”€ Discretisation - Master converter class โ”‚ -โ”œโ”€ Spatial Methods: โ”‚ -โ”‚ โ”œโ”€ FiniteVolume - 1D/2D finite volume schemes โ”‚ -โ”‚ โ”œโ”€ SpectralVolume - Spectral approach โ”‚ -โ”‚ โ”œโ”€ ScikitFiniteElement - 1D unstructured meshes โ”‚ -โ”‚ โ”œโ”€ ScikitFiniteElement3D- 3D tetrahedral meshes โ”‚ -โ”‚ โ””โ”€ ZeroDimensionalMethod- Lumped (0D) approximations โ”‚ -โ”œโ”€ Meshes: โ”‚ -โ”‚ โ”œโ”€ 1D Submeshes - Line domains โ”‚ -โ”‚ โ”œโ”€ 2D Submeshes - Sheet domains โ”‚ -โ”‚ โ”œโ”€ 3D Submeshes - Volume domains (via scikit-fem)โ”‚ -โ”‚ โ””โ”€ Composite Meshes - Combined domains โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ SOLVER LAYER (DAE System Integration) โ”‚ -โ”‚ Converts discrete system โ†’ numerical solution โ”‚ -โ”œโ”€ Solver Interfaces: โ”‚ -โ”‚ โ”œโ”€ BaseSolver - Abstract interface โ”‚ -โ”‚ โ”œโ”€ ODE Solvers: โ”‚ -โ”‚ โ”‚ โ”œโ”€ ScipySolver - scipy.integrate.ode โ”‚ -โ”‚ โ”‚ โ”œโ”€ JAXSolver - JAX backend (jit-compiled) โ”‚ -โ”‚ โ”‚ โ”œโ”€ JAXBDFSolver - JAX BDF method โ”‚ -โ”‚ โ”‚ โ””โ”€ IDAKLUSolver - SUNDIALS IDA (C++ wrapper) โ”‚ -โ”‚ โ”œโ”€ DAE Solvers: โ”‚ -โ”‚ โ”‚ โ”œโ”€ CasadiSolver - CasADi symbolic optimization โ”‚ -โ”‚ โ”‚ โ”œโ”€ IDakluJax - IDA + JAX hybrid โ”‚ -โ”‚ โ”‚ โ””โ”€ AlgebraicSolver - Solve algebraic eqns only โ”‚ -โ”‚ โ””โ”€ Special: โ”‚ -โ”‚ โ”œโ”€ DummySolver - Testing/debugging โ”‚ -โ”‚ โ””โ”€ Solution - Stores results + post-process โ”‚ -โ”œโ”€ Features: โ”‚ -โ”‚ โ”œโ”€ Jacobian Computation - Auto diff or symbolic โ”‚ -โ”‚ โ”œโ”€ Event Detection - Trigger on state changes โ”‚ -โ”‚ โ”œโ”€ Callbacks - Hooks during integration โ”‚ -โ”‚ โ””โ”€ Processed Variables - Post-compute derived quantitiesโ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ PARAMETER & DATA LAYER โ”‚ -โ”‚ Manages model coefficients and experimental data โ”‚ -โ”œโ”€ ParameterValues - Substitutes symbols โ†’ numbers โ”‚ -โ”œโ”€ Parameter Sets: โ”‚ -โ”‚ โ”œโ”€ Lithium-Ion Parameter Sets (Chen2020, OKane2022, etc)โ”‚ -โ”‚ โ”œโ”€ Lead-Acid Parameter Sets (Sulzer2019) โ”‚ -โ”‚ โ”œโ”€ Sodium-Ion Parameter Sets (Chayambuka2022) โ”‚ -โ”‚ โ””โ”€ ECM Parameter Sets (voltage model coefficients) โ”‚ -โ”œโ”€ Special Parameters: โ”‚ -โ”‚ โ”œโ”€ ElectricalParameters - Conductivity, diffusivity โ”‚ -โ”‚ โ”œโ”€ ThermalParameters - Heat capacity, conductivity โ”‚ -โ”‚ โ”œโ”€ GeometricParameters - Dimensions, areas, volumes โ”‚ -โ”‚ โ””โ”€ ProcessParameterData - Fit to experimental results โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ VISUALIZATION & POST-PROCESSING โ”‚ -โ”‚ Analysis and interpretation of results โ”‚ -โ”œโ”€ Plotting Modules: โ”‚ -โ”‚ โ”œโ”€ quick_plot() - 1-line quick visualization โ”‚ -โ”‚ โ”œโ”€ plot() - Customizable plotting โ”‚ -โ”‚ โ”œโ”€ plot_voltage_components() - Decompose voltage โ”‚ -โ”‚ โ”œโ”€ plot_summary_variables() - Key metrics โ”‚ -โ”‚ โ”œโ”€ plot_3d_heatmap() - 3D temperature fields โ”‚ -โ”‚ โ””โ”€ plot_3d_cross_section() - 2D slices of 3D โ”‚ -โ”œโ”€ Dynamic Plotting: โ”‚ -โ”‚ โ””โ”€ DynamicPlot - Live update during solving โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +analyzeCommand (cli/analyze.ts) + โ†’ runPipelineFromRepo (ingestion/pipeline.ts) + โ†’ walkRepository โ€” concurrent file I/O (32 parallel reads) + โ†’ processStructure โ€” folder/file graph nodes + โ†’ processParsing โ€” tree-sitter AST parsing (worker threads) + โ†’ processImports โ€” import/require/use resolution + โ†’ processCalls โ€” function call tracing with confidence + โ†’ processHeritage โ€” extends/implements relationships + โ†’ processCommunities โ€” Leiden community detection + โ†’ processProcesses โ€” execution flow tracing + โ†’ loadGraphToKuzu โ€” CSV export โ†’ KuzuDB bulk import + โ†’ runEmbeddingPipeline โ€” generate + store symbol embeddings + โ†’ generateAIContextFiles โ€” write CLAUDE.md, AGENTS.md, skills ``` ---- +### 2. MCP Server Request Flow -## ๐Ÿ“Š Core Components Deep Dive - -### 1. **Expression Tree (Symbolic Computation)** - -**Purpose:** Represents mathematical expressions as a directed acyclic graph (DAG). - -**Key Classes:** +When an AI agent calls a GitNexus MCP tool: ``` -Symbol (Base Class) -โ”œโ”€โ”€ Variable - Represents y(t), y_dot(t) -โ”œโ”€โ”€ Parameter - Fixed model coefficients -โ”œโ”€โ”€ Scalar/Array - Numerical constants -โ”œโ”€โ”€ StateVector - Discretized spatial variables -โ”œโ”€โ”€ InputParameter - Time-varying inputs (current, temperature) -โ”‚ -BinaryOperator -โ”œโ”€โ”€ Addition/Subtraction -โ”œโ”€โ”€ Multiplication/Division -โ”œโ”€โ”€ Power -โ”œโ”€โ”€ MatrixMultiplication -โ””โ”€โ”€ Equality (for algebraic equations) - -UnaryOperator -โ”œโ”€โ”€ Exponential, Logarithm -โ”œโ”€โ”€ Trigonometric (sin, cos, tan) -โ”œโ”€โ”€ Sign, Absolute Value -โ””โ”€โ”€ Specialized (exp, log, cosh, etc.) +mcpCommand (cli/mcp.ts) + โ†’ startMCPServer (mcp/server.ts) + โ†’ callTool (mcp/local/local-backend.ts) + โ†’ ensureInitialized โ€” lazy KuzuDB connection + embedder init + โ†’ query/context/impact โ€” graph traversal via KuzuDB Cypher + โ†’ semanticSearch โ€” ONNX embedding + vector similarity + โ†’ readResource (mcp/resources.ts) + โ†’ queryClusters/queryProcesses โ€” direct graph queries ``` -**Why This Matters:** -- Enables **symbolic differentiation** (Jacobian computation) -- **Backend-agnostic**: Same expression can be evaluated as Python, CasADi, or JAX code -- Supports **automatic code generation** for performance - ---- - -### 2. **Model Hierarchy** - -**Top Level: `BaseModel`** -- Holds empty RHS and algebraic equation dictionaries -- Manages variables, parameters, boundary conditions -- Coordinates discretisation and conversion - -**Next Level: `BaseBatteryModel`** -- Enforces battery-specific physics constraints -- Implements standard lifecycle: `build_model()` โ†’ `discretise()` โ†’ `solve()` - -**Bottom Level: Concrete Models (Plug-and-Play Architecture)** - -| Model | Type | Complexity | Use Case | -|-------|------|-----------|----------| -| **SPM** | Lithium-Ion | Simplest | Quick simulations, education | -| **SPMe** | Lithium-Ion | Medium | Semi-empirical electrolyte | -| **DFN** | Lithium-Ion | Complex | High accuracy, research | -| **MSMR** | Lithium-Ion | Very Complex | Multi-scale particle size dist. | -| **MPM** | Lithium-Ion | Complex | Mesoscale particle modeling | -| **Half-Cell** | Lithium-Ion | Custom | Single electrode testing | -| **Thermal Models** | Any | Adds complexity | Temperature effects | -| **ECM (Thevenin)** | Equivalent Circuit | Simple | Real-time estimation | - -**Submodel Pattern:** -``` -Full Models = Combination of pluggable submodels - -Example: DFN Model -โ”œโ”€โ”€ Active Material (constant or loss) -โ”œโ”€โ”€ Particle Diffusion (negative & positive electrodes) -โ”œโ”€โ”€ Electrode Kinetics (interface reactions) -โ”œโ”€โ”€ Open Circuit Potential (voltage lookup) -โ”œโ”€โ”€ SEI Growth (lithium loss) -โ”œโ”€โ”€ Current Collector (ohmic drop) -โ”œโ”€โ”€ Convection (internal flow) -โ”œโ”€โ”€ Thermal (heat generation & transfer) -โ””โ”€โ”€ External Circuit (boundary conditions) -``` - ---- - -### 3. **Discretisation Pipeline** - -**Convert PDEs โ†’ Finite-Dimensional ODEs/DAEs** +### 3. Graph Storage Pipeline ``` -Physics-Based PDE - โ†“ -[Spatial Method Selected: Finite Volume / Spectral / FEM] - โ†“ -Mesh Generation (1D/2D/3D depending on model) - โ†“ -Gradient/Divergence Operators Discretized - โ†“ -Boundary Conditions Applied - โ†“ -Expression Tree Converted (y โ†’ discretized vector) - โ†“ -Final System: M*dy/dt = f(t,y) + g(t,y) = 0 [DAE form] +loadGraphToKuzu (kuzu/kuzu-adapter.ts) + โ†’ generateAllCSVs (kuzu/csv-generator.ts) + โ†’ generateFileCSV โ€” node CSVs per label type + โ†’ escapeCSVField โ€” UTF-8 sanitization + โ†’ KuzuDB COPY FROM โ€” bulk CSV import + โ†’ createIndexes โ€” property indexes for query performance ``` -**Mesh Strategy:** -- **1D**: Uniform or non-uniform grids (electrodes, separator) -- **2D**: Cartesian or polar (pouch cell cross-sections) -- **3D**: Tetrahedral (scikit-fem), complex geometries - ---- - -### 4. **Solver Pipeline** - -**Goal:** Integrate DAE system over time - -**Solver Family:** -- **ScipySolver**: Reliable, well-tested, pure Python -- **CasadiSolver**: Symbolic optimization, slow but accurate -- **IDAKLUSolver**: C++ SUNDIALS, fastest -- **JAXSolver**: JIT-compiled, GPU-capable -- **IDakluJax**: Hybrid IDA + JAX - -**Key Features:** -- **Event Detection**: Stop when voltage hits limit -- **Jacobian**: Computed symbolically or via auto-diff -- **Callbacks**: Monitor state during integration -- **Mass Matrix**: Handle DAE systems with singular mass matrices - ---- - -### 5. **Parameter System** - -**Strategy:** Keep symbolic model separate from numerical values +### 4. Embedding Pipeline ``` -Model Construction: - pybamm.Parameter("Conductivity") โ†’ generic symbol - โ†“ - [Stored in expression tree] - โ†“ -Before Solving: - parameter_values = pybamm.ParameterValues({ - "Conductivity": 1.23 # Numerical value - }) - parameter_values.process_model(model) - โ†“ - All symbols substituted with values - โ†“ - Ready to solve! +runEmbeddingPipeline (embeddings/embedding-pipeline.ts) + โ†’ generateBatchEmbeddingTexts (embeddings/text-generator.ts) + โ†’ generateEmbeddingText โ€” symbol โ†’ natural language description + โ†’ generateFunctionText โ€” include signature, calls, file context + โ†’ cleanContent โ€” strip noise, truncate + โ†’ embedBatch โ€” ONNX Runtime inference (all-MiniLM-L6-v2) + โ†’ storeEmbeddings โ€” KuzuDB vector storage ``` -**Pre-built Parameter Sets:** -- **Lithium-Ion**: Chen2020, OKane2022, Ai2020, Ecker2015, ORegan2022 -- **Lead-Acid**: Sulzer2019 -- **Sodium-Ion**: Chayambuka2022 -- **ECM**: Thevenin model coefficients - ---- - -## ๐Ÿ”„ Execution Flow: From Model to Solution - -### Example: Simple SPM Simulation - -```python -import pybamm - -# Step 1: Create model -model = pybamm.lithium_ion.SPM() - -# Step 2: Define simulation -sim = pybamm.Simulation( - model, - parameter_values=pybamm.ParameterValues("Chen2020"), - solver=pybamm.IDAKLUSolver() -) - -# Step 3: Run -sim.solve([0, 3600]) # Solve 1 hour - -# Step 4: Plot -sim.plot() -``` - -**Behind the Scenes:** - -1. **Model Initialization** โ†’ Submodels concatenated -2. **Build Phase** โ†’ RHS, algebraic equations assembled -3. **Parameter Substitution** โ†’ Symbols replaced with values -4. **Discretisation** โ†’ Spatial PDE โ†’ ODE/DAE -5. **Jacobian Computation** โ†’ Auto-differentiation -6. **Solver Setup** โ†’ Initial conditions, events configured -7. **Integration Loop** โ†’ Time-stepping with callbacks -8. **Post-Processing** โ†’ Compute derived variables (impedance, etc.) -9. **Visualization** โ†’ Plot results - ---- - -## ๐Ÿ”— Key Dependencies & Data Flow - -### Upstream (Inputs) -``` -Experiment (current profile) - โ†“ -ParameterValues (physical constants) - โ†“ -Geometry (cell dimensions) - โ†“ -ModelOptions (choose submodels) - โ†“ -BaseModel -``` - -### Downstream (Outputs) -``` -Discretisation - โ†“ -DAE System (M*dy/dt = f(t,y)) - โ†“ -Solver - โ†“ -Solution object (t, y, processed_variables) - โ†“ -Plotting/Analysis - โ†“ -Results (voltage, capacity, temperature, etc.) -``` - ---- - -## ๐ŸŒณ Hotspot Nodes (Most Connected Components) - -These are the "hubs" that everything depends on: - -| Node | Type | Connections | Role | -|------|------|-----------|------| -| `src/pybamm/__init__.py` | File | **500** | Central export hub | -| `Variable` | Class | **474** | Core state representation | -| `Scalar` | Class | **397** | Constant handling | -| `evaluate()` | Function | **344** | Expression evaluation | -| `solve()` | Function | **311** | Solver invocation | -| `BaseModel` | Class | **305** | Model parent | -| `Discretisation` | Class | **289** | Discretisation orchestration | -| `linspace()` | Function | **267** | Mesh generation | - ---- - -## ๐Ÿ“ Directory Structure +### 5. Web App Pipeline (Browser) ``` -src/pybamm/ -โ”œโ”€โ”€ models/ # Model hierarchy -โ”‚ โ”œโ”€โ”€ base_model.py # Abstract base -โ”‚ โ”œโ”€โ”€ full_battery_models/ # Concrete implementations -โ”‚ โ”‚ โ”œโ”€โ”€ lithium_ion/ -โ”‚ โ”‚ โ”œโ”€โ”€ lead_acid/ -โ”‚ โ”‚ โ”œโ”€โ”€ sodium_ion/ -โ”‚ โ”‚ โ””โ”€โ”€ equivalent_circuit/ -โ”‚ โ””โ”€โ”€ submodels/ # Pluggable physics components -โ”‚ โ”œโ”€โ”€ interface/ # Electrode kinetics, SEI, OCP -โ”‚ โ”œโ”€โ”€ particle/ # Particle diffusion -โ”‚ โ”œโ”€โ”€ thermal/ # Heat transfer -โ”‚ โ”œโ”€โ”€ electrode/ # Ohmic drop -โ”‚ โ”œโ”€โ”€ convection/ # Internal flow -โ”‚ โ””โ”€โ”€ [more...] -โ”‚ -โ”œโ”€โ”€ expression_tree/ # Symbolic DAG -โ”‚ โ”œโ”€โ”€ symbol.py # Base class -โ”‚ โ”œโ”€โ”€ binary_operators.py # +, -, *, / -โ”‚ โ”œโ”€โ”€ unary_operators.py # sin, exp, log -โ”‚ โ”œโ”€โ”€ operations/ # Evaluation, Jacobian, serialization -โ”‚ โ””โ”€โ”€ [more...] -โ”‚ -โ”œโ”€โ”€ discretisations/ # PDE โ†’ ODE conversion -โ”‚ โ””โ”€โ”€ discretisation.py -โ”‚ -โ”œโ”€โ”€ spatial_methods/ # Finite volume, spectral, FEM -โ”‚ โ”œโ”€โ”€ finite_volume.py -โ”‚ โ”œโ”€โ”€ spectral_volume.py -โ”‚ โ””โ”€โ”€ [more...] -โ”‚ -โ”œโ”€โ”€ meshes/ # Grid generation -โ”‚ โ”œโ”€โ”€ meshes.py -โ”‚ โ””โ”€โ”€ [submesh types...] -โ”‚ -โ”œโ”€โ”€ solvers/ # DAE integration -โ”‚ โ”œโ”€โ”€ base_solver.py -โ”‚ โ”œโ”€โ”€ scipy_solver.py -โ”‚ โ”œโ”€โ”€ casadi_solver.py -โ”‚ โ”œโ”€โ”€ idaklu_solver.py -โ”‚ โ””โ”€โ”€ [more...] -โ”‚ -โ”œโ”€โ”€ parameters/ # Physical coefficients -โ”‚ โ”œโ”€โ”€ base_parameters.py -โ”‚ โ”œโ”€โ”€ parameter_values.py -โ”‚ โ”œโ”€โ”€ lithium_ion_parameters.py -โ”‚ โ””โ”€โ”€ input/ -โ”‚ โ””โ”€โ”€ parameters/ # Pre-built parameter sets -โ”‚ -โ”œโ”€โ”€ plotting/ # Visualization -โ”‚ โ”œโ”€โ”€ plot.py -โ”‚ โ”œโ”€โ”€ quick_plot.py -โ”‚ โ”œโ”€โ”€ plot_voltage_components.py -โ”‚ โ””โ”€โ”€ [more...] -โ”‚ -โ”œโ”€โ”€ batch_study.py # Multi-parameter studies -โ”œโ”€โ”€ simulation.py # High-level runner -โ”œโ”€โ”€ experiment/ # Charge/discharge cycles -โ””โ”€โ”€ [more...] - -tests/ -โ”œโ”€โ”€ unit/ # Isolated component tests -โ””โ”€โ”€ integration/ # End-to-end tests +AppStateProvider (hooks/useAppState.tsx) + โ†’ runPipeline (workers/ingestion.worker.ts) โ€” Web Worker + โ†’ runPipelineFromFiles (ingestion/pipeline.ts) + โ†’ createKnowledgeGraph + โ†’ processParsing โ€” WASM tree-sitter + โ†’ processImports/Calls/Heritage + โ†’ processCommunities/Processes + โ†’ loadGraphToKuzu โ€” in-browser KuzuDB (WASM) ``` ---- +## Architecture Diagram -## ๐ŸŽฏ Design Patterns +```mermaid +graph TB + subgraph CLI["CLI Layer"] + analyze["analyze command"] + setup["setup command"] + mcp_cmd["mcp command"] + serve["serve command"] + augment["augment command"] + end -### 1. **Plugin Architecture (Submodels)** -- Models are built by combining plug-and-play submodels -- Easy to swap implementations (e.g., different kinetics models) -- **Example**: Switch from Butler-Volmer to Marcus kinetics + subgraph Ingestion["Ingestion Pipeline"] + walker["Filesystem Walker
(concurrent I/O)"] + structure["Structure Processor"] + parsing["Parsing Processor
(worker threads)"] + imports["Import Processor"] + calls["Call Processor"] + heritage["Heritage Processor"] + communities["Community Detection
(Leiden)"] + processes["Process Tracing"] + end -### 2. **Expression Tree Pattern** -- Decouple symbolic math from backend -- Same expression โ†’ Python, CasADi, or JAX code -- Enables automatic differentiation + subgraph TreeSitter["Tree-Sitter"] + parser_loader["Parser Loader"] + ts_queries["Language Queries
(9 languages)"] + worker_pool["Worker Pool"] + parse_worker["Parse Workers"] + end -### 3. **Factory Pattern (Solvers)** -- `solve()` returns appropriate solver based on model type -- User doesn't need to know solver implementation details + subgraph MCP["MCP Server"] + server["MCP Server
(stdio transport)"] + tools["Tools: query, context,
impact, rename, cypher"] + resources["Resources: clusters,
processes, schema"] + backend["Local Backend"] + end -### 4. **Strategy Pattern (Spatial Methods)** -- Choose discretization strategy (FV, Spectral, FEM) at runtime -- Swap without changing model code + subgraph Storage["Storage Layer"] + kuzu["KuzuDB
(graph store)"] + csv_gen["CSV Generator"] + repo_mgr["Repo Manager
(~/.gitnexus registry)"] + end -### 5. **Template Method (Model Lifecycle)** -1. `model.build_model()` -2. `disc.discretise(model)` -3. `solver.solve(t_eval, y0)` + subgraph Search["Search Engine"] + bm25["BM25 Keyword Index"] + semantic["Semantic Search
(all-MiniLM-L6-v2)"] + embedder["ONNX Embedder"] + end ---- + subgraph Hooks["IDE Integration"] + claude_hook["Claude Code Hooks"] + cursor_hook["Cursor Hooks"] + skills["Skills
(exploring, debugging,
impact, refactoring)"] + end -## ๐Ÿš€ Performance Considerations + subgraph Web["Web Frontend"] + app["React App"] + web_worker["Web Worker
(WASM ingestion)"] + components["Graph Visualization"] + end -### Bottlenecks -1. **Discretisation**: Large spatial grids โ†’ huge state vectors -2. **Jacobian Computation**: Dense matrices for implicit solvers -3. **Parameter Substitution**: Re-expression tree traversal + %% CLI โ†’ Ingestion + analyze --> walker + walker --> structure --> parsing --> imports --> calls --> heritage --> communities --> processes -### Optimizations -1. **CasADi Backend**: Symbolic optimization + JIT -2. **JAX Solver**: GPU acceleration, batched derivatives -3. **IDA Solver**: C++ wrapper, sparse Jacobian support -4. **LRU Caching**: Avoid recomputation + %% Parsing uses tree-sitter workers + parsing --> worker_pool --> parse_worker + parse_worker --> parser_loader + parse_worker --> ts_queries ---- + %% Ingestion โ†’ Storage + processes --> csv_gen --> kuzu + processes --> embedder -## ๐Ÿ” Testing Strategy + %% MCP flow + mcp_cmd --> server --> tools --> backend --> kuzu + server --> resources --> backend + backend --> bm25 + backend --> semantic --> embedder -### Unit Tests (973 files) -- Component-level validation -- Expression tree operations -- Spatial method correctness + %% CLI โ†’ Setup + setup --> repo_mgr + setup --> claude_hook + setup --> cursor_hook + setup --> skills -### Integration Tests -- Full model runs -- Solver convergence -- Different parameter sets + %% Hooks โ†’ MCP + claude_hook -.->|augments searches| tools + cursor_hook -.->|augments searches| tools + augment -.->|fast CLI path| backend -### Benchmark Tests -- Performance tracking -- Memory profiling -- Scaling analysis + %% Web + app --> web_worker --> components + web_worker --> kuzu ---- - -## ๐Ÿ“š Key Math Concepts - -### Governing Equations -**DAE System:** -``` -M(t,y) * dy/dt = f(t, y, u(t)) [Differential equations] -0 = g(t, y, u(t)) [Algebraic equations] + %% Serve + serve --> backend ``` -where: -- `y` = state vector (concentrations, potentials, temperature) -- `u(t)` = inputs (applied current, ambient temperature) -- `M` = mass matrix (handles singular systems) +## Data Flow Summary -### Typical Physics - -**Particle Diffusion (Fick's Law):** ``` -โˆ‚c/โˆ‚t = โˆ‡ยท(Dโˆ‡c) +Source Code + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Ingestion Pipeline (8 phases) โ”‚ +โ”‚ Files โ†’ AST โ†’ Symbols โ†’ Relationships โ”‚ +โ”‚ โ†’ Communities โ†’ Execution Flows โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ–ผ โ–ผ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ KuzuDB โ”‚ โ”‚ Embeddings โ”‚ + โ”‚ (graph) โ”‚ โ”‚ (vectors) โ”‚ + โ””โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ–ผ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ MCP Server โ”‚ + โ”‚ (7 tools) โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ–ผ โ–ผ โ–ผ + Claude Cursor Other + Code Editor MCP Clients ``` -**Charge Conservation (Poisson):** -``` -โˆ‡ยท(ฯƒโˆ‡ฯ†) = i -``` - -**Energy Balance (Heat Equation):** -``` -ฯCp โˆ‚T/โˆ‚t = โˆ‡ยท(kโˆ‡T) + Q_gen -``` - ---- - -## ๐ŸŽ“ Learning Path - -1. **Start**: Run SPM model (`pybamm.lithium_ion.SPM()`) -2. **Progress**: Modify parameter set, change solver -3. **Intermediate**: Swap submodels (DFN, thermal) -4. **Advanced**: Create custom submodel -5. **Expert**: Implement new spatial method - ---- - -## ๐Ÿ”ฎ Architecture Strengths - -โœ… **Modularity**: Plug-and-play submodels -โœ… **Extensibility**: Easy to add new models/solvers -โœ… **Physics-First**: Expression tree mirrors actual equations -โœ… **Backend-Agnostic**: Switch solvers without changing model -โœ… **Scientific Quality**: Validated against experiments -โœ… **Performance**: Multiple backends (Python, C++, JAX) - ---- - -## โš ๏ธ Architecture Tradeoffs - -โš–๏ธ **Complexity**: Large learning curve -โš–๏ธ **Symbolic Overhead**: DAG construction has memory cost -โš–๏ธ **Debug Difficulty**: Multiple abstraction layers -โš–๏ธ **Startup Time**: Model compilation + discretisation - ---- - -## ๐ŸŽฏ Conclusion - -PyBaMM's architecture is a **layered, modular system** optimized for: -- **Scientific fidelity** (physics-based discretisation) -- **Extensibility** (plug-and-play submodels) -- **Performance** (multiple backends) -- **Usability** (high-level simulation API) - -The design cleanly separates concerns across 7 layers, from symbolic math to numerical solvers, making it suitable for both research and production use. - ---- - -*Analysis powered by GitNexus MCP - Code Intelligence Engine* - - - - - - - - +## Supported Languages +Tree-sitter grammars are included for: **TypeScript**, **JavaScript**, **Python**, **Java**, **C**, **C++**, **C#**, **Go**, **Rust**. +## Key Design Decisions +1. **Augmentation over replacement** โ€” Hooks enrich existing AI agent tools (Grep, Glob, Bash) with graph context rather than replacing them +2. **Native tree-sitter** โ€” Uses N-API bindings (not WASM) in the CLI for performance; WASM in the browser +3. **Worker thread parsing** โ€” CPU-bound tree-sitter parsing parallelized across `cpus - 1` worker threads +4. **Hybrid search** โ€” BM25 keyword + semantic vector search combined with Reciprocal Rank Fusion for ranking +5. **LRU AST cache** โ€” Parsed trees are cached across pipeline phases to avoid redundant re-parsing +6. **Deterministic IDs** โ€” `generateId(label, qualifiedName)` ensures idempotent graph construction diff --git a/CLAUDE.md b/CLAUDE.md index aef4e66516..cf96c279c5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,7 +1,7 @@ # GitNexus MCP -This project is indexed by GitNexus as **GitnexusV2** (1021 symbols, 2497 relationships, 77 execution flows). +This project is indexed by GitNexus as **GitnexusV2** (1295 symbols, 3262 relationships, 99 execution flows). GitNexus provides a knowledge graph over this codebase โ€” call chains, blast radius, execution flows, and semantic search. diff --git a/CLAUDE.md.bak b/CLAUDE.md.bak deleted file mode 100644 index 876be3fd77..0000000000 --- a/CLAUDE.md.bak +++ /dev/null @@ -1,61 +0,0 @@ - -# GitNexus MCP - -This project is indexed by GitNexus as **GitnexusV2** (940 symbols, 2298 relationships, 13 clusters, 67 processes). - -GitNexus provides a knowledge graph over this codebase โ€” clusters, call chains, blast radius, execution flows, and semantic search. - -## Always Start Here - -For any task involving code understanding, debugging, impact analysis, or refactoring, you must: - -1. **Read `gitnexus://repo/{name}/context`** โ€” codebase overview + check index freshness -2. **Match your task to a skill below** and **read that skill file** -3. **Follow the skill's workflow and checklist** - -> If step 1 warns the index is stale, run `npx gitnexus analyze` in the terminal first. - -## Skills - -| Task | Read this skill file | -|------|---------------------| -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus/refactoring/SKILL.md` | - -## Tools Reference - -| Tool | What it gives you | -|------|-------------------| -| `search` | Semantic + keyword code search with cluster context | -| `explore` | Symbol deep dive โ€” callers, callees, cluster membership, processes | -| `impact` | Blast radius โ€” what breaks at depth 1/2/3 with confidence scores | -| `overview` | All clusters and processes at a glance | -| `cypher` | Raw graph queries (read `gitnexus://repo/{name}/schema` first) | -| `list_repos` | Discover indexed repos | - -## Resources Reference - -Lightweight reads (~100-500 tokens) for navigation: - -| Resource | Content | -|----------|---------| -| `gitnexus://repo/{name}/context` | Stats, staleness check | -| `gitnexus://repo/{name}/clusters` | All clusters with cohesion scores | -| `gitnexus://repo/{name}/cluster/{clusterName}` | Cluster members | -| `gitnexus://repo/{name}/processes` | All execution flows | -| `gitnexus://repo/{name}/process/{processName}` | Step-by-step trace | -| `gitnexus://repo/{name}/schema` | Graph schema for Cypher | - -## Graph Schema - -**Nodes:** File, Function, Class, Interface, Method, Community, Process -**Edges (via CodeRelation.type):** CALLS, IMPORTS, EXTENDS, IMPLEMENTS, DEFINES, MEMBER_OF, STEP_IN_PROCESS - -```cypher -MATCH (caller)-[:CodeRelation {type: 'CALLS'}]->(f:Function {name: "myFunc"}) -RETURN caller.name, caller.filePath -``` - - diff --git a/README.md b/README.md index 7927e24a01..87776f3aff 100644 --- a/README.md +++ b/README.md @@ -17,15 +17,15 @@ https://github.com/user-attachments/assets/abfd0300-0aae-4296-b8d3-8b72ed882433 ## Two Ways to Use GitNexus -| | **CLI + MCP** | **Web UI** | -|---|---|---| -| **What** | Index repos locally, connect AI agents via MCP | Visual graph explorer + AI chat in browser | -| **For** | Daily development with Cursor, Claude Code, Windsurf, OpenCode | Quick exploration, demos, one-off analysis | -| **Scale** | Full repos, any size | Limited by browser memory (~5k files) | -| **Install** | `npm install -g gitnexus` | No install โ€” [gitnexus.vercel.app](https://gitnexus.vercel.app) | -| **Storage** | KuzuDB native (fast, persistent) | KuzuDB WASM (in-memory, per session) | -| **Parsing** | Tree-sitter native bindings | Tree-sitter WASM | -| **Privacy** | Everything local, no network | Everything in-browser, no server | +| | **CLI + MCP** | **Web UI** | +| ----------------- | -------------------------------------------------------------- | ------------------------------------------------------------ | +| **What** | Index repos locally, connect AI agents via MCP | Visual graph explorer + AI chat in browser | +| **For** | Daily development with Cursor, Claude Code, Windsurf, OpenCode | Quick exploration, demos, one-off analysis | +| **Scale** | Full repos, any size | Limited by browser memory (~5k files) | +| **Install** | `npm install -g gitnexus` | No install โ€”[gitnexus.vercel.app](https://gitnexus.vercel.app) | +| **Storage** | KuzuDB native (fast, persistent) | KuzuDB WASM (in-memory, per session) | +| **Parsing** | Tree-sitter native bindings | Tree-sitter WASM | +| **Privacy** | Everything local, no network | Everything in-browser, no server | --- @@ -36,31 +36,37 @@ The CLI indexes your repository and runs an MCP server that gives AI agents deep ### Quick Start ```bash -# Install -npm install -g gitnexus - -# One-time setup: configures MCP for Cursor, Claude Code, OpenCode -gitnexus setup - # Index your repo (run from repo root) -gitnexus analyze - -# That's it! Open your editor โ€” the MCP server starts automatically. +npx gitnexus analyze ``` -Or without installing globally: +That's it. This indexes the codebase, installs agent skills, registers Claude Code hooks, and creates `AGENTS.md` / `CLAUDE.md` context files โ€” all in one command. -```bash -npx gitnexus setup # one-time -npx gitnexus analyze # per repo -``` +To configure MCP for your editor, run `npx gitnexus setup` once โ€” or set it up manually below. ### MCP Setup -The `gitnexus setup` command auto-detects your editors and writes the correct MCP config. You only need to run it once. +`gitnexus setup` auto-detects your editors and writes the correct global MCP config. You only need to run it once. + +### Editor Support + +| Editor | MCP | Skills | Hooks (auto-augment) | Support | +| --------------------- | --- | ------ | -------------------- | -------------- | +| **Claude Code** | Yes | Yes | Yes (PreToolUse) | **Full** | +| **Cursor** | Yes | Yes | โ€” | MCP + Skills | +| **Windsurf** | Yes | โ€” | โ€” | MCP | +| **OpenCode** | Yes | Yes | โ€” | MCP + Skills | + +> **Claude Code** gets the deepest integration: MCP tools + agent skills + PreToolUse hooks that automatically enrich grep/glob/bash calls with knowledge graph context. If you prefer manual configuration: +**Claude Code** (full support โ€” MCP + skills + hooks): + +```bash +claude mcp add gitnexus -- npx -y gitnexus@latest mcp +``` + **Cursor** (`~/.cursor/mcp.json` โ€” global, works for all projects): ```json @@ -74,12 +80,6 @@ If you prefer manual configuration: } ``` -**Claude Code:** - -```bash -claude mcp add gitnexus -- npx -y gitnexus@latest mcp -``` - **OpenCode** (`~/.config/opencode/config.json`): ```json @@ -96,44 +96,55 @@ claude mcp add gitnexus -- npx -y gitnexus@latest mcp ### CLI Commands ```bash -gitnexus setup # Configure MCP for your editors (one-time) -gitnexus analyze [path] # Index a repository (or update stale index) -gitnexus analyze --force # Force full re-index -gitnexus mcp # Start MCP server (stdio) โ€” serves all indexed repos -gitnexus serve # Start HTTP server for web UI connection -gitnexus list # List all indexed repositories -gitnexus status # Show index status for current repo -gitnexus clean # Delete index for current repo -gitnexus clean --all # Delete all indexes +gitnexus setup # Configure MCP for your editors (one-time) +gitnexus analyze [path] # Index a repository (or update stale index) +gitnexus analyze --force # Force full re-index +gitnexus analyze --skip-embeddings # Skip embedding generation (faster) +gitnexus mcp # Start MCP server (stdio) โ€” serves all indexed repos +gitnexus serve # Start HTTP server for web UI connection +gitnexus list # List all indexed repositories +gitnexus status # Show index status for current repo +gitnexus clean # Delete index for current repo +gitnexus clean --all --force # Delete all indexes +gitnexus wiki [path] # Generate repository wiki from knowledge graph +gitnexus wiki --model # Wiki with custom LLM model (default: gpt-4o-mini) +gitnexus wiki --base-url # Wiki with custom LLM API base URL ``` ### What Your AI Agent Gets **7 tools** exposed via MCP: -| Tool | What It Does | `repo` Param | -|------|-------------|--------------| -| `list_repos` | Discover all indexed repositories | โ€” | -| `search` | Hybrid search (BM25 + semantic) with cluster context | Optional | -| `overview` | List all clusters and processes | Optional | -| `explore` | Deep dive on a symbol, cluster, or process | Optional | -| `impact` | Blast radius analysis with confidence filtering | Optional | -| `cypher` | Raw Cypher graph queries | Optional | -| `analyze` | Index or re-index a repository | Optional | +| Tool | What It Does | `repo` Param | +| ------------------ | ----------------------------------------------------------------- | -------------- | +| `list_repos` | Discover all indexed repositories | โ€” | +| `query` | Process-grouped hybrid search (BM25 + semantic + RRF) | Optional | +| `context` | 360-degree symbol view โ€” categorized refs, process participation | Optional | +| `impact` | Blast radius analysis with depth grouping and confidence | Optional | +| `detect_changes` | Git-diff impact โ€” maps changed lines to affected processes | Optional | +| `rename` | Multi-file coordinated rename with graph + text search | Optional | +| `cypher` | Raw Cypher graph queries | Optional | -> When only one repo is indexed, the `repo` parameter is optional. With multiple repos, specify which one: `search({query: "auth", repo: "my-app"})`. +> When only one repo is indexed, the `repo` parameter is optional. With multiple repos, specify which one: `query({query: "auth", repo: "my-app"})`. **Resources** for instant context: -| Resource | Purpose | -|----------|---------| -| `gitnexus://repos` | List all indexed repositories (read this first) | -| `gitnexus://repo/{name}/context` | Codebase stats and overview | -| `gitnexus://repo/{name}/clusters` | All functional clusters with cohesion scores | -| `gitnexus://repo/{name}/cluster/{name}` | Cluster members and details | -| `gitnexus://repo/{name}/processes` | All execution flows | -| `gitnexus://repo/{name}/process/{name}` | Full process trace with steps | -| `gitnexus://repo/{name}/schema` | Graph schema for Cypher queries | +| Resource | Purpose | +| ----------------------------------------- | ---------------------------------------------------- | +| `gitnexus://repos` | List all indexed repositories (read this first) | +| `gitnexus://repo/{name}/context` | Codebase stats, staleness check, and available tools | +| `gitnexus://repo/{name}/clusters` | All functional clusters with cohesion scores | +| `gitnexus://repo/{name}/cluster/{name}` | Cluster members and details | +| `gitnexus://repo/{name}/processes` | All execution flows | +| `gitnexus://repo/{name}/process/{name}` | Full process trace with steps | +| `gitnexus://repo/{name}/schema` | Graph schema for Cypher queries | + +**2 MCP prompts** for guided workflows: + +| Prompt | What It Does | +| ----------------- | ------------------------------------------------------------------------- | +| `detect_impact` | Pre-commit change analysis โ€” scope, affected processes, risk level | +| `generate_map` | Architecture documentation from the knowledge graph with mermaid diagrams | **4 agent skills** installed to `.claude/skills/` automatically: @@ -218,6 +229,7 @@ The web UI uses the same indexing pipeline as the CLI but runs entirely in WebAs Tools like **Cursor**, **Claude Code**, **Cline**, **Roo Code**, and **Windsurf** are powerful โ€” but they don't truly know your codebase structure. **What happens:** + 1. AI edits `UserService.validate()` 2. Doesn't know 47 functions depend on its return type 3. **Breaking changes ship** @@ -316,24 +328,24 @@ TypeScript, JavaScript, Python, Java, C, C++, C#, Go, Rust GitNexus doesn't just string-match import paths. It understands language-specific module systems: -| Language | What's Resolved | -|----------|----------------| -| **TypeScript** | Path aliases from `tsconfig.json` (e.g. `@/lib/auth` โ†’ `src/lib/auth`) | -| **Rust** | Module paths (`crate::auth::validate`, `super::utils`, `self::handler`) | -| **Java** | Wildcard imports (`com.example.*`) and static imports | -| **Go** | Module paths via `go.mod`, internal package resolution | -| **C/C++** | Relative includes, system include detection | +| Language | What's Resolved | +| -------------------- | ----------------------------------------------------------------------------- | +| **TypeScript** | Path aliases from `tsconfig.json` (e.g. `@/lib/auth` -> `src/lib/auth`) | +| **Rust** | Module paths (`crate::auth::validate`, `super::utils`, `self::handler`) | +| **Java** | Wildcard imports (`com.example.*`) and static imports | +| **Go** | Module paths via `go.mod`, internal package resolution | +| **C/C++** | Relative includes, system include detection | ### Confidence Scoring on CALLS Every function call edge includes a trust score: -| Confidence | Reason | Meaning | -|------------|--------|---------| -| 0.90 | `import-resolved` | Target found in imported file | -| 0.85 | `same-file` | Target defined in same file | -| 0.50 | `fuzzy-global` (1 match) | Single global match by name | -| 0.30 | `fuzzy-global` (N matches) | Multiple matches, first picked | +| Confidence | Reason | Meaning | +| ---------- | ---------------------------- | ------------------------------ | +| 0.90 | `import-resolved` | Target found in imported file | +| 0.85 | `same-file` | Target defined in same file | +| 0.50 | `fuzzy-global` (1 match) | Single global match by name | +| 0.30 | `fuzzy-global` (N matches) | Multiple matches, first picked | The `impact` tool uses `minConfidence` to filter out guesses and return only reliable results. @@ -394,6 +406,7 @@ flowchart TD ``` **Framework detection** boosts scoring for known patterns: + - Next.js: `/pages/`, `/app/page.tsx`, `/api/` - Express: `/routes/`, `/handlers/` - Django: `views.py`, `urls.py` @@ -406,16 +419,16 @@ flowchart TD ### Node Types -| Label | Description | Key Properties | -|-------|-------------|----------------| -| `Folder` | Directory | `name`, `filePath` | -| `File` | Source file | `name`, `filePath`, `language`, `content` | -| `Function` | Function def | `name`, `filePath`, `startLine`, `endLine`, `isExported` | -| `Class` | Class def | `name`, `filePath`, `startLine`, `endLine`, `isExported` | -| `Interface` | Interface def | `name`, `filePath`, `startLine`, `endLine`, `isExported` | -| `Method` | Class method | `name`, `filePath`, `startLine`, `endLine`, `isExported` | -| `Community` | Functional cluster | `label`, `heuristicLabel`, `cohesion`, `symbolCount` | -| `Process` | Execution flow | `label`, `processType`, `stepCount`, `entryPointId` | +| Label | Description | Key Properties | +| ------------- | ------------------ | ------------------------------------------------------------------ | +| `Folder` | Directory | `name`, `filePath` | +| `File` | Source file | `name`, `filePath`, `content` | +| `Function` | Function def | `name`, `filePath`, `startLine`, `endLine`, `isExported` | +| `Class` | Class def | `name`, `filePath`, `startLine`, `endLine`, `isExported` | +| `Interface` | Interface def | `name`, `filePath`, `startLine`, `endLine`, `isExported` | +| `Method` | Class method | `name`, `filePath`, `startLine`, `endLine`, `isExported` | +| `Community` | Functional cluster | `label`, `heuristicLabel`, `cohesion`, `symbolCount` | +| `Process` | Execution flow | `label`, `processType`, `stepCount`, `entryPointId` | Plus language-specific nodes: `Struct`, `Enum`, `Trait`, `Impl`, `TypeAlias`, `Namespace`, `Record`, `Delegate`, `Annotation`, `Constructor`, `Template`, `Module` and more. @@ -423,16 +436,16 @@ Plus language-specific nodes: `Struct`, `Enum`, `Trait`, `Impl`, `TypeAlias`, `N Single edge table with `type` property: -| Type | From | To | Properties | -|------|------|-----|------------| -| `CONTAINS` | Folder | File/Folder | โ€” | -| `DEFINES` | File | Function/Class/etc | โ€” | -| `IMPORTS` | File | File | โ€” | -| `CALLS` | Function/Method | Function/Method | `confidence`, `reason` | -| `EXTENDS` | Class | Class | โ€” | -| `IMPLEMENTS` | Class | Interface | โ€” | -| `MEMBER_OF` | Symbol | Community | โ€” | -| `STEP_IN_PROCESS` | Symbol | Process | `step` (1-indexed) | +| Type | From | To | Properties | +| ------------------- | --------------- | ------------------ | -------------------------- | +| `CONTAINS` | Folder | File/Folder | โ€” | +| `DEFINES` | File | Function/Class/etc | โ€” | +| `IMPORTS` | File | File | โ€” | +| `CALLS` | Function/Method | Function/Method | `confidence`, `reason` | +| `EXTENDS` | Class | Class | โ€” | +| `IMPLEMENTS` | Class | Interface | โ€” | +| `MEMBER_OF` | Symbol | Community | โ€” | +| `STEP_IN_PROCESS` | Symbol | Process | `step` (1-indexed) | --- @@ -441,66 +454,102 @@ Single edge table with `type` property: ### Impact Analysis ``` -gitnexus_impact({target: "UserService", direction: "upstream", minConfidence: 0.8, repo: "my-app"}) +impact({target: "UserService", direction: "upstream", minConfidence: 0.8}) TARGET: Class UserService (src/services/user.ts) UPSTREAM (what depends on this): - Depth 1 (direct callers): - handleLogin [CALLS 90%] โ†’ src/api/auth.ts:45 - handleRegister [CALLS 90%] โ†’ src/api/auth.ts:78 - UserController [CALLS 85%] โ†’ src/controllers/user.ts:12 - Depth 2: - authRouter [IMPORTS] โ†’ src/routes/auth.ts - -8 files affected, 3 clusters touched + Depth 1 (WILL BREAK): + handleLogin [CALLS 90%] -> src/api/auth.ts:45 + handleRegister [CALLS 90%] -> src/api/auth.ts:78 + UserController [CALLS 85%] -> src/controllers/user.ts:12 + Depth 2 (LIKELY AFFECTED): + authRouter [IMPORTS] -> src/routes/auth.ts ``` Options: `maxDepth`, `minConfidence`, `relationTypes` (`CALLS`, `IMPORTS`, `EXTENDS`, `IMPLEMENTS`), `includeTests` -### Search with Cluster Context +### Process-Grouped Search ``` -gitnexus_search({query: "authentication middleware", repo: "my-app"}) +query({query: "authentication middleware"}) -Results: - [1] Function: validateUser - File: src/auth/validate.ts - Cluster: Authentication - Score: 0.042 (hybrid: bm25 + semantic) - [2] Function: checkSession - File: src/auth/session.ts - Cluster: Authentication - Score: 0.038 +processes: + - summary: "LoginFlow" + priority: 0.042 + symbol_count: 4 + process_type: cross_community + step_count: 7 + +process_symbols: + - name: validateUser + type: Function + filePath: src/auth/validate.ts + process_id: proc_login + step_index: 2 + +definitions: + - name: AuthConfig + type: Interface + filePath: src/types/auth.ts ``` -Each result includes the functional cluster it belongs to and optional relationship connections with `depth=full`. - -### Explore a Process +### Context (360-degree Symbol View) ``` -gitnexus_explore({name: "LoginFlow", type: "process", repo: "my-app"}) +context({name: "validateUser"}) -PROCESS: LoginFlow -Type: cross_community | Steps: 7 +symbol: + uid: "Function:validateUser" + kind: Function + filePath: src/auth/validate.ts + startLine: 15 -TRACE: - 1. handleLogin (API) - 2. validateUser (Authentication) - 3. checkRateLimit (RateLimiting) - 4. hashPassword (Authentication) - 5. createSession (Authentication) - 6. storeSession (Database) - 7. generateToken (Authentication) +incoming: + calls: [handleLogin, handleRegister, UserController] + imports: [authRouter] -CLUSTERS TOUCHED: API, Authentication, RateLimiting, Database +outgoing: + calls: [checkPassword, createSession] + +processes: + - name: LoginFlow (step 2/7) + - name: RegistrationFlow (step 3/5) +``` + +### Detect Changes (Pre-Commit) + +``` +detect_changes({scope: "all"}) + +summary: + changed_count: 12 + affected_count: 3 + changed_files: 4 + risk_level: medium + +changed_symbols: [validateUser, AuthService, ...] +affected_processes: [LoginFlow, RegistrationFlow, ...] +``` + +### Rename (Multi-File) + +``` +rename({symbol_name: "validateUser", new_name: "verifyUser", dry_run: true}) + +status: success +files_affected: 5 +total_edits: 8 +graph_edits: 6 (high confidence) +text_search_edits: 2 (review carefully) +changes: [...] ``` ### Cypher Queries ```cypher -- Find what calls auth functions with high confidence -MATCH (c:Community {label: 'Authentication'})<-[:CodeRelation {type: 'MEMBER_OF'}]-(fn) +MATCH (c:Community {heuristicLabel: 'Authentication'})<-[:CodeRelation {type: 'MEMBER_OF'}]-(fn) MATCH (caller)-[r:CodeRelation {type: 'CALLS'}]->(fn) WHERE r.confidence > 0.8 RETURN caller.name, fn.name, r.confidence @@ -509,20 +558,40 @@ ORDER BY r.confidence DESC --- +## Wiki Generation + +Generate LLM-powered documentation from your knowledge graph: + +```bash +# Requires an LLM API key (OPENAI_API_KEY, etc.) +gitnexus wiki + +# Use a custom model or provider +gitnexus wiki --model gpt-4o +gitnexus wiki --base-url https://api.anthropic.com/v1 + +# Force full regeneration +gitnexus wiki --force +``` + +The wiki generator reads the indexed graph structure, groups files into modules via LLM, generates per-module documentation pages, and creates an overview page โ€” all with cross-references to the knowledge graph. + +--- + ## Tech Stack -| Layer | CLI | Web | -|-------|-----|-----| -| **Runtime** | Node.js (native) | Browser (WASM) | -| **Parsing** | Tree-sitter native bindings | Tree-sitter WASM | -| **Database** | KuzuDB native | KuzuDB WASM | -| **Embeddings** | HuggingFace transformers | transformers.js (WebGPU/WASM) | -| **Search** | BM25 + semantic + RRF | BM25 + semantic + RRF | -| **Agent Interface** | MCP (stdio) | LangChain ReAct agent | -| **Visualization** | โ€” | Sigma.js + Graphology (WebGL) | -| **Frontend** | โ€” | React 18, TypeScript, Vite, Tailwind v4 | -| **Clustering** | Graphology + Leiden | Graphology + Leiden | -| **Concurrency** | Node.js async | Web Workers + Comlink | +| Layer | CLI | Web | +| ------------------------- | ------------------------------------- | --------------------------------------- | +| **Runtime** | Node.js (native) | Browser (WASM) | +| **Parsing** | Tree-sitter native bindings | Tree-sitter WASM | +| **Database** | KuzuDB native | KuzuDB WASM | +| **Embeddings** | HuggingFace transformers.js (GPU/CPU) | transformers.js (WebGPU/WASM) | +| **Search** | BM25 + semantic + RRF | BM25 + semantic + RRF | +| **Agent Interface** | MCP (stdio) | LangChain ReAct agent | +| **Visualization** | โ€” | Sigma.js + Graphology (WebGL) | +| **Frontend** | โ€” | React 18, TypeScript, Vite, Tailwind v4 | +| **Clustering** | Graphology + Leiden | Graphology + Leiden | +| **Concurrency** | Worker threads + async | Web Workers + Comlink | --- @@ -532,24 +601,28 @@ ORDER BY r.confidence DESC - [ ] **LLM Cluster Enrichment** โ€” Semantic cluster names via LLM API - [ ] **AST Decorator Detection** โ€” Parse @Controller, @Get, etc. -- [ ] **Diff-Aware Impact** โ€” Show blast radius for uncommitted changes +- [ ] **Incremental Indexing** โ€” Only re-index changed files ### Recently Completed -- [x] **Multi-Repo MCP** โ€” Global registry + lazy connection pool, one MCP server for all repos -- [x] **Zero-Config Setup** โ€” `gitnexus setup` auto-configures Cursor, Claude Code, OpenCode -- [x] **Unified CLI + MCP** โ€” `npm install -g gitnexus` for indexing and MCP server -- [x] **Language-Aware Imports** โ€” TS path aliases, Rust modules, Java wildcards, Go packages -- [x] **Community Detection** โ€” Leiden algorithm for functional clustering -- [x] **Process Detection** โ€” Entry point tracing with framework awareness -- [x] **9 Language Support** โ€” TypeScript, JavaScript, Python, Java, C, C++, C#, Go, Rust -- [x] **Confidence Scoring** โ€” Trust levels on CALLS edges (0.3-0.9) -- [x] **Blast Radius Tool** โ€” `impact` with minConfidence, relationTypes, includeTests -- [x] **Hybrid Search** โ€” BM25 + semantic + Reciprocal Rank Fusion + cluster context -- [x] **Browser Embeddings** โ€” snowflake-arctic-embed-xs (22M params) -- [x] **Vector Index** โ€” HNSW in KuzuDB for semantic search -- [x] **Grounded Citations** โ€” `[[file:line]]` format in AI responses -- [x] **Multiple LLM Providers** โ€” OpenAI, Azure, Gemini, Anthropic, Ollama +- [X] **Wiki Generation** โ€” LLM-powered docs from knowledge graph (`gitnexus wiki`) +- [X] **Multi-File Rename** โ€” Graph-aware rename with confidence tags (`rename` tool) +- [X] **Git-Diff Impact** โ€” Pre-commit change analysis (`detect_changes` tool) +- [X] **Process-Grouped Search** โ€” Query results grouped by execution flow (`query` tool) +- [X] **360-Degree Context** โ€” Categorized refs + process participation (`context` tool) +- [X] **Claude Code Hooks** โ€” Auto-augment grep/glob with graph context +- [X] **MCP Prompts** โ€” Guided workflows for impact detection and architecture docs +- [X] **Multi-Repo MCP** โ€” Global registry + lazy connection pool, one MCP server for all repos +- [X] **Zero-Config Setup** โ€” `gitnexus setup` auto-configures Cursor, Claude Code, OpenCode +- [X] **Unified CLI + MCP** โ€” `npm install -g gitnexus` for indexing and MCP server +- [X] **Language-Aware Imports** โ€” TS path aliases, Rust modules, Java wildcards, Go packages +- [X] **Community Detection** โ€” Leiden algorithm for functional clustering +- [X] **Process Detection** โ€” Entry point tracing with framework awareness +- [X] **9 Language Support** โ€” TypeScript, JavaScript, Python, Java, C, C++, C#, Go, Rust +- [X] **Confidence Scoring** โ€” Trust levels on CALLS edges (0.3-0.9) +- [X] **Blast Radius Tool** โ€” `impact` with minConfidence, relationTypes, includeTests +- [X] **Hybrid Search** โ€” BM25 + semantic + Reciprocal Rank Fusion +- [X] **Vector Index** โ€” HNSW in KuzuDB for semantic search --- diff --git a/eval/.env.example b/eval/.env.example new file mode 100644 index 0000000000..7942d8f02c --- /dev/null +++ b/eval/.env.example @@ -0,0 +1,23 @@ +# โ”€โ”€โ”€ GitNexus SWE-bench Eval โ€” API Keys โ”€โ”€โ”€ +# Copy this file to .env and fill in the keys you have. +# You only need keys for the models you plan to test. + +# OpenRouter (covers Claude, MiniMax, GLM, and 200+ other models) +# Get yours at: https://openrouter.ai/keys +OPENROUTER_API_KEY= + +# Anthropic (direct โ€” optional if using OpenRouter) +# Get yours at: https://console.anthropic.com/ +ANTHROPIC_API_KEY= + +# ZhipuAI / GLM (direct โ€” optional if using OpenRouter) +# Get yours at: https://open.bigmodel.cn/ +ZHIPUAI_API_KEY= + +# MiniMax (direct โ€” optional if using OpenRouter) +MINIMAX_API_KEY= + +# โ”€โ”€โ”€ Optional โ”€โ”€โ”€ + +# Cost tracking: set to "ignore_errors" if litellm can't find pricing for a model +# MSWEA_COST_TRACKING=ignore_errors diff --git a/eval/.gitignore b/eval/.gitignore new file mode 100644 index 0000000000..d1ac9f241a --- /dev/null +++ b/eval/.gitignore @@ -0,0 +1,16 @@ +# Evaluation results (large, should not be committed) +results/ +*.traj.json +preds.json + +# Python +__pycache__/ +*.pyc +*.egg-info/ +.eggs/ +dist/ +build/ + +# Environment +.env +.venv/ diff --git a/eval/README.md b/eval/README.md new file mode 100644 index 0000000000..d268e67b05 --- /dev/null +++ b/eval/README.md @@ -0,0 +1,210 @@ +# GitNexus SWE-bench Evaluation Harness + +Evaluate whether GitNexus code intelligence improves AI agent performance on real software engineering tasks. Runs SWE-bench instances across multiple models and compares baseline (no graph) vs GitNexus-enhanced configurations. + +## What This Tests + +**Hypothesis**: Giving AI agents structural code intelligence (call graphs, execution flows, blast radius analysis) improves their ability to resolve real GitHub issues โ€” measured by resolve rate, cost, and efficiency. + +**Evaluation modes:** + +| Mode | What the agent gets | +|------|-------------------| +| `baseline` | Standard bash tools (grep, find, cat, sed) โ€” control group | +| `native` | Baseline + explicit GitNexus tools via eval-server (~100ms) | +| `native_augment` | Native tools + grep results automatically enriched with graph context (**recommended**) | + +> **Recommended**: Use `native_augment` mode. It mirrors the Claude Code model โ€” the agent gets both explicit GitNexus tools (fast bash commands) AND automatic enrichment of grep results with callers, callees, and execution flows. The agent decides when to use explicit tools vs rely on enriched search output. + +**Models supported:** + +- Claude 3.5 Haiku, Claude Sonnet 4, Claude Opus 4 +- MiniMax M1 2.5 +- GLM 4.7, GLM 5 +- Any model supported by litellm (add a YAML config) + +## Prerequisites + +- Python 3.11+ +- Docker (for SWE-bench containers) +- Node.js 18+ (for GitNexus) +- API keys for your chosen models + +## Setup + +```bash +cd eval + +# Install dependencies +pip install -e . + +# Set up API keys โ€” copy the template and fill in your keys +cp .env.example .env +# Then edit .env and paste your key(s) +``` + +All models are routed through **OpenRouter** by default, so a single `OPENROUTER_API_KEY` is all you need. To use provider APIs directly (Anthropic, ZhipuAI, etc.), edit the model YAML in `configs/models/` and set the corresponding key in `.env`. + +```bash +# Pull SWE-bench Docker images (pulled on-demand, but you can pre-pull) +docker pull swebench/sweb.eval.x86_64.django_1776_django-16527:latest +``` + +## Quick Start + +### Debug a single instance + +```bash +# Fastest way to verify everything works +python run_eval.py debug -m claude-haiku -i django__django-16527 --subset lite +``` + +### Run a single configuration + +```bash +# 5 instances, Claude Sonnet, native_augment mode (default) +python run_eval.py single -m claude-sonnet --subset lite --slice 0:5 + +# Baseline comparison (no GitNexus) +python run_eval.py single -m claude-sonnet --mode baseline --subset lite --slice 0:5 + +# Full Lite benchmark, 4 parallel workers +python run_eval.py single -m claude-sonnet --subset lite -w 4 +``` + +### Run the full matrix + +```bash +# All models x all modes +python run_eval.py matrix --subset lite -w 4 + +# Key comparison: baseline vs native_augment +python run_eval.py matrix -m claude-sonnet -m claude-haiku --modes baseline --modes native_augment --subset lite --slice 0:50 +``` + +### Analyze results + +```bash +# Summary table +python -m analysis.analyze_results results/ + +# Compare modes for a specific model +python -m analysis.analyze_results compare-modes results/ -m claude-sonnet + +# GitNexus tool usage analysis +python -m analysis.analyze_results gitnexus-usage results/ + +# Export as CSV for further analysis +python -m analysis.analyze_results summary results/ --format csv > results.csv + +# Run official SWE-bench test evaluation +python -m analysis.analyze_results summary results/ --swebench-eval +``` + +### List available configurations + +```bash +python run_eval.py list-configs +``` + +## Architecture + +``` +eval/ + run_eval.py # Main entry point (single, matrix, debug commands) + agents/ + gitnexus_agent.py # GitNexusAgent: extends DefaultAgent with augmentation + metrics + environments/ + gitnexus_docker.py # Docker env with GitNexus + eval-server + standalone tool scripts + bridge/ + gitnexus_tools.sh # Bash wrappers (legacy โ€” now standalone scripts are installed directly) + mcp_bridge.py # Legacy MCP bridge (kept for reference) + prompts/ + system_baseline.jinja # System: persona + format rules + instance_baseline.jinja # Instance: task + workflow + system_native.jinja # System: + GitNexus tool reference + instance_native.jinja # Instance: + GitNexus debugging workflow + system_native_augment.jinja # System: + GitNexus tools + grep enrichment docs + instance_native_augment.jinja # Instance: + GitNexus workflow + risk assessment + configs/ + models/ # Per-model YAML configs + modes/ # Per-mode YAML configs (baseline, native, native_augment) + analysis/ + analyze_results.py # Post-run comparative analysis + results/ # Output directory (gitignored) +``` + +## How It Works + +### Template structure + +mini-swe-agent requires two Jinja templates: +- **system_template** โ†’ system message: persona, format rules, tool reference (static) +- **instance_template** โ†’ first user message: task, workflow, rules, examples (contains `{{task}}`) + +Each mode has a `system_{mode}.jinja` + `instance_{mode}.jinja` pair. The agent loads both automatically based on the configured mode. + +### Per-instance flow + +1. Docker container starts with SWE-bench instance (repo at specific commit) +2. **GitNexus setup**: Node.js + gitnexus installed, `gitnexus analyze` runs (or restores from cache) +3. **Eval-server starts**: `gitnexus eval-server` daemon (persistent HTTP server, keeps KuzuDB warm) +4. **Standalone tool scripts installed** in `/usr/local/bin/` โ€” works with `subprocess.run` (no `.bashrc` needed) +5. Agent runs with the configured model + system prompt + GitNexus tools +6. Agent's patch is extracted as a git diff +7. Metrics collected: cost, tokens, tool calls, GitNexus usage, augmentation stats + +### Tool architecture + +``` +Agent โ†’ bash command โ†’ /usr/local/bin/gitnexus-query + โ†’ curl localhost:4848/tool/query (fast path: eval-server, ~100ms) + โ†’ npx gitnexus query (fallback: cold CLI, ~5-10s) +``` + +Each tool script in `/usr/local/bin/` is standalone โ€” no sourcing, no env inheritance needed. This is critical because mini-swe-agent runs every command via `subprocess.run` in a fresh subshell. + +### Eval-server + +The eval-server is a lightweight HTTP daemon that: +- Keeps KuzuDB warm in memory (no cold start per tool call) +- Returns LLM-friendly text (not raw JSON โ€” saves tokens) +- Includes next-step hints to guide tool chaining (query โ†’ context โ†’ impact โ†’ fix) +- Auto-shuts down after idle timeout + +### Index caching + +SWE-bench repos repeat (Django has 200+ instances at different commits). The harness caches GitNexus indexes per `(repo, commit)` hash in `~/.gitnexus-eval-cache/` to avoid redundant re-indexing. + +### Grep augmentation (native_augment mode) + +When the agent runs `grep` or `rg`, the observation is post-processed: the agent class calls `gitnexus-augment` on the search pattern and appends `[GitNexus]` annotations showing callers, callees, and execution flows for matched symbols. This mirrors the Claude Code / Cursor hook integration. + +## Adding Models + +Create a YAML file in `configs/models/`: + +```yaml +# configs/models/my-model.yaml +model: + model_name: "openrouter/provider/model-name" + cost_tracking: "ignore_errors" # if not in litellm's cost DB + model_kwargs: + max_tokens: 8192 + temperature: 0 +``` + +The model name follows [litellm conventions](https://docs.litellm.ai/docs/providers). + +## Metrics Collected + +| Metric | Description | +|--------|-------------| +| Patch Rate | % of instances where agent produced a patch | +| Resolve Rate | % of instances where patch passes tests (requires --swebench-eval) | +| Total Cost | API cost across all instances | +| Avg Cost/Instance | Cost efficiency | +| API Calls | Number of LLM calls | +| GN Tool Calls | How many GitNexus tools the agent used | +| Augment Hits | How many grep/find results got enriched | +| Augment Hit Rate | % of search commands that got useful enrichment | diff --git a/eval/__init__.py b/eval/__init__.py new file mode 100644 index 0000000000..fb4a4032a8 --- /dev/null +++ b/eval/__init__.py @@ -0,0 +1 @@ +# GitNexus SWE-bench Evaluation Harness diff --git a/eval/agents/__init__.py b/eval/agents/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/eval/agents/gitnexus_agent.py b/eval/agents/gitnexus_agent.py new file mode 100644 index 0000000000..a419ba1e6f --- /dev/null +++ b/eval/agents/gitnexus_agent.py @@ -0,0 +1,209 @@ +""" +GitNexus-Enhanced Agent for SWE-bench Evaluation + +Extends mini-swe-agent's DefaultAgent with: +1. Native augment mode: GitNexus tools via eval-server + grep enrichment (recommended) +2. Native mode: GitNexus tools via eval-server only +3. Baseline mode: Pure mini-swe-agent (no GitNexus โ€” control group) + +The agent class itself is minimal โ€” the heavy lifting is in: +- Prompt selection (system + instance templates per mode) +- Observation post-processing (grep result augmentation) +- Metrics tracking (which tools the agent actually uses) + +Template structure (matches mini-swe-agent's expectations): + system_template โ†’ system message: persona + format rules + tool reference + instance_template โ†’ first user message: task + workflow + rules + examples +""" + +import logging +import re +import time +from enum import Enum +from pathlib import Path + +from minisweagent import Environment, Model +from minisweagent.agents.default import AgentConfig, DefaultAgent + +logger = logging.getLogger("gitnexus_agent") + +PROMPTS_DIR = Path(__file__).parent.parent / "prompts" + + +class GitNexusMode(str, Enum): + """Evaluation modes for GitNexus integration.""" + BASELINE = "baseline" # No GitNexus โ€” pure mini-swe-agent + NATIVE = "native" # GitNexus tools via eval-server + NATIVE_AUGMENT = "native_augment" # Native tools + grep enrichment (recommended) + + +class GitNexusAgentConfig(AgentConfig): + """Extended config for GitNexus evaluation agent.""" + gitnexus_mode: GitNexusMode = GitNexusMode.BASELINE + augment_timeout: float = 5.0 + augment_min_pattern_length: int = 3 + track_gitnexus_usage: bool = True + + +class GitNexusAgent(DefaultAgent): + """ + Agent that optionally enriches its capabilities with GitNexus code intelligence. + + In BASELINE mode, behaves identically to DefaultAgent. + In NATIVE mode, GitNexus tools are available as bash commands via eval-server. + In NATIVE_AUGMENT mode, GitNexus tools + automatic grep result enrichment. + """ + + def __init__(self, model: Model, env: Environment, *, config_class: type = GitNexusAgentConfig, **kwargs): + mode = kwargs.get("gitnexus_mode", GitNexusMode.BASELINE) + if isinstance(mode, str): + mode = GitNexusMode(mode) + + # Load system template + system_file = PROMPTS_DIR / f"system_{mode.value}.jinja" + if system_file.exists() and "system_template" not in kwargs: + kwargs["system_template"] = system_file.read_text() + + # Load instance template + instance_file = PROMPTS_DIR / f"instance_{mode.value}.jinja" + if instance_file.exists() and "instance_template" not in kwargs: + kwargs["instance_template"] = instance_file.read_text() + + super().__init__(model, env, config_class=config_class, **kwargs) + self.gitnexus_mode = mode + self.gitnexus_metrics = GitNexusMetrics() + + def execute_actions(self, message: dict) -> list[dict]: + """Execute actions with optional GitNexus augmentation and tracking.""" + if self.config.track_gitnexus_usage: + self._track_tool_usage(message) + + outputs = [self.env.execute(action) for action in message.get("extra", {}).get("actions", [])] + + # Augment grep/find observations in NATIVE_AUGMENT mode + if self.gitnexus_mode == GitNexusMode.NATIVE_AUGMENT: + actions = message.get("extra", {}).get("actions", []) + for i, (action, output) in enumerate(zip(actions, outputs)): + augmented = self._maybe_augment(action, output) + if augmented: + outputs[i] = augmented + + return self.add_messages( + *self.model.format_observation_messages(message, outputs, self.get_template_vars()) + ) + + def _maybe_augment(self, action: dict, output: dict) -> dict | None: + """ + If the action is a search command (grep, find, rg, ag), augment the output + with GitNexus knowledge graph context. + """ + command = action.get("command", "") + if not command: + return None + + pattern = self._extract_search_pattern(command) + if not pattern or len(pattern) < self.config.augment_min_pattern_length: + return None + + start = time.time() + try: + augment_result = self.env.execute({ + "command": f'gitnexus-augment "{pattern}" 2>&1 || true', + "timeout": self.config.augment_timeout, + }) + elapsed = time.time() - start + self.gitnexus_metrics.augmentation_calls += 1 + self.gitnexus_metrics.augmentation_time += elapsed + + augment_text = augment_result.get("output", "").strip() + if augment_text and "[GitNexus]" in augment_text: + original_output = output.get("output", "") + output = dict(output) + output["output"] = f"{original_output}\n\n{augment_text}" + self.gitnexus_metrics.augmentation_hits += 1 + return output + except Exception as e: + logger.debug(f"Augmentation failed for pattern '{pattern}': {e}") + self.gitnexus_metrics.augmentation_errors += 1 + + return None + + @staticmethod + def _extract_search_pattern(command: str) -> str | None: + """Extract the search pattern from a grep/find/rg command.""" + patterns = [ + r'(?:grep|rg|ag)\s+(?:-[a-zA-Z]*\s+)*["\']([^"\']+)["\']', + r'(?:grep|rg|ag)\s+(?:-[a-zA-Z]*\s+)*(\S+)', + ] + + for pat in patterns: + match = re.search(pat, command) + if match: + result = match.group(1) + if result.startswith("/") or result.startswith("."): + continue + if result.startswith("-"): + continue + return result + + return None + + def _track_tool_usage(self, message: dict): + """Track which GitNexus tools the agent uses.""" + for action in message.get("extra", {}).get("actions", []): + command = action.get("command", "") + if "gitnexus-query" in command: + self.gitnexus_metrics.tool_calls["query"] += 1 + elif "gitnexus-context" in command: + self.gitnexus_metrics.tool_calls["context"] += 1 + elif "gitnexus-impact" in command: + self.gitnexus_metrics.tool_calls["impact"] += 1 + elif "gitnexus-cypher" in command: + self.gitnexus_metrics.tool_calls["cypher"] += 1 + elif "gitnexus-overview" in command: + self.gitnexus_metrics.tool_calls["overview"] += 1 + + def serialize(self, *extra_dicts) -> dict: + """Serialize with GitNexus-specific metrics.""" + gitnexus_data = { + "info": { + "gitnexus": { + "mode": self.gitnexus_mode.value, + "metrics": self.gitnexus_metrics.to_dict(), + }, + }, + } + return super().serialize(gitnexus_data, *extra_dicts) + + +class GitNexusMetrics: + """Tracks GitNexus-specific metrics during evaluation.""" + + def __init__(self): + self.tool_calls: dict[str, int] = { + "query": 0, + "context": 0, + "impact": 0, + "cypher": 0, + "overview": 0, + } + self.augmentation_calls: int = 0 + self.augmentation_hits: int = 0 + self.augmentation_errors: int = 0 + self.augmentation_time: float = 0.0 + self.index_time: float = 0.0 + + @property + def total_tool_calls(self) -> int: + return sum(self.tool_calls.values()) + + def to_dict(self) -> dict: + return { + "tool_calls": dict(self.tool_calls), + "total_tool_calls": self.total_tool_calls, + "augmentation_calls": self.augmentation_calls, + "augmentation_hits": self.augmentation_hits, + "augmentation_errors": self.augmentation_errors, + "augmentation_time_seconds": round(self.augmentation_time, 2), + "index_time_seconds": round(self.index_time, 2), + } diff --git a/eval/analysis/__init__.py b/eval/analysis/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/eval/analysis/analyze_results.py b/eval/analysis/analyze_results.py new file mode 100644 index 0000000000..69c5f5cdbb --- /dev/null +++ b/eval/analysis/analyze_results.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +""" +Results Analyzer for GitNexus SWE-bench Evaluation + +Reads evaluation results and generates comparative analysis: +- Resolve rate by model x mode +- Cost comparison (total, per-instance) +- Token/API call efficiency +- GitNexus tool usage patterns +- Augmentation hit rates + +Usage: + python -m analysis.analyze_results /path/to/results + python -m analysis.analyze_results /path/to/results --format markdown + python -m analysis.analyze_results /path/to/results --swebench-eval # run actual test verification +""" + +import json +import logging +import os +import subprocess +from pathlib import Path +from typing import Any + +import typer +from rich.console import Console +from rich.table import Table + +logger = logging.getLogger("analyze_results") +console = Console() +app = typer.Typer(rich_markup_mode="rich", add_completion=False) + + +def load_run_results(results_dir: Path) -> dict[str, dict]: + """ + Load all run results from the results directory. + + Returns: {run_id: {summary, preds, instances}} + """ + runs = {} + + for run_dir in sorted(results_dir.iterdir()): + if not run_dir.is_dir(): + continue + + run_id = run_dir.name + run_data: dict[str, Any] = {"run_id": run_id, "dir": run_dir} + + # Load summary + summary_path = run_dir / "summary.json" + if summary_path.exists(): + run_data["summary"] = json.loads(summary_path.read_text()) + + # Load predictions + preds_path = run_dir / "preds.json" + if preds_path.exists(): + run_data["preds"] = json.loads(preds_path.read_text()) + + # Load individual trajectories for detailed metrics + run_data["trajectories"] = {} + for traj_dir in run_dir.iterdir(): + if not traj_dir.is_dir(): + continue + for traj_file in traj_dir.glob("*.traj.json"): + try: + traj = json.loads(traj_file.read_text()) + instance_id = traj.get("instance_id", traj_dir.name) + run_data["trajectories"][instance_id] = traj + except Exception: + pass + + if run_data.get("preds") or run_data.get("summary"): + runs[run_id] = run_data + + return runs + + +def parse_run_id(run_id: str) -> tuple[str, str]: + """Parse 'model_mode' into (model, mode).""" + # Handle multi-word model names like 'minimax-2.5' + # Modes are: baseline, mcp, augment, full + known_modes = {"baseline", "mcp", "augment", "full"} + parts = run_id.rsplit("_", 1) + if len(parts) == 2 and parts[1] in known_modes: + return parts[0], parts[1] + return run_id, "unknown" + + +def compute_metrics(run_data: dict) -> dict: + """Compute evaluation metrics for a single run.""" + preds = run_data.get("preds", {}) + summary = run_data.get("summary", {}) + trajectories = run_data.get("trajectories", {}) + + n_instances = len(preds) + n_with_patch = sum(1 for p in preds.values() if p.get("model_patch", "").strip()) + + # Cost and API call metrics from trajectories + costs = [] + api_calls = [] + gn_tool_calls = [] + gn_augment_hits = [] + gn_augment_calls = [] + + for instance_id, traj in trajectories.items(): + info = traj.get("info", {}) + model_stats = info.get("model_stats", {}) + costs.append(model_stats.get("instance_cost", 0)) + api_calls.append(model_stats.get("api_calls", 0)) + + gn = info.get("gitnexus", {}).get("metrics", {}) + if gn: + gn_tool_calls.append(gn.get("total_tool_calls", 0)) + gn_augment_hits.append(gn.get("augmentation_hits", 0)) + gn_augment_calls.append(gn.get("augmentation_calls", 0)) + + # Also try summary-level metrics + if not costs and summary: + results = summary.get("results", []) + for r in results: + costs.append(r.get("cost", 0)) + api_calls.append(r.get("n_calls", 0)) + gn = r.get("gitnexus_metrics", {}) + if gn: + gn_tool_calls.append(gn.get("total_tool_calls", 0)) + gn_augment_hits.append(gn.get("augmentation_hits", 0)) + gn_augment_calls.append(gn.get("augmentation_calls", 0)) + + total_cost = sum(costs) + total_calls = sum(api_calls) + + return { + "n_instances": n_instances, + "n_with_patch": n_with_patch, + "patch_rate": n_with_patch / max(n_instances, 1), + "total_cost": total_cost, + "avg_cost": total_cost / max(n_instances, 1), + "total_api_calls": total_calls, + "avg_api_calls": total_calls / max(n_instances, 1), + "total_gn_tool_calls": sum(gn_tool_calls), + "avg_gn_tool_calls": sum(gn_tool_calls) / max(len(gn_tool_calls), 1) if gn_tool_calls else 0, + "total_augment_hits": sum(gn_augment_hits), + "total_augment_calls": sum(gn_augment_calls), + "augment_hit_rate": sum(gn_augment_hits) / max(sum(gn_augment_calls), 1) if gn_augment_calls else 0, + } + + +def run_swebench_evaluation(results_dir: Path, run_id: str, subset: str = "lite") -> dict | None: + """ + Run the official SWE-bench evaluation on predictions. + + Requires: pip install swebench + """ + preds_path = results_dir / run_id / "preds.json" + if not preds_path.exists(): + return None + + dataset_mapping = { + "lite": "princeton-nlp/SWE-Bench_Lite", + "verified": "princeton-nlp/SWE-Bench_Verified", + "full": "princeton-nlp/SWE-Bench", + } + + try: + eval_output = results_dir / run_id / "swebench_eval" + cmd = [ + "python", "-m", "swebench.harness.run_evaluation", + "--dataset_name", dataset_mapping.get(subset, subset), + "--predictions_path", str(preds_path), + "--max_workers", "4", + "--run_id", run_id, + "--output_dir", str(eval_output), + ] + + logger.info(f"Running SWE-bench evaluation for {run_id}...") + result = subprocess.run(cmd, capture_output=True, text=True, timeout=600) + + if result.returncode == 0: + # Parse evaluation results + report_path = eval_output / run_id / "results.json" + if report_path.exists(): + return json.loads(report_path.read_text()) + + logger.error(f"SWE-bench eval failed: {result.stderr[:500]}") + return None + + except Exception as e: + logger.error(f"SWE-bench eval error: {e}") + return None + + +# โ”€โ”€โ”€ CLI Commands โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +@app.command() +def summary( + results_dir: str = typer.Argument(..., help="Path to results directory"), + format: str = typer.Option("table", "--format", help="Output format: table, markdown, json, csv"), + swebench_eval: bool = typer.Option(False, "--swebench-eval", help="Run official SWE-bench test evaluation"), + subset: str = typer.Option("lite", "--subset", help="SWE-bench subset (for --swebench-eval)"), +): + """Generate comparative analysis of evaluation results.""" + results_path = Path(results_dir) + if not results_path.exists(): + console.print(f"[red]Results directory not found: {results_path}[/red]") + raise typer.Exit(1) + + runs = load_run_results(results_path) + if not runs: + console.print("[yellow]No evaluation results found[/yellow]") + raise typer.Exit(0) + + console.print(f"\n[bold]Found {len(runs)} evaluation runs[/bold]\n") + + # Compute metrics per run + all_metrics = {} + for run_id, run_data in runs.items(): + model, mode = parse_run_id(run_id) + metrics = compute_metrics(run_data) + metrics["model"] = model + metrics["mode"] = mode + + # Optionally run SWE-bench evaluation + if swebench_eval: + eval_result = run_swebench_evaluation(results_path, run_id, subset) + if eval_result: + metrics["resolved"] = eval_result.get("resolved", 0) + metrics["resolve_rate"] = eval_result.get("resolved", 0) / max(metrics["n_instances"], 1) + + all_metrics[run_id] = metrics + + if format == "table": + _print_table(all_metrics) + elif format == "markdown": + _print_markdown(all_metrics) + elif format == "json": + console.print(json.dumps(all_metrics, indent=2)) + elif format == "csv": + _print_csv(all_metrics) + + +@app.command() +def compare_modes( + results_dir: str = typer.Argument(..., help="Path to results directory"), + model: str = typer.Option(..., "-m", "--model", help="Model to compare across modes"), +): + """Compare modes for a specific model (baseline vs mcp vs augment vs full).""" + results_path = Path(results_dir) + runs = load_run_results(results_path) + + # Filter to the specified model + model_runs = { + run_id: data for run_id, data in runs.items() + if parse_run_id(run_id)[0] == model + } + + if not model_runs: + console.print(f"[yellow]No results found for model: {model}[/yellow]") + raise typer.Exit(1) + + console.print(f"\n[bold]Mode comparison for {model}[/bold]\n") + + metrics = {} + for run_id, run_data in model_runs.items(): + _, mode = parse_run_id(run_id) + metrics[mode] = compute_metrics(run_data) + + # Print comparison table + table = Table(title=f"Mode Comparison: {model}") + table.add_column("Metric", style="bold") + for mode in ["baseline", "mcp", "augment", "full"]: + if mode in metrics: + table.add_column(mode, justify="right") + + rows = [ + ("Instances", "n_instances", "d"), + ("With Patch", "n_with_patch", "d"), + ("Patch Rate", "patch_rate", ".1%"), + ("Total Cost", "total_cost", "$.4f"), + ("Avg Cost", "avg_cost", "$.4f"), + ("Total API Calls", "total_api_calls", "d"), + ("Avg API Calls", "avg_api_calls", ".1f"), + ("GN Tool Calls", "total_gn_tool_calls", "d"), + ("Augment Hits", "total_augment_hits", "d"), + ("Augment Hit Rate", "augment_hit_rate", ".1%"), + ] + + for label, key, fmt in rows: + values = [] + for mode in ["baseline", "mcp", "augment", "full"]: + if mode in metrics: + v = metrics[mode].get(key, 0) + if fmt == ".1%": + values.append(f"{v:.1%}") + elif fmt == "$.4f": + values.append(f"${v:.4f}") + elif fmt == ".1f": + values.append(f"{v:.1f}") + else: + values.append(str(v)) + table.add_row(label, *values) + + # Add delta rows (improvement over baseline) + if "baseline" in metrics: + baseline_cost = metrics["baseline"]["avg_cost"] + baseline_calls = metrics["baseline"]["avg_api_calls"] + + table.add_section() + for mode in ["mcp", "augment", "full"]: + if mode not in metrics: + continue + mode_cost = metrics[mode]["avg_cost"] + mode_calls = metrics[mode]["avg_api_calls"] + + cost_delta = ((mode_cost - baseline_cost) / max(baseline_cost, 0.001)) * 100 + calls_delta = ((mode_calls - baseline_calls) / max(baseline_calls, 1)) * 100 + + cost_str = f"{cost_delta:+.1f}%" + calls_str = f"{calls_delta:+.1f}%" + + # Color-code: negative is good (cheaper/fewer calls) + cost_color = "green" if cost_delta < 0 else "red" + calls_color = "green" if calls_delta < 0 else "red" + + console.print(f" {mode} vs baseline: cost [{cost_color}]{cost_str}[/{cost_color}], calls [{calls_color}]{calls_str}[/{calls_color}]") + + console.print(table) + + +@app.command() +def gitnexus_usage( + results_dir: str = typer.Argument(..., help="Path to results directory"), +): + """Analyze GitNexus tool usage patterns across all runs.""" + results_path = Path(results_dir) + runs = load_run_results(results_path) + + console.print("\n[bold]GitNexus Tool Usage Analysis[/bold]\n") + + table = Table(title="Tool Usage by Run") + table.add_column("Run", style="bold") + table.add_column("query", justify="right") + table.add_column("context", justify="right") + table.add_column("impact", justify="right") + table.add_column("cypher", justify="right") + table.add_column("Total", justify="right") + table.add_column("Augment Hits", justify="right") + + for run_id, run_data in sorted(runs.items()): + _, mode = parse_run_id(run_id) + if mode == "baseline": + continue + + # Aggregate tool calls across trajectories + tool_totals: dict[str, int] = {"query": 0, "context": 0, "impact": 0, "cypher": 0, "overview": 0} + augment_hits = 0 + + for traj in run_data.get("trajectories", {}).values(): + gn = traj.get("info", {}).get("gitnexus", {}).get("metrics", {}) + for tool, count in gn.get("tool_calls", {}).items(): + tool_totals[tool] = tool_totals.get(tool, 0) + count + augment_hits += gn.get("augmentation_hits", 0) + + # Also check summary + for r in run_data.get("summary", {}).get("results", []): + gn = r.get("gitnexus_metrics", {}) + for tool, count in gn.get("tool_calls", {}).items(): + tool_totals[tool] = tool_totals.get(tool, 0) + count + augment_hits += gn.get("augmentation_hits", 0) + + total = sum(tool_totals.values()) + if total > 0 or augment_hits > 0: + table.add_row( + run_id, + str(tool_totals.get("query", 0)), + str(tool_totals.get("context", 0)), + str(tool_totals.get("impact", 0)), + str(tool_totals.get("cypher", 0)), + str(total), + str(augment_hits), + ) + + console.print(table) + + +# โ”€โ”€โ”€ Output Formatters โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def _print_table(all_metrics: dict): + """Print rich table summary.""" + table = Table(title="Evaluation Results") + table.add_column("Run", style="bold") + table.add_column("Model") + table.add_column("Mode") + table.add_column("N", justify="right") + table.add_column("Patched", justify="right") + table.add_column("Rate", justify="right") + table.add_column("Cost", justify="right") + table.add_column("Calls", justify="right") + table.add_column("GN Tools", justify="right") + + for run_id, m in sorted(all_metrics.items()): + resolved_str = "" + if "resolve_rate" in m: + resolved_str = f" ({m['resolve_rate']:.0%})" + + table.add_row( + run_id, + m["model"], + m["mode"], + str(m["n_instances"]), + str(m["n_with_patch"]), + f"{m['patch_rate']:.0%}{resolved_str}", + f"${m['total_cost']:.2f}", + str(m["total_api_calls"]), + str(m["total_gn_tool_calls"]) if m["total_gn_tool_calls"] > 0 else "-", + ) + + console.print(table) + + +def _print_markdown(all_metrics: dict): + """Print markdown table.""" + print("| Run | Model | Mode | N | Patched | Rate | Cost | Calls | GN Tools |") + print("|-----|-------|------|---|---------|------|------|-------|----------|") + for run_id, m in sorted(all_metrics.items()): + gn = str(m["total_gn_tool_calls"]) if m["total_gn_tool_calls"] > 0 else "-" + print(f"| {run_id} | {m['model']} | {m['mode']} | {m['n_instances']} | {m['n_with_patch']} | {m['patch_rate']:.0%} | ${m['total_cost']:.2f} | {m['total_api_calls']} | {gn} |") + + +def _print_csv(all_metrics: dict): + """Print CSV output.""" + print("run_id,model,mode,n_instances,n_with_patch,patch_rate,total_cost,avg_cost,total_api_calls,avg_api_calls,total_gn_tool_calls,total_augment_hits,augment_hit_rate") + for run_id, m in sorted(all_metrics.items()): + print( + f"{run_id},{m['model']},{m['mode']},{m['n_instances']},{m['n_with_patch']}," + f"{m['patch_rate']:.4f},{m['total_cost']:.4f},{m['avg_cost']:.4f}," + f"{m['total_api_calls']},{m['avg_api_calls']:.1f},{m['total_gn_tool_calls']}," + f"{m['total_augment_hits']},{m['augment_hit_rate']:.4f}" + ) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + app() diff --git a/eval/bridge/__init__.py b/eval/bridge/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/eval/bridge/gitnexus_tools.sh b/eval/bridge/gitnexus_tools.sh new file mode 100644 index 0000000000..be926d81f3 --- /dev/null +++ b/eval/bridge/gitnexus_tools.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# GitNexus CLI tool wrappers for SWE-bench evaluation +# +# These functions call the GitNexus eval-server (HTTP daemon) for near-instant +# tool responses. The eval-server keeps KuzuDB warm in memory. +# +# If the eval-server is not running, falls back to direct CLI commands. +# +# Usage: +# gitnexus-query "how does authentication work" +# gitnexus-context "validateUser" +# gitnexus-impact "AuthService" upstream +# gitnexus-cypher "MATCH (n:Function) RETURN n.name LIMIT 10" +# gitnexus-overview + +GITNEXUS_EVAL_PORT="${GITNEXUS_EVAL_PORT:-4848}" +GITNEXUS_EVAL_URL="http://127.0.0.1:${GITNEXUS_EVAL_PORT}" + +_gitnexus_call() { + local tool="$1" + shift + local json_body="$1" + + # Try eval-server first (fastest path โ€” KuzuDB stays warm) + local result + result=$(curl -sf -X POST "${GITNEXUS_EVAL_URL}/tool/${tool}" \ + -H "Content-Type: application/json" \ + -d "${json_body}" 2>/dev/null) + + if [ $? -eq 0 ] && [ -n "$result" ]; then + echo "$result" + return 0 + fi + + # Fallback: direct CLI (cold start, slower but always works) + case "$tool" in + query) + local q=$(echo "$json_body" | python3 -c "import sys,json; print(json.load(sys.stdin).get('query',''))" 2>/dev/null) + npx gitnexus query "$q" 2>&1 + ;; + context) + local n=$(echo "$json_body" | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null) + npx gitnexus context "$n" 2>&1 + ;; + impact) + local t=$(echo "$json_body" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('target',''))" 2>/dev/null) + local d=$(echo "$json_body" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('direction','upstream'))" 2>/dev/null) + npx gitnexus impact "$t" --direction "$d" 2>&1 + ;; + cypher) + local cq=$(echo "$json_body" | python3 -c "import sys,json; print(json.load(sys.stdin).get('query',''))" 2>/dev/null) + npx gitnexus cypher "$cq" 2>&1 + ;; + *) + echo "Unknown tool: $tool" >&2 + return 1 + ;; + esac +} + +gitnexus-query() { + local query="$1" + local task_context="${2:-}" + local goal="${3:-}" + + if [ -z "$query" ]; then + echo "Usage: gitnexus-query [task_context] [goal]" + echo "Search the code knowledge graph for execution flows related to a concept." + echo "" + echo "Examples:" + echo ' gitnexus-query "authentication flow"' + echo ' gitnexus-query "database connection" "fixing connection pool leak"' + return 1 + fi + + local args="{\"query\": \"$query\"" + [ -n "$task_context" ] && args="$args, \"task_context\": \"$task_context\"" + [ -n "$goal" ] && args="$args, \"goal\": \"$goal\"" + args="$args}" + + _gitnexus_call query "$args" +} + +gitnexus-context() { + local name="$1" + local file_path="${2:-}" + + if [ -z "$name" ]; then + echo "Usage: gitnexus-context [file_path]" + echo "Get a 360-degree view of a code symbol: callers, callees, processes, file location." + echo "" + echo "Examples:" + echo ' gitnexus-context "validateUser"' + echo ' gitnexus-context "AuthService" "src/auth/service.py"' + return 1 + fi + + local args="{\"name\": \"$name\"" + [ -n "$file_path" ] && args="$args, \"file_path\": \"$file_path\"" + args="$args}" + + _gitnexus_call context "$args" +} + +gitnexus-impact() { + local target="$1" + local direction="${2:-upstream}" + + if [ -z "$target" ]; then + echo "Usage: gitnexus-impact [upstream|downstream]" + echo "Analyze the blast radius of changing a code symbol." + echo "" + echo " upstream = what depends on this (what breaks if you change it)" + echo " downstream = what this depends on (what it uses)" + echo "" + echo "Examples:" + echo ' gitnexus-impact "AuthService" upstream' + echo ' gitnexus-impact "validateUser" downstream' + return 1 + fi + + _gitnexus_call impact "{\"target\": \"$target\", \"direction\": \"$direction\"}" +} + +gitnexus-cypher() { + local query="$1" + + if [ -z "$query" ]; then + echo "Usage: gitnexus-cypher " + echo "Execute a raw Cypher query against the code knowledge graph." + echo "" + echo "Schema: Nodes: File, Function, Class, Method, Interface, Community, Process" + echo "Edges via CodeRelation.type: CALLS, IMPORTS, EXTENDS, IMPLEMENTS, DEFINES, MEMBER_OF, STEP_IN_PROCESS" + echo "" + echo "Examples:" + echo " gitnexus-cypher 'MATCH (a)-[:CodeRelation {type: \"CALLS\"}]->(b:Function {name: \"save\"}) RETURN a.name, a.filePath'" + echo " gitnexus-cypher 'MATCH (n:Class) RETURN n.name, n.filePath LIMIT 20'" + return 1 + fi + + _gitnexus_call cypher "{\"query\": \"$query\"}" +} + +gitnexus-overview() { + echo "=== Code Knowledge Graph Overview ===" + _gitnexus_call list_repos '{}' +} + +# Export functions so they're available in subshells +export -f _gitnexus_call 2>/dev/null +export -f gitnexus-query 2>/dev/null +export -f gitnexus-context 2>/dev/null +export -f gitnexus-impact 2>/dev/null +export -f gitnexus-cypher 2>/dev/null +export -f gitnexus-overview 2>/dev/null diff --git a/eval/bridge/mcp_bridge.py b/eval/bridge/mcp_bridge.py new file mode 100644 index 0000000000..1be0bde6e0 --- /dev/null +++ b/eval/bridge/mcp_bridge.py @@ -0,0 +1,336 @@ +""" +MCP Bridge for GitNexus + +Starts the GitNexus MCP server as a subprocess and provides a Python interface +to call MCP tools. Used by the bash wrapper scripts and the augmentation layer. + +The bridge communicates with the MCP server via stdio using the JSON-RPC protocol. +""" + +import json +import logging +import os +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Any + +logger = logging.getLogger("mcp_bridge") + + +class MCPBridge: + """ + Manages a GitNexus MCP server subprocess and proxies tool calls to it. + + Usage: + bridge = MCPBridge(repo_path="/path/to/repo") + bridge.start() + result = bridge.call_tool("query", {"query": "authentication"}) + bridge.stop() + """ + + def __init__(self, repo_path: str | None = None): + self.repo_path = repo_path or os.getcwd() + self.process: subprocess.Popen | None = None + self._request_id = 0 + self._lock = threading.Lock() + self._started = False + + def start(self) -> bool: + """Start the GitNexus MCP server subprocess.""" + if self._started: + return True + + try: + # Find gitnexus binary + gitnexus_bin = self._find_gitnexus() + if not gitnexus_bin: + logger.error("GitNexus not found. Install with: npm install -g gitnexus") + return False + + self.process = subprocess.Popen( + [gitnexus_bin, "mcp"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.repo_path, + text=False, + ) + + # Send initialize request + init_result = self._send_request("initialize", { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "gitnexus-eval", "version": "0.1.0"}, + }) + + if init_result is None: + logger.error("MCP server failed to initialize") + self.stop() + return False + + # Send initialized notification + self._send_notification("notifications/initialized", {}) + self._started = True + logger.info("MCP bridge started successfully") + return True + + except Exception as e: + logger.error(f"Failed to start MCP bridge: {e}") + self.stop() + return False + + def stop(self): + """Stop the MCP server subprocess.""" + if self.process: + try: + self.process.stdin.close() + self.process.terminate() + self.process.wait(timeout=5) + except Exception: + try: + self.process.kill() + except Exception: + pass + self.process = None + self._started = False + + def call_tool(self, tool_name: str, arguments: dict[str, Any] | None = None) -> dict[str, Any] | None: + """ + Call a GitNexus MCP tool and return the result. + + Returns the tool result content or None on error. + """ + if not self._started: + logger.error("MCP bridge not started") + return None + + result = self._send_request("tools/call", { + "name": tool_name, + "arguments": arguments or {}, + }) + + if result is None: + return None + + # Extract text content from MCP response + content = result.get("content", []) + if content and isinstance(content, list): + texts = [item.get("text", "") for item in content if item.get("type") == "text"] + return {"text": "\n".join(texts), "raw": content} + + return {"text": "", "raw": content} + + def list_tools(self) -> list[dict]: + """List available MCP tools.""" + result = self._send_request("tools/list", {}) + if result: + return result.get("tools", []) + return [] + + def read_resource(self, uri: str) -> str | None: + """Read an MCP resource by URI.""" + result = self._send_request("resources/read", {"uri": uri}) + if result: + contents = result.get("contents", []) + if contents: + return contents[0].get("text", "") + return None + + def _find_gitnexus(self) -> str | None: + """Find the gitnexus CLI binary.""" + # Check if npx is available (preferred - uses local install) + for cmd in ["npx"]: + try: + result = subprocess.run( + [cmd, "gitnexus", "--version"], + capture_output=True, text=True, timeout=15, + cwd=self.repo_path, + ) + if result.returncode == 0: + return cmd # Will use "npx gitnexus mcp" + except Exception: + continue + + # Check for global install + try: + result = subprocess.run( + ["gitnexus", "--version"], + capture_output=True, text=True, timeout=10, + ) + if result.returncode == 0: + return "gitnexus" + except Exception: + pass + + return None + + def _next_id(self) -> int: + with self._lock: + self._request_id += 1 + return self._request_id + + def _send_request(self, method: str, params: dict) -> dict | None: + """Send a JSON-RPC request and wait for response.""" + if not self.process or not self.process.stdin or not self.process.stdout: + return None + + request_id = self._next_id() + request = { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params, + } + + try: + message = json.dumps(request) + # MCP uses Content-Length header framing + header = f"Content-Length: {len(message.encode('utf-8'))}\r\n\r\n" + self.process.stdin.write(header.encode("utf-8")) + self.process.stdin.write(message.encode("utf-8")) + self.process.stdin.flush() + + # Read response + response = self._read_response(timeout=30) + if response and response.get("id") == request_id: + if "error" in response: + logger.error(f"MCP error: {response['error']}") + return None + return response.get("result") + return None + + except Exception as e: + logger.error(f"MCP request failed: {e}") + return None + + def _send_notification(self, method: str, params: dict): + """Send a JSON-RPC notification (no response expected).""" + if not self.process or not self.process.stdin: + return + + notification = { + "jsonrpc": "2.0", + "method": method, + "params": params, + } + + try: + message = json.dumps(notification) + header = f"Content-Length: {len(message.encode('utf-8'))}\r\n\r\n" + self.process.stdin.write(header.encode("utf-8")) + self.process.stdin.write(message.encode("utf-8")) + self.process.stdin.flush() + except Exception as e: + logger.error(f"MCP notification failed: {e}") + + def _read_response(self, timeout: float = 30) -> dict | None: + """Read a JSON-RPC response from the MCP server.""" + if not self.process or not self.process.stdout: + return None + + start = time.time() + + try: + while time.time() - start < timeout: + # Read Content-Length header + header_line = b"" + while True: + byte = self.process.stdout.read(1) + if not byte: + return None + header_line += byte + if header_line.endswith(b"\r\n\r\n"): + break + if header_line.endswith(b"\n\n"): + break + + # Parse content length + header_str = header_line.decode("utf-8").strip() + content_length = None + for line in header_str.split("\r\n"): + if line.lower().startswith("content-length:"): + content_length = int(line.split(":")[1].strip()) + break + + if content_length is None: + continue + + # Read body + body = self.process.stdout.read(content_length) + if not body: + return None + + message = json.loads(body.decode("utf-8")) + + # Skip notifications (no id), return responses + if "id" in message: + return message + + return None + + except Exception as e: + logger.error(f"Error reading MCP response: {e}") + return None + + +class MCPToolCLI: + """ + CLI wrapper that exposes MCP tools as simple command-line calls. + Used by the bash wrapper scripts inside Docker containers. + + Usage from bash: + python -m bridge.mcp_bridge query '{"query": "authentication"}' + python -m bridge.mcp_bridge context '{"name": "validateUser"}' + """ + + def __init__(self): + self.bridge = MCPBridge() + + def run(self, tool_name: str, args_json: str = "{}") -> int: + """Run a single tool call and print the result.""" + try: + args = json.loads(args_json) + except json.JSONDecodeError: + # Try to parse as simple key=value pairs + args = self._parse_simple_args(args_json) + + if not self.bridge.start(): + print("ERROR: Failed to start GitNexus MCP bridge", file=sys.stderr) + return 1 + + try: + result = self.bridge.call_tool(tool_name, args) + if result: + print(result.get("text", "")) + return 0 + else: + print("No results", file=sys.stderr) + return 1 + finally: + self.bridge.stop() + + @staticmethod + def _parse_simple_args(args_str: str) -> dict: + """Parse 'key=value key2=value2' style arguments.""" + args = {} + for part in args_str.split(): + if "=" in part: + key, value = part.split("=", 1) + args[key] = value + return args + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python -m bridge.mcp_bridge [args_json]", file=sys.stderr) + print("Tools: query, context, impact, cypher, list_repos, detect_changes, rename", file=sys.stderr) + sys.exit(1) + + tool = sys.argv[1] + args_json = sys.argv[2] if len(sys.argv) > 2 else "{}" + + cli = MCPToolCLI() + sys.exit(cli.run(tool, args_json)) diff --git a/eval/configs/models/claude-haiku.yaml b/eval/configs/models/claude-haiku.yaml new file mode 100644 index 0000000000..89746cdbdc --- /dev/null +++ b/eval/configs/models/claude-haiku.yaml @@ -0,0 +1,9 @@ +# Claude 3.5 Haiku โ€” fast, cheap, good baseline +# Via OpenRouter (set OPENROUTER_API_KEY in .env) +# To use Anthropic directly, change to: anthropic/claude-3-5-haiku-20241022 +model: + model_name: "openrouter/anthropic/claude-3.5-haiku" + cost_tracking: "ignore_errors" + model_kwargs: + max_tokens: 8192 + temperature: 0 diff --git a/eval/configs/models/claude-opus.yaml b/eval/configs/models/claude-opus.yaml new file mode 100644 index 0000000000..3c3e114c41 --- /dev/null +++ b/eval/configs/models/claude-opus.yaml @@ -0,0 +1,9 @@ +# Claude Opus 4 โ€” most capable, highest cost +# Via OpenRouter (set OPENROUTER_API_KEY in .env) +# To use Anthropic directly, change to: anthropic/claude-opus-4-20250514 +model: + model_name: "openrouter/anthropic/claude-opus-4" + cost_tracking: "ignore_errors" + model_kwargs: + max_tokens: 16384 + temperature: 0 diff --git a/eval/configs/models/claude-sonnet.yaml b/eval/configs/models/claude-sonnet.yaml new file mode 100644 index 0000000000..2f8ab14930 --- /dev/null +++ b/eval/configs/models/claude-sonnet.yaml @@ -0,0 +1,9 @@ +# Claude Sonnet 4 โ€” strong all-around model +# Via OpenRouter (set OPENROUTER_API_KEY in .env) +# To use Anthropic directly, change to: anthropic/claude-sonnet-4-20250514 +model: + model_name: "openrouter/anthropic/claude-sonnet-4" + cost_tracking: "ignore_errors" + model_kwargs: + max_tokens: 16384 + temperature: 0 diff --git a/eval/configs/models/glm-4.7.yaml b/eval/configs/models/glm-4.7.yaml new file mode 100644 index 0000000000..8dc3111a17 --- /dev/null +++ b/eval/configs/models/glm-4.7.yaml @@ -0,0 +1,7 @@ +# GLM 4.7 โ€” via OpenRouter (set OPENROUTER_API_KEY in .env) +model: + model_name: "openrouter/zhipuai/glm-4.7" + cost_tracking: "ignore_errors" + model_kwargs: + max_tokens: 8192 + temperature: 0 diff --git a/eval/configs/models/glm-5.yaml b/eval/configs/models/glm-5.yaml new file mode 100644 index 0000000000..f37a162e6b --- /dev/null +++ b/eval/configs/models/glm-5.yaml @@ -0,0 +1,7 @@ +# GLM 5 โ€” via OpenRouter (set OPENROUTER_API_KEY in .env) +model: + model_name: "openrouter/zhipuai/glm-5" + cost_tracking: "ignore_errors" + model_kwargs: + max_tokens: 8192 + temperature: 0 diff --git a/eval/configs/models/minimax-2.5.yaml b/eval/configs/models/minimax-2.5.yaml new file mode 100644 index 0000000000..f3b43d5548 --- /dev/null +++ b/eval/configs/models/minimax-2.5.yaml @@ -0,0 +1,7 @@ +# MiniMax M1 2.5 โ€” via OpenRouter (set OPENROUTER_API_KEY in .env) +model: + model_name: "openrouter/minimax/minimax-m1-2.5" + cost_tracking: "ignore_errors" + model_kwargs: + max_tokens: 8192 + temperature: 0 diff --git a/eval/configs/modes/baseline.yaml b/eval/configs/modes/baseline.yaml new file mode 100644 index 0000000000..c9c73b6379 --- /dev/null +++ b/eval/configs/modes/baseline.yaml @@ -0,0 +1,9 @@ +# Baseline mode โ€” no GitNexus, pure mini-swe-agent (control group) +agent: + agent_class: "eval.agents.gitnexus_agent.GitNexusAgent" + gitnexus_mode: "baseline" + step_limit: 30 + cost_limit: 3.0 + +environment: + environment_class: "docker" diff --git a/eval/configs/modes/native.yaml b/eval/configs/modes/native.yaml new file mode 100644 index 0000000000..573d125e7c --- /dev/null +++ b/eval/configs/modes/native.yaml @@ -0,0 +1,19 @@ +# Native mode โ€” GitNexus tools only, no grep enrichment +# +# Explicit tools: gitnexus-query, gitnexus-context, gitnexus-impact, gitnexus-cypher +# Available as fast bash commands (~100ms via eval-server) +# +# Use this mode to isolate the value of explicit tools without grep augmentation. +agent: + agent_class: "eval.agents.gitnexus_agent.GitNexusAgent" + gitnexus_mode: "native" + step_limit: 30 + cost_limit: 3.0 + track_gitnexus_usage: true + +environment: + environment_class: "eval.environments.gitnexus_docker.GitNexusDockerEnvironment" + enable_gitnexus: true + skip_embeddings: true + gitnexus_timeout: 120 + eval_server_port: 4848 diff --git a/eval/configs/modes/native_augment.yaml b/eval/configs/modes/native_augment.yaml new file mode 100644 index 0000000000..fb9b79729e --- /dev/null +++ b/eval/configs/modes/native_augment.yaml @@ -0,0 +1,24 @@ +# Native + Augment mode โ€” the primary evaluation mode +# +# Combines two capabilities (mirroring the Claude Code model): +# 1. Explicit GitNexus tools: gitnexus-query, gitnexus-context, gitnexus-impact, gitnexus-cypher +# Available as fast bash commands (~100ms via eval-server) +# 2. Automatic grep enrichment: grep/rg results are transparently augmented with +# [GitNexus] annotations showing callers, callees, and execution flows +# +# The agent decides when to use explicit tools vs rely on enriched grep results. +agent: + agent_class: "eval.agents.gitnexus_agent.GitNexusAgent" + gitnexus_mode: "native_augment" + step_limit: 30 + cost_limit: 3.0 + augment_timeout: 5.0 + augment_min_pattern_length: 3 + track_gitnexus_usage: true + +environment: + environment_class: "eval.environments.gitnexus_docker.GitNexusDockerEnvironment" + enable_gitnexus: true + skip_embeddings: true + gitnexus_timeout: 120 + eval_server_port: 4848 diff --git a/eval/environments/__init__.py b/eval/environments/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/eval/environments/gitnexus_docker.py b/eval/environments/gitnexus_docker.py new file mode 100644 index 0000000000..95ad07bac8 --- /dev/null +++ b/eval/environments/gitnexus_docker.py @@ -0,0 +1,397 @@ +""" +GitNexus Docker Environment for SWE-bench Evaluation + +Extends mini-swe-agent's Docker environment to: +1. Install GitNexus (Node.js + npm + gitnexus package) +2. Run `gitnexus analyze` on the repository +3. Start the eval-server daemon (persistent HTTP server with warm KuzuDB) +4. Install standalone tool scripts in /usr/local/bin/ (works with subprocess.run) +5. Cache indexes per (repo, base_commit) to avoid re-indexing + +IMPORTANT: mini-swe-agent runs every command with subprocess.run in a fresh subshell. +This means .bashrc is NOT sourced, exported functions are NOT available, and env vars +don't persist. The tool scripts must be standalone executables in $PATH. + +Architecture: + Agent bash cmd โ†’ /usr/local/bin/gitnexus-query โ†’ curl localhost:4848/tool/query โ†’ eval-server โ†’ KuzuDB + Fallback: โ†’ npx gitnexus query (cold start, slower) + +Tool call latency: ~50-100ms via eval-server, ~5-10s via CLI fallback. +""" + +import hashlib +import json +import logging +import shutil +import time +from pathlib import Path + +from minisweagent.environments.docker import DockerEnvironment + +logger = logging.getLogger("gitnexus_docker") + +DEFAULT_CACHE_DIR = Path.home() / ".gitnexus-eval-cache" +EVAL_SERVER_PORT = 4848 + +# Standalone tool scripts installed into /usr/local/bin/ inside the container. +# Each script calls the eval-server via curl, with a CLI fallback. +# These are standalone โ€” no sourcing, no env inheritance needed. + +TOOL_SCRIPT_QUERY = r'''#!/bin/bash +PORT="${GITNEXUS_EVAL_PORT:-__PORT__}" +query="$1"; task_ctx="${2:-}"; goal="${3:-}" +[ -z "$query" ] && echo "Usage: gitnexus-query [task_context] [goal]" && exit 1 +args="{\"query\": \"$query\"" +[ -n "$task_ctx" ] && args="$args, \"task_context\": \"$task_ctx\"" +[ -n "$goal" ] && args="$args, \"goal\": \"$goal\"" +args="$args}" +result=$(curl -sf -X POST "http://127.0.0.1:${PORT}/tool/query" -H "Content-Type: application/json" -d "$args" 2>/dev/null) +if [ $? -eq 0 ] && [ -n "$result" ]; then echo "$result"; exit 0; fi +cd /testbed && npx gitnexus query "$query" 2>&1 +''' + +TOOL_SCRIPT_CONTEXT = r'''#!/bin/bash +PORT="${GITNEXUS_EVAL_PORT:-__PORT__}" +name="$1"; file_path="${2:-}" +[ -z "$name" ] && echo "Usage: gitnexus-context [file_path]" && exit 1 +args="{\"name\": \"$name\"" +[ -n "$file_path" ] && args="$args, \"file_path\": \"$file_path\"" +args="$args}" +result=$(curl -sf -X POST "http://127.0.0.1:${PORT}/tool/context" -H "Content-Type: application/json" -d "$args" 2>/dev/null) +if [ $? -eq 0 ] && [ -n "$result" ]; then echo "$result"; exit 0; fi +cd /testbed && npx gitnexus context "$name" 2>&1 +''' + +TOOL_SCRIPT_IMPACT = r'''#!/bin/bash +PORT="${GITNEXUS_EVAL_PORT:-__PORT__}" +target="$1"; direction="${2:-upstream}" +[ -z "$target" ] && echo "Usage: gitnexus-impact [upstream|downstream]" && exit 1 +result=$(curl -sf -X POST "http://127.0.0.1:${PORT}/tool/impact" -H "Content-Type: application/json" -d "{\"target\": \"$target\", \"direction\": \"$direction\"}" 2>/dev/null) +if [ $? -eq 0 ] && [ -n "$result" ]; then echo "$result"; exit 0; fi +cd /testbed && npx gitnexus impact "$target" --direction "$direction" 2>&1 +''' + +TOOL_SCRIPT_CYPHER = r'''#!/bin/bash +PORT="${GITNEXUS_EVAL_PORT:-__PORT__}" +query="$1" +[ -z "$query" ] && echo "Usage: gitnexus-cypher " && exit 1 +result=$(curl -sf -X POST "http://127.0.0.1:${PORT}/tool/cypher" -H "Content-Type: application/json" -d "{\"query\": \"$query\"}" 2>/dev/null) +if [ $? -eq 0 ] && [ -n "$result" ]; then echo "$result"; exit 0; fi +cd /testbed && npx gitnexus cypher "$query" 2>&1 +''' + +TOOL_SCRIPT_AUGMENT = r'''#!/bin/bash +cd /testbed && npx gitnexus augment "$1" 2>&1 || true +''' + +TOOL_SCRIPT_OVERVIEW = r'''#!/bin/bash +PORT="${GITNEXUS_EVAL_PORT:-__PORT__}" +echo "=== Code Knowledge Graph Overview ===" +result=$(curl -sf -X POST "http://127.0.0.1:${PORT}/tool/list_repos" -H "Content-Type: application/json" -d "{}" 2>/dev/null) +if [ $? -eq 0 ] && [ -n "$result" ]; then echo "$result"; exit 0; fi +cd /testbed && npx gitnexus list 2>&1 +''' + + +class GitNexusDockerEnvironment(DockerEnvironment): + """ + Docker environment with GitNexus pre-installed, indexed, and eval-server running. + + Setup flow: + 1. Start Docker container (base SWE-bench image) + 2. Install Node.js + gitnexus inside the container + 3. Run `gitnexus analyze` (or restore from cache) + 4. Start `gitnexus eval-server` daemon (keeps KuzuDB warm) + 5. Install standalone tool scripts in /usr/local/bin/ + 6. Agent runs with near-instant GitNexus tool calls + """ + + def __init__( + self, + *, + enable_gitnexus: bool = True, + cache_dir: str | Path | None = None, + skip_embeddings: bool = True, + gitnexus_timeout: int = 120, + eval_server_port: int = EVAL_SERVER_PORT, + **kwargs, + ): + super().__init__(**kwargs) + self.enable_gitnexus = enable_gitnexus + self.cache_dir = Path(cache_dir) if cache_dir else DEFAULT_CACHE_DIR + self.skip_embeddings = skip_embeddings + self.gitnexus_timeout = gitnexus_timeout + self.eval_server_port = eval_server_port + self.index_time: float = 0.0 + self._gitnexus_ready = False + + def start(self) -> dict: + """Start the container and set up GitNexus.""" + result = super().start() + + if self.enable_gitnexus: + try: + self._setup_gitnexus() + except Exception as e: + logger.warning(f"GitNexus setup failed, continuing without it: {e}") + self._gitnexus_ready = False + + return result + + def _setup_gitnexus(self): + """Install and configure GitNexus in the container.""" + start = time.time() + + self._ensure_nodejs() + self._install_gitnexus() + self._index_repository() + self._start_eval_server() + self._install_tools() + + self.index_time = time.time() - start + self._gitnexus_ready = True + logger.info(f"GitNexus setup completed in {self.index_time:.1f}s") + + def _ensure_nodejs(self): + """Ensure Node.js >= 18 is available in the container.""" + check = self.execute({"command": "node --version 2>/dev/null || echo 'NOT_FOUND'"}) + output = check.get("output", "").strip() + + if "NOT_FOUND" in output: + logger.info("Installing Node.js in container...") + install_cmds = [ + "apt-get update -qq", + "apt-get install -y -qq curl ca-certificates", + "curl -fsSL https://deb.nodesource.com/setup_20.x | bash -", + "apt-get install -y -qq nodejs", + ] + for cmd in install_cmds: + result = self.execute({"command": cmd, "timeout": 60}) + if result.get("returncode", 1) != 0: + raise RuntimeError(f"Failed to install Node.js: {result.get('output', '')}") + else: + logger.info(f"Node.js already available: {output}") + + def _install_gitnexus(self): + """Install the gitnexus npm package globally.""" + check = self.execute({"command": "npx gitnexus --version 2>/dev/null || echo 'NOT_FOUND'"}) + if "NOT_FOUND" in check.get("output", ""): + logger.info("Installing gitnexus...") + result = self.execute({ + "command": "npm install -g gitnexus", + "timeout": 60, + }) + if result.get("returncode", 1) != 0: + raise RuntimeError(f"Failed to install gitnexus: {result.get('output', '')}") + + def _index_repository(self): + """Run gitnexus analyze on the repo, using cache if available.""" + repo_info = self._get_repo_info() + cache_key = self._make_cache_key(repo_info) + cache_path = self.cache_dir / cache_key + + if cache_path.exists(): + logger.info(f"Restoring GitNexus index from cache: {cache_key}") + self._restore_cache(cache_path) + return + + logger.info("Running gitnexus analyze...") + skip_flag = "--skip-embeddings" if self.skip_embeddings else "" + result = self.execute({ + "command": f"cd /testbed && npx gitnexus analyze . {skip_flag} 2>&1", + "timeout": self.gitnexus_timeout, + }) + + if result.get("returncode", 1) != 0: + output = result.get("output", "") + if "error" in output.lower() and "indexed" not in output.lower(): + raise RuntimeError(f"gitnexus analyze failed: {output[-500:]}") + + self._save_cache(cache_path, repo_info) + + def _start_eval_server(self): + """Start the GitNexus eval-server daemon in the background.""" + logger.info(f"Starting eval-server on port {self.eval_server_port}...") + + self.execute({ + "command": ( + f"nohup npx gitnexus eval-server --port {self.eval_server_port} " + f"--idle-timeout 600 " + f"> /tmp/gitnexus-eval-server.log 2>&1 &" + ), + "timeout": 5, + }) + + # Wait for the server to be ready (up to 15s for KuzuDB init) + for i in range(30): + time.sleep(0.5) + health = self.execute({ + "command": f"curl -sf http://127.0.0.1:{self.eval_server_port}/health 2>/dev/null || echo 'NOT_READY'", + "timeout": 3, + }) + output = health.get("output", "").strip() + if "NOT_READY" not in output and "ok" in output: + logger.info(f"Eval-server ready after {(i + 1) * 0.5:.1f}s") + return + + log_output = self.execute({ + "command": "cat /tmp/gitnexus-eval-server.log 2>/dev/null | tail -20", + }) + logger.warning( + f"Eval-server didn't become ready in 15s. " + f"Tools will fall back to direct CLI.\n" + f"Server log: {log_output.get('output', 'N/A')}" + ) + + def _install_tools(self): + """ + Install standalone GitNexus tool scripts in /usr/local/bin/. + + Each script is a self-contained bash script that: + 1. Calls the eval-server via curl (fast path, ~100ms) + 2. Falls back to direct CLI if eval-server is unavailable + + These are standalone executables โ€” no sourcing, env inheritance, or .bashrc + needed. This is critical because mini-swe-agent runs every command via + subprocess.run in a fresh subshell. + + Uses heredocs with quoted delimiter to avoid all quoting/escaping issues. + """ + port = str(self.eval_server_port) + + tools = { + "gitnexus-query": TOOL_SCRIPT_QUERY, + "gitnexus-context": TOOL_SCRIPT_CONTEXT, + "gitnexus-impact": TOOL_SCRIPT_IMPACT, + "gitnexus-cypher": TOOL_SCRIPT_CYPHER, + "gitnexus-augment": TOOL_SCRIPT_AUGMENT, + "gitnexus-overview": TOOL_SCRIPT_OVERVIEW, + } + + for name, script in tools.items(): + script_content = script.replace("__PORT__", port).strip() + # Use heredoc with quoted delimiter โ€” prevents all variable expansion and quoting issues + self.execute({ + "command": f"cat << 'GITNEXUS_SCRIPT_EOF' > /usr/local/bin/{name}\n{script_content}\nGITNEXUS_SCRIPT_EOF\nchmod +x /usr/local/bin/{name}", + "timeout": 5, + }) + + logger.info(f"Installed {len(tools)} GitNexus tool scripts in /usr/local/bin/") + + def _get_repo_info(self) -> dict: + """Get repository identity info from the container.""" + repo_result = self.execute({ + "command": "cd /testbed && basename $(git remote get-url origin 2>/dev/null || basename $(pwd)) .git" + }) + commit_result = self.execute({"command": "cd /testbed && git rev-parse HEAD 2>/dev/null || echo unknown"}) + + return { + "repo": repo_result.get("output", "unknown").strip(), + "commit": commit_result.get("output", "unknown").strip(), + } + + @staticmethod + def _make_cache_key(repo_info: dict) -> str: + """Create a deterministic cache key from repo info.""" + content = f"{repo_info['repo']}:{repo_info['commit']}" + return hashlib.sha256(content.encode()).hexdigest()[:16] + + def _save_cache(self, cache_path: Path, repo_info: dict): + """Save the GitNexus index to the host cache directory.""" + try: + cache_path.mkdir(parents=True, exist_ok=True) + + find_result = self.execute({ + "command": "find /root/.gitnexus -name 'kuzu' -type d 2>/dev/null | head -1" + }) + gitnexus_dir = find_result.get("output", "").strip() + + if gitnexus_dir: + parent = str(Path(gitnexus_dir).parent) + self.execute({ + "command": f"cd {parent} && tar czf /tmp/gitnexus-cache.tar.gz .", + "timeout": 30, + }) + + container_id = getattr(self, "_container_id", None) or getattr(self, "container_id", None) + if container_id: + import subprocess as sp + sp.run( + ["docker", "cp", f"{container_id}:/tmp/gitnexus-cache.tar.gz", + str(cache_path / "index.tar.gz")], + check=True, capture_output=True, + ) + (cache_path / "metadata.json").write_text(json.dumps(repo_info, indent=2)) + logger.info(f"Cached GitNexus index: {cache_path}") + + except Exception as e: + logger.warning(f"Failed to cache GitNexus index: {e}") + if cache_path.exists(): + shutil.rmtree(cache_path, ignore_errors=True) + + def _restore_cache(self, cache_path: Path): + """Restore a cached GitNexus index into the container.""" + try: + cache_tarball = cache_path / "index.tar.gz" + if not cache_tarball.exists(): + logger.warning("Cache tarball not found, re-indexing") + self._index_repository() + return + + container_id = getattr(self, "_container_id", None) or getattr(self, "container_id", None) + if container_id: + import subprocess as sp + + self.execute({"command": "mkdir -p /root/.gitnexus"}) + + storage_result = self.execute({ + "command": "npx gitnexus list 2>/dev/null | grep -o '/root/.gitnexus/[^ ]*' | head -1 || echo '/root/.gitnexus/repos/default'" + }) + storage_path = storage_result.get("output", "").strip() or "/root/.gitnexus/repos/default" + self.execute({"command": f"mkdir -p {storage_path}"}) + + sp.run( + ["docker", "cp", str(cache_tarball), f"{container_id}:/tmp/gitnexus-cache.tar.gz"], + check=True, capture_output=True, + ) + self.execute({ + "command": f"cd {storage_path} && tar xzf /tmp/gitnexus-cache.tar.gz", + "timeout": 30, + }) + logger.info("GitNexus index restored from cache") + + except Exception as e: + logger.warning(f"Failed to restore cache, re-indexing: {e}") + self._index_repository() + + def stop(self) -> dict: + """Stop the container, shutting down eval-server first.""" + if self._gitnexus_ready: + try: + self.execute({ + "command": f"curl -sf -X POST http://127.0.0.1:{self.eval_server_port}/shutdown 2>/dev/null || true", + "timeout": 3, + }) + except Exception: + pass + + return super().stop() + + def get_template_vars(self) -> dict: + """Add GitNexus-specific template variables.""" + base_vars = super().get_template_vars() + base_vars["gitnexus_ready"] = self._gitnexus_ready + base_vars["gitnexus_index_time"] = self.index_time + return base_vars + + def serialize(self) -> dict: + """Include GitNexus environment info in serialization.""" + base = super().serialize() + base.setdefault("info", {})["gitnexus_env"] = { + "enabled": self.enable_gitnexus, + "ready": self._gitnexus_ready, + "index_time_seconds": round(self.index_time, 2), + "skip_embeddings": self.skip_embeddings, + "eval_server_port": self.eval_server_port, + } + return base diff --git a/eval/prompts/instance_baseline.jinja b/eval/prompts/instance_baseline.jinja new file mode 100644 index 0000000000..11a939edea --- /dev/null +++ b/eval/prompts/instance_baseline.jinja @@ -0,0 +1,80 @@ +Please solve this issue: {{task}} + +You can execute bash commands and edit files to implement the necessary changes. + +## Recommended Workflow + +This workflows should be done step-by-step so that you can iterate on your changes and any possible problems. + +1. Analyze the codebase by finding and reading relevant files +2. Create a script to reproduce the issue +3. Edit the source code to resolve the issue +4. Verify your fix works by running your script again +5. Test edge cases to ensure your fix is robust +6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`. + Do not combine it with any other command. After this command, you cannot continue working on this task. + +## Important Rules + +1. Every response must contain exactly one action +2. The action must be enclosed in triple backticks +3. Directory or environment variable changes are not persistent. Every action is executed in a new subshell. + However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files + + +{{system}} {{release}} {{version}} {{machine}} + + +## Formatting your response + +Here is an example of a correct response: + + +THOUGHT: I need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase. + +```mswea_bash_command +ls -la +``` + + +## Useful command examples + +### Create a new file: + +```bash +cat <<'EOF' > newfile.py +import numpy as np +hello = "world" +print(hello) +EOF +``` + +### Edit files with sed: + +{%- if system == "Darwin" -%} + +You are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`. + +{%- endif -%} + +```bash +# Replace all occurrences +sed -i 's/old_string/new_string/g' filename.py +# Replace only first occurrence +sed -i 's/old_string/new_string/' filename.py +# Replace all occurrences in lines 1-10 +sed -i '1,10s/old_string/new_string/g' filename.py +``` + +### View file content: + +```bash +# View specific lines with numbers +nl -ba filename.py | sed -n '10,20p' +``` + +### Any other command you want to run + +```bash +anything +``` diff --git a/eval/prompts/instance_native.jinja b/eval/prompts/instance_native.jinja new file mode 100644 index 0000000000..0b4f14ed9b --- /dev/null +++ b/eval/prompts/instance_native.jinja @@ -0,0 +1,102 @@ +Please solve this issue: {{task}} + +You can execute bash commands and edit files to implement the necessary changes. + +## Recommended Workflow + +Work step-by-step so you can iterate on your changes and catch problems early. + +1. **Understand the issue** โ€” read the problem statement, identify the symptom and affected area +2. **Find the relevant code** โ€” use `gitnexus-query ""` to find execution flows, or `grep` for specific strings +3. **Understand the suspect** โ€” use `gitnexus-context ""` to see all callers and callees, then `cat` to read the source +4. **Check blast radius** โ€” before editing shared code, run `gitnexus-impact "" upstream` to see what depends on it +5. **Implement the fix** โ€” make minimal, targeted changes +6. **Verify** โ€” run relevant tests, check edge cases +7. **Submit** โ€” issue: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` + Do not combine it with any other command. After this command, you cannot continue working on this task. + +## Debugging Patterns + +| Symptom | Approach | +|---------|----------| +| Error message / exception | `gitnexus-query` for error text โ†’ `gitnexus-context` on throw sites | +| Wrong return value | `gitnexus-context` on the function โ†’ trace callees for data flow | +| Missing feature / incomplete behavior | `gitnexus-query` for feature area โ†’ find the execution flow โ†’ locate the gap | +| Need to understand callers | `gitnexus-context` โ€” graph-complete, finds callers grep would miss | + +## Risk Assessment + +Before editing shared code, check the blast radius: + +| Impact | Risk | Action | +|--------|------|--------| +| <5 symbols at d=1 | Low | Fix with confidence | +| 5-15 symbols at d=1 | Medium | Fix carefully, run broader tests | +| >15 symbols at d=1 | High | Minimal change, run full test suite | + +## Important Rules + +1. Every response must contain exactly one action +2. The action must be enclosed in triple backticks +3. Directory or environment variable changes are not persistent. Every action is executed in a new subshell. + However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files +4. Make minimal, targeted changes. Don't refactor unrelated code. +5. GitNexus tools are ~100ms. Use them when they save you multiple grep iterations. + + +{{system}} {{release}} {{version}} {{machine}} + + +## Formatting your response + +Here is an example of a correct response: + + +THOUGHT: The issue mentions a problem with form field validation. Let me search the code knowledge graph for the relevant execution flows to understand how validation works in this codebase. + +```mswea_bash_command +gitnexus-query "form field validation" +``` + + +## Useful command examples + +### Create a new file: + +```bash +cat <<'EOF' > newfile.py +import numpy as np +hello = "world" +print(hello) +EOF +``` + +### Edit files with sed: + +{%- if system == "Darwin" -%} + +You are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`. + +{%- endif -%} + +```bash +# Replace all occurrences +sed -i 's/old_string/new_string/g' filename.py +# Replace only first occurrence +sed -i 's/old_string/new_string/' filename.py +# Replace all occurrences in lines 1-10 +sed -i '1,10s/old_string/new_string/g' filename.py +``` + +### View file content: + +```bash +# View specific lines with numbers +nl -ba filename.py | sed -n '10,20p' +``` + +### Any other command you want to run + +```bash +anything +``` diff --git a/eval/prompts/instance_native_augment.jinja b/eval/prompts/instance_native_augment.jinja new file mode 100644 index 0000000000..aafe1bbee9 --- /dev/null +++ b/eval/prompts/instance_native_augment.jinja @@ -0,0 +1,103 @@ +Please solve this issue: {{task}} + +You can execute bash commands and edit files to implement the necessary changes. + +## Recommended Workflow + +Work step-by-step so you can iterate on your changes and catch problems early. + +1. **Understand the issue** โ€” read the problem statement, identify the symptom and affected area +2. **Find the relevant code** โ€” use `gitnexus-query ""` to find execution flows, or `grep` for specific strings +3. **Understand the suspect** โ€” use `gitnexus-context ""` to see all callers and callees, then `cat` to read the source +4. **Check blast radius** โ€” before editing shared code, run `gitnexus-impact "" upstream` to see what depends on it +5. **Implement the fix** โ€” make minimal, targeted changes +6. **Verify** โ€” run relevant tests, check edge cases +7. **Submit** โ€” issue: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` + Do not combine it with any other command. After this command, you cannot continue working on this task. + +## Debugging Patterns + +| Symptom | Approach | +|---------|----------| +| Error message / exception | `gitnexus-query` for error text โ†’ `gitnexus-context` on throw sites | +| Wrong return value | `gitnexus-context` on the function โ†’ trace callees for data flow | +| Missing feature / incomplete behavior | `gitnexus-query` for feature area โ†’ find the execution flow โ†’ locate the gap | +| Need to understand callers | `gitnexus-context` โ€” graph-complete, finds callers grep would miss | + +## Risk Assessment + +Before editing shared code, check the blast radius: + +| Impact | Risk | Action | +|--------|------|--------| +| <5 symbols at d=1 | Low | Fix with confidence | +| 5-15 symbols at d=1 | Medium | Fix carefully, run broader tests | +| >15 symbols at d=1 | High | Minimal change, run full test suite | + +## Important Rules + +1. Every response must contain exactly one action +2. The action must be enclosed in triple backticks +3. Directory or environment variable changes are not persistent. Every action is executed in a new subshell. + However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files +4. Make minimal, targeted changes. Don't refactor unrelated code. +5. GitNexus tools are ~100ms. Use them when they save you multiple grep iterations. +6. When grep results show `[GitNexus]` enrichments, use those for navigation. + + +{{system}} {{release}} {{version}} {{machine}} + + +## Formatting your response + +Here is an example of a correct response: + + +THOUGHT: The issue mentions a problem with form field validation. Let me search the code knowledge graph for the relevant execution flows to understand how validation works in this codebase. + +```mswea_bash_command +gitnexus-query "form field validation" +``` + + +## Useful command examples + +### Create a new file: + +```bash +cat <<'EOF' > newfile.py +import numpy as np +hello = "world" +print(hello) +EOF +``` + +### Edit files with sed: + +{%- if system == "Darwin" -%} + +You are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`. + +{%- endif -%} + +```bash +# Replace all occurrences +sed -i 's/old_string/new_string/g' filename.py +# Replace only first occurrence +sed -i 's/old_string/new_string/' filename.py +# Replace all occurrences in lines 1-10 +sed -i '1,10s/old_string/new_string/g' filename.py +``` + +### View file content: + +```bash +# View specific lines with numbers +nl -ba filename.py | sed -n '10,20p' +``` + +### Any other command you want to run + +```bash +anything +``` diff --git a/eval/prompts/system_baseline.jinja b/eval/prompts/system_baseline.jinja new file mode 100644 index 0000000000..45df961ef0 --- /dev/null +++ b/eval/prompts/system_baseline.jinja @@ -0,0 +1,15 @@ +You are a helpful assistant that can interact with a computer to solve software engineering tasks. + +Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||). +Include a THOUGHT section before your command where you explain your reasoning process. +Format your response as shown in. + + +Your reasoning and analysis here. Explain why you want to perform the action. + +```mswea_bash_command +your_command_here +``` + + +Failure to follow these rules will cause your response to be rejected. diff --git a/eval/prompts/system_native.jinja b/eval/prompts/system_native.jinja new file mode 100644 index 0000000000..6a736846fc --- /dev/null +++ b/eval/prompts/system_native.jinja @@ -0,0 +1,54 @@ +You are a helpful assistant that can interact with a computer to solve software engineering tasks. + +Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||). +Include a THOUGHT section before your command where you explain your reasoning process. +Format your response as shown in. + + +Your reasoning and analysis here. Explain why you want to perform the action. + +```mswea_bash_command +your_command_here +``` + + +Failure to follow these rules will cause your response to be rejected. + +## Code Intelligence + +You have **GitNexus** โ€” a knowledge graph over this entire codebase. It knows every function call chain, class hierarchy, execution flow, and symbol relationship. These are fast bash commands (~100ms). Use them when useful, skip them when a simple grep suffices. + +### GitNexus Commands + +**gitnexus-query ""** โ€” Find execution flows related to a concept. +Returns ranked execution flow traces with participating symbols and file locations. +```bash +gitnexus-query "form field validation" +``` + +**gitnexus-context "" [""]** โ€” 360-degree view of a symbol. +Returns ALL callers, ALL callees, and execution flows. Graph-complete โ€” finds callers that grep misses. +```bash +gitnexus-context "BoundField" "django/forms/boundfield.py" +``` + +**gitnexus-impact "" [upstream|downstream]** โ€” Blast radius analysis. +What breaks if you change this: d=1 WILL BREAK, d=2 LIKELY AFFECTED, d=3 MAY NEED TESTING. +```bash +gitnexus-impact "BoundField" upstream +``` + +**gitnexus-cypher ""** โ€” Raw Cypher query against the code graph. +```bash +gitnexus-cypher 'MATCH (a)-[:CodeRelation {type: "CALLS"}]->(b:Function {name: "clean"}) RETURN a.name, a.filePath' +``` + +### When to Use What + +| I need to... | Use | +|---|---| +| Understand how a feature works end-to-end | `gitnexus-query` | +| Find ALL callers of a function | `gitnexus-context` | +| Know what breaks if I change something | `gitnexus-impact` upstream | +| Find a string literal or error message | `grep` | +| Read source code | `cat` / `nl -ba` | diff --git a/eval/prompts/system_native_augment.jinja b/eval/prompts/system_native_augment.jinja new file mode 100644 index 0000000000..c9379a110d --- /dev/null +++ b/eval/prompts/system_native_augment.jinja @@ -0,0 +1,56 @@ +You are a helpful assistant that can interact with a computer to solve software engineering tasks. + +Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||). +Include a THOUGHT section before your command where you explain your reasoning process. +Format your response as shown in. + + +Your reasoning and analysis here. Explain why you want to perform the action. + +```mswea_bash_command +your_command_here +``` + + +Failure to follow these rules will cause your response to be rejected. + +## Code Intelligence + +You have **GitNexus** โ€” a knowledge graph over this entire codebase. It knows every function call chain, class hierarchy, execution flow, and symbol relationship. These are fast bash commands (~100ms). Use them when useful, skip them when a simple grep suffices. + +Your `grep` results are also automatically enriched with `[GitNexus]` annotations showing callers, callees, and execution flows for matched symbols. Pay attention to these โ€” they often point you to the right code without extra tool calls. + +### GitNexus Commands + +**gitnexus-query ""** โ€” Find execution flows related to a concept. +Returns ranked execution flow traces with participating symbols and file locations. +```bash +gitnexus-query "form field validation" +``` + +**gitnexus-context "" [""]** โ€” 360-degree view of a symbol. +Returns ALL callers, ALL callees, and execution flows. Graph-complete โ€” finds callers that grep misses. +```bash +gitnexus-context "BoundField" "django/forms/boundfield.py" +``` + +**gitnexus-impact "" [upstream|downstream]** โ€” Blast radius analysis. +What breaks if you change this: d=1 WILL BREAK, d=2 LIKELY AFFECTED, d=3 MAY NEED TESTING. +```bash +gitnexus-impact "BoundField" upstream +``` + +**gitnexus-cypher ""** โ€” Raw Cypher query against the code graph. +```bash +gitnexus-cypher 'MATCH (a)-[:CodeRelation {type: "CALLS"}]->(b:Function {name: "clean"}) RETURN a.name, a.filePath' +``` + +### When to Use What + +| I need to... | Use | +|---|---| +| Understand how a feature works end-to-end | `gitnexus-query` | +| Find ALL callers of a function | `gitnexus-context` | +| Know what breaks if I change something | `gitnexus-impact` upstream | +| Find a string literal or error message | `grep` | +| Read source code | `cat` / `nl -ba` | diff --git a/eval/pyproject.toml b/eval/pyproject.toml new file mode 100644 index 0000000000..ae9d2ad921 --- /dev/null +++ b/eval/pyproject.toml @@ -0,0 +1,35 @@ +[project] +name = "gitnexus-swebench-eval" +version = "0.1.0" +description = "SWE-bench evaluation harness with GitNexus code intelligence integration" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "mini-swe-agent>=2.0.0", + "litellm>=1.50.0", + "datasets>=3.0.0", + "typer>=0.12.0", + "rich>=13.0.0", + "pyyaml>=6.0", + "pandas>=2.0.0", + "tabulate>=0.9.0", + "python-dotenv>=1.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "ruff>=0.5.0", +] + +[project.scripts] +gitnexus-eval = "run_eval:app" +gitnexus-eval-analyze = "analysis.analyze_results:app" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.ruff] +line-length = 120 +target-version = "py311" diff --git a/eval/run_eval.py b/eval/run_eval.py new file mode 100644 index 0000000000..38dc7a473b --- /dev/null +++ b/eval/run_eval.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +""" +GitNexus SWE-bench Evaluation Runner + +Main entry point for running SWE-bench evaluations with and without GitNexus. +Supports running a single configuration or a full matrix of models x modes. + +Usage: + # Single run (default: native_augment mode โ€” GitNexus tools + grep enrichment) + python run_eval.py single -m claude-sonnet --subset lite --slice 0:5 + + # Baseline comparison (no GitNexus) + python run_eval.py single -m claude-sonnet --mode baseline --subset lite --slice 0:5 + + # Matrix run (all models x all modes) + python run_eval.py matrix --subset lite --slice 0:50 --workers 4 + + # Single instance for debugging + python run_eval.py debug -m claude-haiku -i django__django-16527 +""" + +import concurrent.futures +import json +import logging +import os +import threading +import time +import traceback +from itertools import product +from pathlib import Path +from typing import Any + +import typer +import yaml +from rich.console import Console +from rich.live import Live +from rich.table import Table + +# Load .env file from eval/ directory +_env_file = Path(__file__).parent / ".env" +if _env_file.exists(): + for line in _env_file.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" in line: + key, _, value = line.partition("=") + key, value = key.strip(), value.strip() + if value and key not in os.environ: # Don't override existing env vars + os.environ[key] = value + +logger = logging.getLogger("gitnexus_eval") +console = Console() +app = typer.Typer(rich_markup_mode="rich", add_completion=False) + +# Directory paths +EVAL_DIR = Path(__file__).parent +CONFIGS_DIR = EVAL_DIR / "configs" +MODELS_DIR = CONFIGS_DIR / "models" +MODES_DIR = CONFIGS_DIR / "modes" +DEFAULT_OUTPUT_DIR = EVAL_DIR / "results" + +# Available models and modes (discovered from config files) +AVAILABLE_MODELS = sorted([p.stem for p in MODELS_DIR.glob("*.yaml")]) +AVAILABLE_MODES = sorted([p.stem for p in MODES_DIR.glob("*.yaml")]) + +# SWE-bench dataset mapping (same as mini-swe-agent) +DATASET_MAPPING = { + "full": "princeton-nlp/SWE-Bench", + "verified": "princeton-nlp/SWE-Bench_Verified", + "lite": "princeton-nlp/SWE-Bench_Lite", +} + +_output_lock = threading.Lock() + + +def load_yaml_config(path: Path) -> dict: + """Load a YAML config file.""" + with open(path) as f: + return yaml.safe_load(f) or {} + + +def merge_configs(*configs: dict) -> dict: + """Recursively merge multiple config dicts (later values win).""" + result = {} + for config in configs: + for key, value in config.items(): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + result[key] = merge_configs(result[key], value) + else: + result[key] = value + return result + + +def build_config(model_name: str, mode_name: str) -> dict: + """Build a complete config from model + mode YAML files.""" + model_file = MODELS_DIR / f"{model_name}.yaml" + mode_file = MODES_DIR / f"{mode_name}.yaml" + + if not model_file.exists(): + raise FileNotFoundError(f"Model config not found: {model_file}") + if not mode_file.exists(): + raise FileNotFoundError(f"Mode config not found: {mode_file}") + + model_config = load_yaml_config(model_file) + mode_config = load_yaml_config(mode_file) + + return merge_configs(mode_config, model_config) + + +def load_instances(subset: str, split: str, slice_spec: str = "", filter_spec: str = "") -> list[dict]: + """Load SWE-bench instances.""" + from datasets import load_dataset + import re + + dataset_path = DATASET_MAPPING.get(subset, subset) + logger.info(f"Loading dataset: {dataset_path}, split: {split}") + instances = list(load_dataset(dataset_path, split=split)) + + if filter_spec: + instances = [i for i in instances if re.match(filter_spec, i["instance_id"])] + + if slice_spec: + values = [int(x) if x else None for x in slice_spec.split(":")] + instances = instances[slice(*values)] + + logger.info(f"Loaded {len(instances)} instances") + return instances + + +def get_swebench_docker_image(instance: dict) -> str: + """Get Docker image name for a SWE-bench instance.""" + image_name = instance.get("image_name") + if image_name is None: + iid = instance["instance_id"] + id_docker = iid.replace("__", "_1776_") + image_name = f"docker.io/swebench/sweb.eval.x86_64.{id_docker}:latest".lower() + return image_name + + +def process_instance( + instance: dict, + config: dict, + output_dir: Path, + model_name: str, + mode_name: str, +) -> dict: + """ + Process a single SWE-bench instance with the given config. + Returns result dict with instance_id, exit_status, submission, metrics. + """ + from minisweagent.models import get_model + + instance_id = instance["instance_id"] + run_id = f"{model_name}_{mode_name}" + instance_dir = output_dir / run_id / instance_id + instance_dir.mkdir(parents=True, exist_ok=True) + + result = { + "instance_id": instance_id, + "model": model_name, + "mode": mode_name, + "exit_status": None, + "submission": "", + "cost": 0.0, + "n_calls": 0, + "gitnexus_metrics": {}, + } + + agent = None + + try: + # Build model + model = get_model(config=config.get("model", {})) + + # Build environment + env_config = dict(config.get("environment", {})) + env_class_name = env_config.pop("environment_class", "docker") + + if env_class_name == "eval.environments.gitnexus_docker.GitNexusDockerEnvironment": + from eval.environments.gitnexus_docker import GitNexusDockerEnvironment + env_config["image"] = get_swebench_docker_image(instance) + env = GitNexusDockerEnvironment(**env_config) + else: + from minisweagent.environments.docker import DockerEnvironment + env = DockerEnvironment(image=get_swebench_docker_image(instance), **env_config) + + # Build agent + agent_config = dict(config.get("agent", {})) + agent_class_name = agent_config.pop("agent_class", "eval.agents.gitnexus_agent.GitNexusAgent") + + from eval.agents.gitnexus_agent import GitNexusAgent + traj_path = instance_dir / f"{instance_id}.traj.json" + agent_config["output_path"] = traj_path + agent = GitNexusAgent(model, env, **agent_config) + + # Run + logger.info(f"[{run_id}] Starting {instance_id}") + info = agent.run(instance["problem_statement"]) + + result["exit_status"] = info.get("exit_status") + result["submission"] = info.get("submission", "") + result["cost"] = agent.cost + result["n_calls"] = agent.n_calls + result["gitnexus_metrics"] = agent.gitnexus_metrics.to_dict() + + except Exception as e: + logger.error(f"[{run_id}] Error on {instance_id}: {e}") + result["exit_status"] = type(e).__name__ + result["error"] = str(e) + result["traceback"] = traceback.format_exc() + + finally: + if agent: + agent.save( + instance_dir / f"{instance_id}.traj.json", + {"instance_id": instance_id, "run_id": run_id}, + ) + + # Update predictions file + _update_preds(output_dir / run_id / "preds.json", instance_id, model_name, result) + + return result + + +def _update_preds(preds_path: Path, instance_id: str, model_name: str, result: dict): + """Thread-safe update of predictions file.""" + with _output_lock: + preds_path.parent.mkdir(parents=True, exist_ok=True) + data = {} + if preds_path.exists(): + data = json.loads(preds_path.read_text()) + data[instance_id] = { + "model_name_or_path": model_name, + "instance_id": instance_id, + "model_patch": result.get("submission", ""), + } + preds_path.write_text(json.dumps(data, indent=2)) + + +def run_configuration( + model_name: str, + mode_name: str, + instances: list[dict], + output_dir: Path, + workers: int = 1, + redo_existing: bool = False, +) -> list[dict]: + """Run a single (model, mode) configuration across all instances.""" + config = build_config(model_name, mode_name) + run_id = f"{model_name}_{mode_name}" + run_dir = output_dir / run_id + + # Skip existing instances + if not redo_existing and (run_dir / "preds.json").exists(): + existing = set(json.loads((run_dir / "preds.json").read_text()).keys()) + instances = [i for i in instances if i["instance_id"] not in existing] + if not instances: + logger.info(f"[{run_id}] All instances already completed, skipping") + return [] + + console.print(f" [bold]{run_id}[/bold]: {len(instances)} instances, {workers} workers") + + results = [] + + if workers <= 1: + for instance in instances: + result = process_instance(instance, config, output_dir, model_name, mode_name) + results.append(result) + else: + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + futures = { + executor.submit( + process_instance, instance, config, output_dir, model_name, mode_name + ): instance["instance_id"] + for instance in instances + } + for future in concurrent.futures.as_completed(futures): + try: + results.append(future.result()) + except Exception as e: + iid = futures[future] + logger.error(f"[{run_id}] Uncaught error for {iid}: {e}") + + # Save run summary + summary = { + "run_id": run_id, + "model": model_name, + "mode": mode_name, + "config": config, + "total_instances": len(results), + "completed": sum(1 for r in results if r["exit_status"] not in [None, "error"]), + "total_cost": sum(r.get("cost", 0) for r in results), + "total_api_calls": sum(r.get("n_calls", 0) for r in results), + "results": results, + } + (run_dir / "summary.json").mkdir(parents=True, exist_ok=True) if not run_dir.exists() else None + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "summary.json").write_text(json.dumps(summary, indent=2, default=str)) + + return results + + +# โ”€โ”€โ”€ CLI Commands โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +@app.command() +def single( + model: str = typer.Option(..., "-m", "--model", help=f"Model config name. Available: {', '.join(AVAILABLE_MODELS)}"), + mode: str = typer.Option("native_augment", "--mode", help=f"Evaluation mode. Available: {', '.join(AVAILABLE_MODES)}"), + subset: str = typer.Option("lite", "--subset", help="SWE-bench subset: lite, verified, full"), + split: str = typer.Option("dev", "--split", help="Dataset split"), + slice_spec: str = typer.Option("", "--slice", help="Slice spec (e.g., '0:5')"), + filter_spec: str = typer.Option("", "--filter", help="Filter instance IDs by regex"), + workers: int = typer.Option(1, "-w", "--workers", help="Parallel workers"), + output: str = typer.Option(str(DEFAULT_OUTPUT_DIR), "-o", "--output", help="Output directory"), + redo: bool = typer.Option(False, "--redo", help="Redo existing instances"), +): + """Run a single (model, mode) configuration on SWE-bench.""" + output_dir = Path(output) + instances = load_instances(subset, split, slice_spec, filter_spec) + + console.print(f"\n[bold]Running evaluation:[/bold] {model} + {mode}") + console.print(f" Instances: {len(instances)}") + console.print(f" Output: {output_dir}\n") + + results = run_configuration(model, mode, instances, output_dir, workers, redo) + + # Print summary + _print_summary(results, model, mode) + + +@app.command() +def matrix( + models: list[str] = typer.Option(AVAILABLE_MODELS, "-m", "--models", help="Models to evaluate (comma-separated or repeated)"), + modes: list[str] = typer.Option(AVAILABLE_MODES, "--modes", help="Modes to evaluate"), + subset: str = typer.Option("lite", "--subset", help="SWE-bench subset"), + split: str = typer.Option("dev", "--split", help="Dataset split"), + slice_spec: str = typer.Option("", "--slice", help="Slice spec"), + filter_spec: str = typer.Option("", "--filter", help="Filter instances by regex"), + workers: int = typer.Option(1, "-w", "--workers", help="Parallel workers per config"), + output: str = typer.Option(str(DEFAULT_OUTPUT_DIR), "-o", "--output", help="Output directory"), + redo: bool = typer.Option(False, "--redo", help="Redo existing instances"), +): + """Run the full evaluation matrix: all models x all modes.""" + output_dir = Path(output) + instances = load_instances(subset, split, slice_spec, filter_spec) + + combos = list(product(models, modes)) + console.print(f"\n[bold]Matrix evaluation:[/bold] {len(models)} models x {len(modes)} modes = {len(combos)} configs") + console.print(f" Models: {', '.join(models)}") + console.print(f" Modes: {', '.join(modes)}") + console.print(f" Instances per config: {len(instances)}") + console.print(f" Total runs: {len(combos) * len(instances)}") + console.print(f" Output: {output_dir}\n") + + all_results = {} + for model_name, mode_name in combos: + run_id = f"{model_name}_{mode_name}" + console.print(f"\n[bold cyan]โ”โ”โ” {run_id} โ”โ”โ”[/bold cyan]") + results = run_configuration(model_name, mode_name, instances, output_dir, workers, redo) + all_results[run_id] = results + + # Print comparative summary + _print_matrix_summary(all_results) + + # Save master summary + master = { + "timestamp": time.time(), + "models": models, + "modes": modes, + "subset": subset, + "n_instances": len(instances), + "runs": { + run_id: { + "total": len(results), + "cost": sum(r.get("cost", 0) for r in results), + "api_calls": sum(r.get("n_calls", 0) for r in results), + } + for run_id, results in all_results.items() + }, + } + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "matrix_summary.json").write_text(json.dumps(master, indent=2, default=str)) + console.print(f"\n[green]Results saved to {output_dir}[/green]") + + +@app.command() +def debug( + model: str = typer.Option("claude-haiku", "-m", "--model", help="Model config name"), + mode: str = typer.Option("native_augment", "--mode", help="Evaluation mode"), + instance_id: str = typer.Option(..., "-i", "--instance", help="SWE-bench instance ID"), + subset: str = typer.Option("lite", "--subset", help="SWE-bench subset"), + split: str = typer.Option("dev", "--split"), + output: str = typer.Option(str(DEFAULT_OUTPUT_DIR / "debug"), "-o", "--output"), +): + """Debug a single SWE-bench instance.""" + from datasets import load_dataset + + dataset_path = DATASET_MAPPING.get(subset, subset) + instances = {inst["instance_id"]: inst for inst in load_dataset(dataset_path, split=split)} + + if instance_id not in instances: + console.print(f"[red]Instance '{instance_id}' not found in {subset}/{split}[/red]") + raise typer.Exit(1) + + instance = instances[instance_id] + config = build_config(model, mode) + output_dir = Path(output) + + console.print(f"\n[bold]Debug run:[/bold] {model} + {mode}") + console.print(f" Instance: {instance_id}") + console.print(f" Problem: {instance['problem_statement'][:200]}...\n") + + result = process_instance(instance, config, output_dir, model, mode) + _print_summary([result], model, mode) + + +@app.command() +def list_configs(): + """List available model and mode configurations.""" + console.print("\n[bold]Available Models:[/bold]") + for name in AVAILABLE_MODELS: + config = load_yaml_config(MODELS_DIR / f"{name}.yaml") + model_name = config.get("model", {}).get("model_name", "unknown") + console.print(f" {name:<20} {model_name}") + + console.print("\n[bold]Available Modes:[/bold]") + for name in AVAILABLE_MODES: + config = load_yaml_config(MODES_DIR / f"{name}.yaml") + gn_mode = config.get("agent", {}).get("gitnexus_mode", "baseline") + console.print(f" {name:<20} gitnexus_mode={gn_mode}") + + console.print(f"\n[bold]Matrix:[/bold] {len(AVAILABLE_MODELS)} models x {len(AVAILABLE_MODES)} modes = {len(AVAILABLE_MODELS) * len(AVAILABLE_MODES)} configurations") + + +# โ”€โ”€โ”€ Summary Output โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +def _print_summary(results: list[dict], model: str, mode: str): + """Print a summary table for a single run.""" + if not results: + console.print("[yellow]No results to display[/yellow]") + return + + table = Table(title=f"{model} + {mode}") + table.add_column("Metric", style="bold") + table.add_column("Value") + + total = len(results) + completed = sum(1 for r in results if r.get("submission")) + total_cost = sum(r.get("cost", 0) for r in results) + total_calls = sum(r.get("n_calls", 0) for r in results) + + table.add_row("Instances", str(total)) + table.add_row("Completed", f"{completed}/{total}") + table.add_row("Total Cost", f"${total_cost:.4f}") + table.add_row("Total API Calls", str(total_calls)) + table.add_row("Avg Cost/Instance", f"${total_cost / max(total, 1):.4f}") + table.add_row("Avg Calls/Instance", f"{total_calls / max(total, 1):.1f}") + + # GitNexus-specific metrics + gn_tool_calls = sum( + r.get("gitnexus_metrics", {}).get("total_tool_calls", 0) for r in results + ) + gn_augment_hits = sum( + r.get("gitnexus_metrics", {}).get("augmentation_hits", 0) for r in results + ) + if gn_tool_calls > 0: + table.add_row("GitNexus Tool Calls", str(gn_tool_calls)) + if gn_augment_hits > 0: + table.add_row("Augmentation Hits", str(gn_augment_hits)) + + console.print(table) + + +def _print_matrix_summary(all_results: dict[str, list[dict]]): + """Print a comparative matrix summary.""" + table = Table(title="Evaluation Matrix Summary") + table.add_column("Configuration", style="bold") + table.add_column("Instances") + table.add_column("Completed") + table.add_column("Cost") + table.add_column("API Calls") + table.add_column("GN Tools") + + for run_id, results in sorted(all_results.items()): + total = len(results) + completed = sum(1 for r in results if r.get("submission")) + cost = sum(r.get("cost", 0) for r in results) + calls = sum(r.get("n_calls", 0) for r in results) + gn_calls = sum(r.get("gitnexus_metrics", {}).get("total_tool_calls", 0) for r in results) + + table.add_row( + run_id, + str(total), + f"{completed}/{total}", + f"${cost:.2f}", + str(calls), + str(gn_calls) if gn_calls > 0 else "-", + ) + + console.print(table) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(name)s] %(message)s") + app() diff --git a/gitnexus/README.md b/gitnexus/README.md index 4219303b1c..4acf9d36da 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -18,31 +18,37 @@ AI coding tools don't understand your codebase structure. They edit a function w ## Quick Start ```bash -# Install -npm install -g gitnexus - -# One-time: configure MCP for your editors -gitnexus setup - -# Index your repository (run from repo root) -gitnexus analyze - -# Done! Open your editor โ€” MCP connects automatically. +# Index your repo (run from repo root) +npx gitnexus analyze ``` -Or without installing globally: +That's it. This indexes the codebase, installs agent skills, registers Claude Code hooks, and creates `AGENTS.md` / `CLAUDE.md` context files โ€” all in one command. -```bash -npx gitnexus setup # one-time -npx gitnexus analyze # per repo -``` +To configure MCP for your editor, run `npx gitnexus setup` once โ€” or set it up manually below. -The `setup` command auto-detects Cursor, Claude Code, and OpenCode, then writes the correct global MCP config. You only run it once. +`gitnexus setup` auto-detects your editors and writes the correct global MCP config. You only need to run it once. + +### Editor Support + +| Editor | MCP | Skills | Hooks (auto-augment) | Support | +|--------|-----|--------|---------------------|---------| +| **Claude Code** | Yes | Yes | Yes (PreToolUse) | **Full** | +| **Cursor** | Yes | Yes | โ€” | MCP + Skills | +| **Windsurf** | Yes | โ€” | โ€” | MCP | +| **OpenCode** | Yes | Yes | โ€” | MCP + Skills | + +> **Claude Code** gets the deepest integration: MCP tools + agent skills + PreToolUse hooks that automatically enrich grep/glob/bash calls with knowledge graph context. ## MCP Setup (manual) If you prefer to configure manually instead of using `gitnexus setup`: +### Claude Code (full support โ€” MCP + skills + hooks) + +```bash +claude mcp add gitnexus -- npx -y gitnexus@latest mcp +``` + ### Cursor / Windsurf Add to `~/.cursor/mcp.json` (global โ€” works for all projects): @@ -58,12 +64,6 @@ Add to `~/.cursor/mcp.json` (global โ€” works for all projects): } ``` -### Claude Code - -```bash -claude mcp add gitnexus -- npx -y gitnexus@latest mcp -``` - ### OpenCode Add to `~/.config/opencode/config.json`: @@ -100,68 +100,59 @@ Your AI agent gets these tools automatically: | Tool | What It Does | `repo` Param | |------|-------------|--------------| | `list_repos` | Discover all indexed repositories | โ€” | -| `search` | Hybrid search (BM25 + semantic) with cluster context | Optional | -| `overview` | List all clusters and processes | Optional | -| `explore` | Deep dive on a symbol, cluster, or process | Optional | -| `impact` | Blast radius analysis | Optional | +| `query` | Process-grouped hybrid search (BM25 + semantic + RRF) | Optional | +| `context` | 360-degree symbol view โ€” categorized refs, process participation | Optional | +| `impact` | Blast radius analysis with depth grouping and confidence | Optional | +| `detect_changes` | Git-diff impact โ€” maps changed lines to affected processes | Optional | +| `rename` | Multi-file coordinated rename with graph + text search | Optional | | `cypher` | Raw Cypher graph queries | Optional | -| `analyze` | Index or re-index a repository | Optional | -> With one indexed repo, the `repo` param is optional. With multiple, specify which: `search({query: "auth", repo: "my-app"})`. +> With one indexed repo, the `repo` param is optional. With multiple, specify which: `query({query: "auth", repo: "my-app"})`. ## MCP Resources | Resource | Purpose | |----------|---------| | `gitnexus://repos` | List all indexed repositories (read first) | -| `gitnexus://repo/{name}/context` | Codebase stats and overview | -| `gitnexus://repo/{name}/clusters` | All functional clusters | +| `gitnexus://repo/{name}/context` | Codebase stats, staleness check, and available tools | +| `gitnexus://repo/{name}/clusters` | All functional clusters with cohesion scores | | `gitnexus://repo/{name}/cluster/{name}` | Cluster members and details | | `gitnexus://repo/{name}/processes` | All execution flows | -| `gitnexus://repo/{name}/process/{name}` | Full process trace | +| `gitnexus://repo/{name}/process/{name}` | Full process trace with steps | | `gitnexus://repo/{name}/schema` | Graph schema for Cypher queries | +## MCP Prompts + +| Prompt | What It Does | +|--------|-------------| +| `detect_impact` | Pre-commit change analysis โ€” scope, affected processes, risk level | +| `generate_map` | Architecture documentation from the knowledge graph with mermaid diagrams | + ## CLI Commands ```bash -gitnexus setup # Configure MCP for your editors (one-time) -gitnexus analyze [path] # Index a repository (or update stale index) -gitnexus analyze --force # Force full re-index -gitnexus mcp # Start MCP server (stdio) โ€” serves all indexed repos -gitnexus serve # Start HTTP server for web UI -gitnexus list # List all indexed repositories -gitnexus status # Show index status for current repo -gitnexus clean # Delete index for current repo -gitnexus clean --all # Delete all indexes +gitnexus setup # Configure MCP for your editors (one-time) +gitnexus analyze [path] # Index a repository (or update stale index) +gitnexus analyze --force # Force full re-index +gitnexus analyze --skip-embeddings # Skip embedding generation (faster) +gitnexus mcp # Start MCP server (stdio) โ€” serves all indexed repos +gitnexus serve # Start HTTP server for web UI +gitnexus list # List all indexed repositories +gitnexus status # Show index status for current repo +gitnexus clean # Delete index for current repo +gitnexus clean --all --force # Delete all indexes +gitnexus wiki [path] # Generate LLM-powered docs from knowledge graph +gitnexus wiki --model # Wiki with custom LLM model (default: gpt-4o-mini) ``` ## Multi-Repo Support -GitNexus supports indexing multiple repositories. Each `gitnexus analyze` registers the repo in a global registry (`~/.gitnexus/registry.json`). The MCP server serves all indexed repos automatically with lazy KuzuDB connections. +GitNexus supports indexing multiple repositories. Each `gitnexus analyze` registers the repo in a global registry (`~/.gitnexus/registry.json`). The MCP server serves all indexed repos automatically with lazy KuzuDB connections (max 5 concurrent, evicted after 5 minutes idle). ## Supported Languages TypeScript, JavaScript, Python, Java, C, C++, C#, Go, Rust -## How Impact Analysis Works - -``` -gitnexus_impact({target: "UserService", direction: "upstream", repo: "my-app"}) - -TARGET: Class UserService (src/services/user.ts) - -UPSTREAM (what depends on this): - Depth 1 (direct callers): - handleLogin [CALLS 90%] โ†’ src/api/auth.ts:45 - handleRegister [CALLS 90%] โ†’ src/api/auth.ts:78 - Depth 2: - authRouter [IMPORTS] โ†’ src/routes/auth.ts - -8 files affected, 3 clusters touched -``` - -Options: `maxDepth`, `minConfidence`, `relationTypes`, `includeTests` - ## Agent Skills GitNexus ships with skill files that teach AI agents how to use the tools effectively: @@ -171,7 +162,7 @@ GitNexus ships with skill files that teach AI agents how to use the tools effect - **Impact Analysis** โ€” Analyze blast radius before changes - **Refactoring** โ€” Plan safe refactors using dependency mapping -These are installed automatically to `.claude/skills/` when you run `gitnexus analyze`. +Installed automatically by both `gitnexus analyze` (per-repo) and `gitnexus setup` (global). ## Requirements diff --git a/gitnexus/_hook_test.json b/gitnexus/_hook_test.json deleted file mode 100644 index 2337c7e110..0000000000 --- a/gitnexus/_hook_test.json +++ /dev/null @@ -1 +0,0 @@ -{"hook_event_name":"PreToolUse","tool_name":"Grep","tool_input":{"pattern":"augment"},"cwd":"d:\\\\Projects\\\\GitnexusV2"} \ No newline at end of file diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 7d91490087..7139c9a085 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -10,7 +10,7 @@ import { runPipelineFromRepo } from '../core/ingestion/pipeline.js'; import { initKuzu, loadGraphToKuzu, getKuzuStats, executeQuery, executeWithReusedStatement, closeKuzu, createFTSIndex, loadCachedEmbeddings } from '../core/kuzu/kuzu-adapter.js'; import { runEmbeddingPipeline } from '../core/embeddings/embedding-pipeline.js'; import { disposeEmbedder } from '../core/embeddings/embedder.js'; -import { getStoragePaths, saveMeta, loadMeta, addToGitignore, registerRepo, getGlobalRegistryPath, getGlobalDir } from '../storage/repo-manager.js'; +import { getStoragePaths, saveMeta, loadMeta, addToGitignore, registerRepo, getGlobalRegistryPath } from '../storage/repo-manager.js'; import { getCurrentCommit, isGitRepo, getGitRoot } from '../storage/git.js'; import { generateAIContextFiles } from './ai-context.js'; import fs from 'fs/promises'; diff --git a/gitnexus/src/cli/eval-server.ts b/gitnexus/src/cli/eval-server.ts new file mode 100644 index 0000000000..171fb07412 --- /dev/null +++ b/gitnexus/src/cli/eval-server.ts @@ -0,0 +1,430 @@ +/** + * Eval Server โ€” Lightweight HTTP server for SWE-bench evaluation + * + * Keeps KuzuDB warm in memory so tool calls from the agent are near-instant. + * Designed to run inside Docker containers during SWE-bench evaluation. + * + * KEY DESIGN: Returns LLM-friendly text, not raw JSON. + * Raw JSON wastes tokens and is hard for models to parse. The text formatter + * converts structured results into compact, readable output that models + * can immediately act on. Next-step hints guide the agent through a + * productive tool-chaining workflow (query โ†’ context โ†’ impact โ†’ fix). + * + * Architecture: + * Agent bash cmd โ†’ curl localhost:PORT/tool/query โ†’ eval-server โ†’ LocalBackend โ†’ format โ†’ text + * + * Usage: + * gitnexus eval-server # default port 4848 + * gitnexus eval-server --port 4848 # explicit port + * gitnexus eval-server --idle-timeout 300 # auto-shutdown after 300s idle + * + * API: + * POST /tool/:name โ€” Call a tool. Body is JSON arguments. Returns formatted text. + * GET /health โ€” Health check. Returns {"status":"ok","repos":[...]} + * POST /shutdown โ€” Graceful shutdown. + */ + +import http from 'http'; +import { LocalBackend } from '../mcp/local/local-backend.js'; + +export interface EvalServerOptions { + port?: string; + idleTimeout?: string; +} + +// โ”€โ”€โ”€ Text Formatters โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Convert structured JSON results into compact, LLM-friendly text. +// Design: minimize tokens, maximize actionability. + +function formatQueryResult(result: any): string { + if (result.error) return `Error: ${result.error}`; + + const lines: string[] = []; + const processes = result.processes || []; + const symbols = result.process_symbols || []; + const defs = result.definitions || []; + + if (processes.length === 0 && defs.length === 0) { + return 'No matching execution flows found. Try a different search term or use grep.'; + } + + lines.push(`Found ${processes.length} execution flow(s):\n`); + + for (let i = 0; i < processes.length; i++) { + const p = processes[i]; + lines.push(`${i + 1}. ${p.summary} (${p.step_count} steps, ${p.symbol_count} symbols)`); + + // Show symbols belonging to this process + const procSymbols = symbols.filter((s: any) => s.process_id === p.id); + for (const s of procSymbols.slice(0, 6)) { + const loc = s.startLine ? `:${s.startLine}` : ''; + lines.push(` ${s.type} ${s.name} โ†’ ${s.filePath}${loc}`); + } + if (procSymbols.length > 6) { + lines.push(` ... and ${procSymbols.length - 6} more`); + } + lines.push(''); + } + + if (defs.length > 0) { + lines.push(`Standalone definitions:`); + for (const d of defs.slice(0, 8)) { + lines.push(` ${d.type || 'Symbol'} ${d.name} โ†’ ${d.filePath || '?'}`); + } + if (defs.length > 8) lines.push(` ... and ${defs.length - 8} more`); + } + + return lines.join('\n').trim(); +} + +function formatContextResult(result: any): string { + if (result.error) return `Error: ${result.error}`; + + if (result.status === 'ambiguous') { + const lines = [`Multiple symbols named '${result.candidates?.[0]?.name || '?'}'. Disambiguate with file path:\n`]; + for (const c of result.candidates || []) { + lines.push(` ${c.kind} ${c.name} โ†’ ${c.filePath}:${c.line || '?'} (uid: ${c.uid})`); + } + lines.push(`\nRe-run: gitnexus-context "${result.candidates?.[0]?.name}" ""`); + return lines.join('\n'); + } + + const sym = result.symbol; + if (!sym) return 'Symbol not found.'; + + const lines: string[] = []; + const loc = sym.startLine ? `:${sym.startLine}-${sym.endLine}` : ''; + lines.push(`${sym.kind} ${sym.name} โ†’ ${sym.filePath}${loc}`); + lines.push(''); + + // Incoming refs (who calls/imports/extends this) + const incoming = result.incoming || {}; + const incomingCount = Object.values(incoming).reduce((sum: number, arr: any) => sum + arr.length, 0) as number; + if (incomingCount > 0) { + lines.push(`Called/imported by (${incomingCount}):`); + for (const [relType, refs] of Object.entries(incoming)) { + for (const ref of (refs as any[]).slice(0, 10)) { + lines.push(` โ† [${relType}] ${ref.kind} ${ref.name} โ†’ ${ref.filePath}`); + } + } + lines.push(''); + } + + // Outgoing refs (what this calls/imports) + const outgoing = result.outgoing || {}; + const outgoingCount = Object.values(outgoing).reduce((sum: number, arr: any) => sum + arr.length, 0) as number; + if (outgoingCount > 0) { + lines.push(`Calls/imports (${outgoingCount}):`); + for (const [relType, refs] of Object.entries(outgoing)) { + for (const ref of (refs as any[]).slice(0, 10)) { + lines.push(` โ†’ [${relType}] ${ref.kind} ${ref.name} โ†’ ${ref.filePath}`); + } + } + lines.push(''); + } + + // Processes + const procs = result.processes || []; + if (procs.length > 0) { + lines.push(`Participates in ${procs.length} execution flow(s):`); + for (const p of procs) { + lines.push(` โ€ข ${p.name} (step ${p.step_index}/${p.step_count})`); + } + } + + if (sym.content) { + lines.push(''); + lines.push(`Source:`); + lines.push(sym.content); + } + + return lines.join('\n').trim(); +} + +function formatImpactResult(result: any): string { + if (result.error) return `Error: ${result.error}`; + + const target = result.target; + const direction = result.direction; + const byDepth = result.byDepth || {}; + const total = result.impactedCount || 0; + + if (total === 0) { + return `${target?.name || '?'}: No ${direction} dependencies found. This symbol appears isolated.`; + } + + const lines: string[] = []; + const dirLabel = direction === 'upstream' ? 'depends on this (will break if changed)' : 'this depends on'; + lines.push(`Blast radius for ${target?.kind || ''} ${target?.name} (${direction}): ${total} symbol(s) ${dirLabel}\n`); + + const depthLabels: Record = { + 1: 'WILL BREAK (direct)', + 2: 'LIKELY AFFECTED (indirect)', + 3: 'MAY NEED TESTING (transitive)', + }; + + for (const depth of [1, 2, 3]) { + const items = byDepth[depth]; + if (!items || items.length === 0) continue; + + lines.push(`d=${depth}: ${depthLabels[depth] || ''} (${items.length})`); + for (const item of items.slice(0, 12)) { + const conf = item.confidence < 1 ? ` (conf: ${item.confidence})` : ''; + lines.push(` ${item.type} ${item.name} โ†’ ${item.filePath} [${item.relationType}]${conf}`); + } + if (items.length > 12) { + lines.push(` ... and ${items.length - 12} more`); + } + lines.push(''); + } + + return lines.join('\n').trim(); +} + +function formatCypherResult(result: any): string { + if (result.error) return `Error: ${result.error}`; + + if (Array.isArray(result)) { + if (result.length === 0) return 'Query returned 0 rows.'; + // Format as simple table + const keys = Object.keys(result[0]); + const lines: string[] = [`${result.length} row(s):\n`]; + for (const row of result.slice(0, 30)) { + const parts = keys.map(k => `${k}: ${row[k]}`); + lines.push(` ${parts.join(' | ')}`); + } + if (result.length > 30) { + lines.push(` ... ${result.length - 30} more rows`); + } + return lines.join('\n'); + } + + return typeof result === 'string' ? result : JSON.stringify(result, null, 2); +} + +function formatDetectChangesResult(result: any): string { + if (result.error) return `Error: ${result.error}`; + + const summary = result.summary || {}; + const lines: string[] = []; + + if (summary.changed_count === 0) { + return 'No changes detected.'; + } + + lines.push(`Changes: ${summary.changed_files || 0} files, ${summary.changed_count || 0} symbols`); + lines.push(`Affected processes: ${summary.affected_count || 0}`); + lines.push(`Risk level: ${summary.risk_level || 'unknown'}\n`); + + const changed = result.changed_symbols || []; + if (changed.length > 0) { + lines.push(`Changed symbols:`); + for (const s of changed.slice(0, 15)) { + lines.push(` ${s.type} ${s.name} โ†’ ${s.filePath}`); + } + if (changed.length > 15) lines.push(` ... and ${changed.length - 15} more`); + lines.push(''); + } + + const affected = result.affected_processes || []; + if (affected.length > 0) { + lines.push(`Affected execution flows:`); + for (const p of affected.slice(0, 10)) { + const steps = (p.changed_steps || []).map((s: any) => s.symbol).join(', '); + lines.push(` โ€ข ${p.name} (${p.step_count} steps) โ€” changed: ${steps}`); + } + } + + return lines.join('\n').trim(); +} + +function formatListReposResult(result: any): string { + if (!Array.isArray(result) || result.length === 0) { + return 'No indexed repositories.'; + } + + const lines = ['Indexed repositories:\n']; + for (const r of result) { + const stats = r.stats || {}; + lines.push(` ${r.name} โ€” ${stats.nodes || '?'} symbols, ${stats.edges || '?'} relationships, ${stats.processes || '?'} flows`); + lines.push(` Path: ${r.path}`); + lines.push(` Indexed: ${r.indexedAt}`); + } + return lines.join('\n'); +} + +/** + * Format a tool result as compact, LLM-friendly text. + */ +function formatToolResult(toolName: string, result: any): string { + switch (toolName) { + case 'query': return formatQueryResult(result); + case 'context': return formatContextResult(result); + case 'impact': return formatImpactResult(result); + case 'cypher': return formatCypherResult(result); + case 'detect_changes': return formatDetectChangesResult(result); + case 'list_repos': return formatListReposResult(result); + default: return typeof result === 'string' ? result : JSON.stringify(result, null, 2); + } +} + +// โ”€โ”€โ”€ Next-Step Hints โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Guide the agent to the logical next tool call. +// Critical for tool chaining: query โ†’ context โ†’ impact โ†’ fix. + +function getNextStepHint(toolName: string): string { + switch (toolName) { + case 'query': + return '\n---\nNext: Pick a symbol above and run gitnexus-context "" to see all its callers, callees, and execution flows.'; + + case 'context': + return '\n---\nNext: To check what breaks if you change this, run gitnexus-impact "" upstream'; + + case 'impact': + return '\n---\nNext: Review d=1 items first (WILL BREAK). Read the source with cat to understand the code, then make your fix.'; + + case 'cypher': + return '\n---\nNext: To explore a result symbol in depth, run gitnexus-context ""'; + + case 'detect_changes': + return '\n---\nNext: Run gitnexus-context "" on high-risk changed symbols to check their callers.'; + + default: + return ''; + } +} + +// โ”€โ”€โ”€ Server โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +export async function evalServerCommand(options?: EvalServerOptions): Promise { + const port = parseInt(options?.port || '4848'); + const idleTimeoutSec = parseInt(options?.idleTimeout || '0'); + + const backend = new LocalBackend(); + const ok = await backend.init(); + + if (!ok) { + console.error('GitNexus eval-server: No indexed repositories found. Run: gitnexus analyze'); + process.exit(1); + } + + const repos = backend.listRepos(); + console.error(`GitNexus eval-server: ${repos.length} repo(s) loaded: ${repos.map(r => r.name).join(', ')}`); + + let idleTimer: ReturnType | null = null; + + function resetIdleTimer() { + if (idleTimeoutSec <= 0) return; + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout(async () => { + console.error('GitNexus eval-server: Idle timeout reached, shutting down'); + await backend.disconnect(); + process.exit(0); + }, idleTimeoutSec * 1000); + } + + const server = http.createServer(async (req, res) => { + resetIdleTimer(); + + try { + // Health check + if (req.method === 'GET' && req.url === '/health') { + res.setHeader('Content-Type', 'application/json'); + res.writeHead(200); + res.end(JSON.stringify({ status: 'ok', repos: repos.map(r => r.name) })); + return; + } + + // Shutdown + if (req.method === 'POST' && req.url === '/shutdown') { + res.setHeader('Content-Type', 'application/json'); + res.writeHead(200); + res.end(JSON.stringify({ status: 'shutting_down' })); + setTimeout(async () => { + await backend.disconnect(); + server.close(); + process.exit(0); + }, 100); + return; + } + + // Tool calls: POST /tool/:name + const toolMatch = req.url?.match(/^\/tool\/(\w+)$/); + if (req.method === 'POST' && toolMatch) { + const toolName = toolMatch[1]; + + const body = await readBody(req); + let args: Record = {}; + if (body.trim()) { + try { + args = JSON.parse(body); + } catch { + res.setHeader('Content-Type', 'text/plain'); + res.writeHead(400); + res.end('Error: Invalid JSON body'); + return; + } + } + + // Call tool, format result as text, append next-step hint + const result = await backend.callTool(toolName, args); + const formatted = formatToolResult(toolName, result); + const hint = getNextStepHint(toolName); + + res.setHeader('Content-Type', 'text/plain'); + res.writeHead(200); + res.end(formatted + hint); + return; + } + + // 404 + res.setHeader('Content-Type', 'text/plain'); + res.writeHead(404); + res.end('Not found. Use POST /tool/:name or GET /health'); + + } catch (err: any) { + res.setHeader('Content-Type', 'text/plain'); + res.writeHead(500); + res.end(`Error: ${err.message || 'Internal error'}`); + } + }); + + server.listen(port, '127.0.0.1', () => { + console.error(`GitNexus eval-server: listening on http://127.0.0.1:${port}`); + console.error(` POST /tool/query โ€” search execution flows`); + console.error(` POST /tool/context โ€” 360-degree symbol view`); + console.error(` POST /tool/impact โ€” blast radius analysis`); + console.error(` POST /tool/cypher โ€” raw Cypher query`); + console.error(` GET /health โ€” health check`); + console.error(` POST /shutdown โ€” graceful shutdown`); + if (idleTimeoutSec > 0) { + console.error(` Auto-shutdown after ${idleTimeoutSec}s idle`); + } + try { + process.stdout.write(`GITNEXUS_EVAL_SERVER_READY:${port}\n`); + } catch { + // stdout may not be available + } + }); + + resetIdleTimer(); + + const shutdown = async () => { + console.error('GitNexus eval-server: shutting down...'); + await backend.disconnect(); + server.close(); + process.exit(0); + }; + + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); +} + +function readBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8'))); + req.on('error', reject); + }); +} diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 0052ae0b8c..de326fb1cb 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -8,12 +8,15 @@ import { mcpCommand } from './mcp.js'; import { cleanCommand } from './clean.js'; import { setupCommand } from './setup.js'; import { augmentCommand } from './augment.js'; +import { wikiCommand } from './wiki.js'; +import { queryCommand, contextCommand, impactCommand, cypherCommand } from './tool.js'; +import { evalServerCommand } from './eval-server.js'; const program = new Command(); program .name('gitnexus') .description('GitNexus local CLI and MCP server') - .version('1.1.1'); + .version('1.1.9'); program .command('setup') @@ -55,9 +58,64 @@ program .option('--all', 'Clean all indexed repos') .action(cleanCommand); +program + .command('wiki [path]') + .description('Generate repository wiki from knowledge graph') + .option('-f, --force', 'Force full regeneration even if up to date') + .option('--model ', 'LLM model name (default: gpt-4o-mini)') + .option('--base-url ', 'LLM API base URL (default: OpenAI)') + .option('--api-key ', 'LLM API key (saved to ~/.gitnexus/config.json)') + .action(wikiCommand); + program .command('augment ') .description('Augment a search pattern with knowledge graph context (used by hooks)') .action(augmentCommand); +// โ”€โ”€โ”€ Direct Tool Commands (no MCP overhead) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// These invoke LocalBackend directly for use in eval, scripts, and CI. + +program + .command('query ') + .description('Search the knowledge graph for execution flows related to a concept') + .option('-r, --repo ', 'Target repository (omit if only one indexed)') + .option('-c, --context ', 'Task context to improve ranking') + .option('-g, --goal ', 'What you want to find') + .option('-l, --limit ', 'Max processes to return (default: 5)') + .option('--content', 'Include full symbol source code') + .action(queryCommand); + +program + .command('context [name]') + .description('360-degree view of a code symbol: callers, callees, processes') + .option('-r, --repo ', 'Target repository') + .option('-u, --uid ', 'Direct symbol UID (zero-ambiguity lookup)') + .option('-f, --file ', 'File path to disambiguate common names') + .option('--content', 'Include full symbol source code') + .action(contextCommand); + +program + .command('impact ') + .description('Blast radius analysis: what breaks if you change a symbol') + .option('-d, --direction ', 'upstream (dependants) or downstream (dependencies)', 'upstream') + .option('-r, --repo ', 'Target repository') + .option('--depth ', 'Max relationship depth (default: 3)') + .option('--include-tests', 'Include test files in results') + .action(impactCommand); + +program + .command('cypher ') + .description('Execute raw Cypher query against the knowledge graph') + .option('-r, --repo ', 'Target repository') + .action(cypherCommand); + +// โ”€โ”€โ”€ Eval Server (persistent daemon for SWE-bench) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +program + .command('eval-server') + .description('Start lightweight HTTP server for fast tool calls during evaluation') + .option('-p, --port ', 'Port number', '4848') + .option('--idle-timeout ', 'Auto-shutdown after N seconds idle (0 = disabled)', '0') + .action(evalServerCommand); + program.parse(process.argv); diff --git a/gitnexus/src/cli/mcp.ts b/gitnexus/src/cli/mcp.ts index e4152d6069..be373826b7 100644 --- a/gitnexus/src/cli/mcp.ts +++ b/gitnexus/src/cli/mcp.ts @@ -11,6 +11,16 @@ import { LocalBackend } from '../mcp/local/local-backend.js'; import { listRegisteredRepos } from '../storage/repo-manager.js'; export const mcpCommand = async () => { + // Prevent unhandled errors from crashing the MCP server process. + // KuzuDB lock conflicts and transient errors should degrade gracefully. + process.on('uncaughtException', (err) => { + console.error(`GitNexus MCP: uncaught exception โ€” ${err.message}`); + }); + process.on('unhandledRejection', (reason) => { + const msg = reason instanceof Error ? reason.message : String(reason); + console.error(`GitNexus MCP: unhandled rejection โ€” ${msg}`); + }); + // Load all registered repos const entries = await listRegisteredRepos({ validate: true }); diff --git a/gitnexus/src/cli/serve.ts b/gitnexus/src/cli/serve.ts index ef43cf9aea..8cde8d6310 100644 --- a/gitnexus/src/cli/serve.ts +++ b/gitnexus/src/cli/serve.ts @@ -5,5 +5,3 @@ export const serveCommand = async (options?: { port?: string }) => { await createServer(port); }; - - diff --git a/gitnexus/src/cli/setup.ts b/gitnexus/src/cli/setup.ts index becdd74e65..77515a49a8 100644 --- a/gitnexus/src/cli/setup.ts +++ b/gitnexus/src/cli/setup.ts @@ -223,28 +223,65 @@ const SKILL_NAMES = ['exploring', 'debugging', 'impact-analysis', 'refactoring'] * Install GitNexus skills to a target directory. * Each skill is installed as {targetDir}/gitnexus-{skillName}/SKILL.md * following the Agent Skills standard (both Cursor and Claude Code). + * + * Supports two source layouts: + * - Flat file: skills/{name}.md โ†’ copied as SKILL.md + * - Directory: skills/{name}/SKILL.md โ†’ copied recursively (includes references/, etc.) */ async function installSkillsTo(targetDir: string): Promise { const installed: string[] = []; - + const skillsRoot = path.join(__dirname, '..', '..', 'skills'); + for (const skillName of SKILL_NAMES) { - const sourcePath = path.join(__dirname, '..', '..', 'skills', `${skillName}.md`); const skillDir = path.join(targetDir, `gitnexus-${skillName}`); - const destPath = path.join(skillDir, 'SKILL.md'); - + try { - const content = await fs.readFile(sourcePath, 'utf-8'); - await fs.mkdir(skillDir, { recursive: true }); - await fs.writeFile(destPath, content, 'utf-8'); - installed.push(skillName); + // Try directory-based skill first (skills/{name}/SKILL.md) + const dirSource = path.join(skillsRoot, skillName); + const dirSkillFile = path.join(dirSource, 'SKILL.md'); + + let isDirectory = false; + try { + const stat = await fs.stat(dirSource); + isDirectory = stat.isDirectory(); + } catch { /* not a directory */ } + + if (isDirectory) { + await copyDirRecursive(dirSource, skillDir); + installed.push(skillName); + } else { + // Fall back to flat file (skills/{name}.md) + const flatSource = path.join(skillsRoot, `${skillName}.md`); + const content = await fs.readFile(flatSource, 'utf-8'); + await fs.mkdir(skillDir, { recursive: true }); + await fs.writeFile(path.join(skillDir, 'SKILL.md'), content, 'utf-8'); + installed.push(skillName); + } } catch { - // Source skill file not found โ€” skip + // Source skill not found โ€” skip } } - + return installed; } +/** + * Recursively copy a directory tree. + */ +async function copyDirRecursive(src: string, dest: string): Promise { + await fs.mkdir(dest, { recursive: true }); + const entries = await fs.readdir(src, { withFileTypes: true }); + for (const entry of entries) { + const srcPath = path.join(src, entry.name); + const destPath = path.join(dest, entry.name); + if (entry.isDirectory()) { + await copyDirRecursive(srcPath, destPath); + } else { + await fs.copyFile(srcPath, destPath); + } + } +} + /** * Install global Cursor skills to ~/.cursor/skills/gitnexus/ */ diff --git a/gitnexus/src/cli/tool.ts b/gitnexus/src/cli/tool.ts new file mode 100644 index 0000000000..41997d80b5 --- /dev/null +++ b/gitnexus/src/cli/tool.ts @@ -0,0 +1,120 @@ +/** + * Direct CLI Tool Commands + * + * Exposes GitNexus tools (query, context, impact, cypher) as direct CLI commands. + * Bypasses MCP entirely โ€” invokes LocalBackend directly for minimal overhead. + * + * Usage: + * gitnexus query "authentication flow" + * gitnexus context --name "validateUser" + * gitnexus impact --target "AuthService" --direction upstream + * gitnexus cypher "MATCH (n:Function) RETURN n.name LIMIT 10" + * + * Note: Output goes to stderr because KuzuDB's native module captures stdout + * at the OS level during init. This is consistent with augment.ts. + */ + +import { LocalBackend } from '../mcp/local/local-backend.js'; + +let _backend: LocalBackend | null = null; + +async function getBackend(): Promise { + if (_backend) return _backend; + _backend = new LocalBackend(); + const ok = await _backend.init(); + if (!ok) { + console.error('GitNexus: No indexed repositories found. Run: gitnexus analyze'); + process.exit(1); + } + return _backend; +} + +function output(data: any): void { + const text = typeof data === 'string' ? data : JSON.stringify(data, null, 2); + // stderr because KuzuDB captures stdout at OS level + process.stderr.write(text + '\n'); +} + +export async function queryCommand(queryText: string, options?: { + repo?: string; + context?: string; + goal?: string; + limit?: string; + content?: boolean; +}): Promise { + if (!queryText?.trim()) { + console.error('Usage: gitnexus query '); + process.exit(1); + } + + const backend = await getBackend(); + const result = await backend.callTool('query', { + query: queryText, + task_context: options?.context, + goal: options?.goal, + limit: options?.limit ? parseInt(options.limit) : undefined, + include_content: options?.content ?? false, + repo: options?.repo, + }); + output(result); +} + +export async function contextCommand(name: string, options?: { + repo?: string; + file?: string; + uid?: string; + content?: boolean; +}): Promise { + if (!name?.trim() && !options?.uid) { + console.error('Usage: gitnexus context [--uid ] [--file ]'); + process.exit(1); + } + + const backend = await getBackend(); + const result = await backend.callTool('context', { + name: name || undefined, + uid: options?.uid, + file_path: options?.file, + include_content: options?.content ?? false, + repo: options?.repo, + }); + output(result); +} + +export async function impactCommand(target: string, options?: { + direction?: string; + repo?: string; + depth?: string; + includeTests?: boolean; +}): Promise { + if (!target?.trim()) { + console.error('Usage: gitnexus impact [--direction upstream|downstream]'); + process.exit(1); + } + + const backend = await getBackend(); + const result = await backend.callTool('impact', { + target, + direction: options?.direction || 'upstream', + maxDepth: options?.depth ? parseInt(options.depth) : undefined, + includeTests: options?.includeTests ?? false, + repo: options?.repo, + }); + output(result); +} + +export async function cypherCommand(query: string, options?: { + repo?: string; +}): Promise { + if (!query?.trim()) { + console.error('Usage: gitnexus cypher '); + process.exit(1); + } + + const backend = await getBackend(); + const result = await backend.callTool('cypher', { + query, + repo: options?.repo, + }); + output(result); +} diff --git a/gitnexus/src/cli/wiki.ts b/gitnexus/src/cli/wiki.ts new file mode 100644 index 0000000000..6589cb8b77 --- /dev/null +++ b/gitnexus/src/cli/wiki.ts @@ -0,0 +1,234 @@ +/** + * Wiki Command + * + * Generates repository documentation from the knowledge graph. + * Usage: gitnexus wiki [path] [options] + */ + +import path from 'path'; +import readline from 'readline'; +import cliProgress from 'cli-progress'; +import { getGitRoot, isGitRepo } from '../storage/git.js'; +import { getStoragePaths, loadMeta, loadCLIConfig, saveCLIConfig } from '../storage/repo-manager.js'; +import { WikiGenerator, type WikiOptions } from '../core/wiki/generator.js'; +import { resolveLLMConfig } from '../core/wiki/llm-client.js'; + +export interface WikiCommandOptions { + force?: boolean; + model?: string; + baseUrl?: string; + apiKey?: string; +} + +/** + * Prompt the user for input via stdin. + */ +function prompt(question: string, hide = false): Promise { + return new Promise((resolve) => { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + if (hide && process.stdin.isTTY) { + // Mask input for API keys + process.stdout.write(question); + let input = ''; + process.stdin.setRawMode(true); + process.stdin.resume(); + process.stdin.setEncoding('utf-8'); + + const onData = (char: string) => { + if (char === '\n' || char === '\r' || char === '\u0004') { + process.stdin.setRawMode(false); + process.stdin.removeListener('data', onData); + process.stdout.write('\n'); + rl.close(); + resolve(input); + } else if (char === '\u0003') { + // Ctrl+C + process.stdin.setRawMode(false); + rl.close(); + process.exit(1); + } else if (char === '\u007F' || char === '\b') { + // Backspace + if (input.length > 0) { + input = input.slice(0, -1); + process.stdout.write('\b \b'); + } + } else { + input += char; + process.stdout.write('*'); + } + }; + process.stdin.on('data', onData); + } else { + rl.question(question, (answer) => { + rl.close(); + resolve(answer.trim()); + }); + } + }); +} + +export const wikiCommand = async ( + inputPath?: string, + options?: WikiCommandOptions, +) => { + console.log('\n GitNexus Wiki Generator\n'); + + // โ”€โ”€ Resolve repo path โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + let repoPath: string; + if (inputPath) { + repoPath = path.resolve(inputPath); + } else { + const gitRoot = getGitRoot(process.cwd()); + if (!gitRoot) { + console.log(' Error: Not inside a git repository\n'); + process.exitCode = 1; + return; + } + repoPath = gitRoot; + } + + if (!isGitRepo(repoPath)) { + console.log(' Error: Not a git repository\n'); + process.exitCode = 1; + return; + } + + // โ”€โ”€ Check for existing index โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + const { storagePath, kuzuPath } = getStoragePaths(repoPath); + const meta = await loadMeta(storagePath); + + if (!meta) { + console.log(' Error: No GitNexus index found.'); + console.log(' Run `gitnexus analyze` first to index this repository.\n'); + process.exitCode = 1; + return; + } + + // โ”€โ”€ Resolve LLM config (with interactive fallback) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + // If --api-key was passed via CLI, save it immediately + if (options?.apiKey) { + const existing = await loadCLIConfig(); + await saveCLIConfig({ ...existing, apiKey: options.apiKey }); + console.log(' API key saved to ~/.gitnexus/config.json\n'); + } + + let llmConfig = await resolveLLMConfig({ + model: options?.model, + baseUrl: options?.baseUrl, + apiKey: options?.apiKey, + }); + + if (!llmConfig.apiKey) { + if (!process.stdin.isTTY) { + console.log(' Error: No LLM API key found.'); + console.log(' Set OPENAI_API_KEY or GITNEXUS_API_KEY environment variable,'); + console.log(' or pass --api-key .\n'); + process.exitCode = 1; + return; + } + + console.log(' No API key configured.\n'); + console.log(' The wiki command requires an LLM API key (OpenAI-compatible).'); + console.log(' You can also set OPENAI_API_KEY or GITNEXUS_API_KEY env var.\n'); + + const key = await prompt(' Enter your API key: ', true); + if (!key) { + console.log('\n No key provided. Aborting.\n'); + process.exitCode = 1; + return; + } + + const save = await prompt(' Save key to ~/.gitnexus/config.json for future use? (Y/n): '); + if (!save || save.toLowerCase() === 'y' || save.toLowerCase() === 'yes') { + const existing = await loadCLIConfig(); + await saveCLIConfig({ ...existing, apiKey: key }); + console.log(' Key saved.\n'); + } else { + console.log(' Key will be used for this session only.\n'); + } + + llmConfig = { ...llmConfig, apiKey: key }; + } + + // โ”€โ”€ Setup progress bar โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + const bar = new cliProgress.SingleBar({ + format: ' {bar} {percentage}% | {phase}', + barCompleteChar: '\u2588', + barIncompleteChar: '\u2591', + hideCursor: true, + barGlue: '', + autopadding: true, + clearOnComplete: false, + stopOnComplete: false, + }, cliProgress.Presets.shades_grey); + + bar.start(100, 0, { phase: 'Initializing...' }); + + const t0 = Date.now(); + + // โ”€โ”€ Run generator โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + const wikiOptions: WikiOptions = { + force: options?.force, + model: options?.model, + baseUrl: options?.baseUrl, + }; + + const generator = new WikiGenerator( + repoPath, + storagePath, + kuzuPath, + llmConfig, + wikiOptions, + (phase, percent, detail) => { + bar.update(percent, { phase: detail || phase }); + }, + ); + + try { + const result = await generator.run(); + + bar.update(100, { phase: 'Done' }); + bar.stop(); + + const elapsed = ((Date.now() - t0) / 1000).toFixed(1); + + if (result.mode === 'up-to-date' && !options?.force) { + console.log('\n Wiki is already up to date.'); + console.log(` ${path.join(storagePath, 'wiki')}\n`); + return; + } + + console.log(`\n Wiki generated successfully (${elapsed}s)\n`); + console.log(` Mode: ${result.mode}`); + console.log(` Pages: ${result.pagesGenerated}`); + console.log(` Output: ${path.join(storagePath, 'wiki')}`); + + if (result.failedModules && result.failedModules.length > 0) { + console.log(`\n Failed modules (${result.failedModules.length}):`); + for (const mod of result.failedModules) { + console.log(` - ${mod}`); + } + console.log(' Re-run to retry failed modules (pages will be regenerated).'); + } + + console.log(''); + } catch (err: any) { + bar.stop(); + + if (err.message?.includes('No source files')) { + console.log(`\n ${err.message}\n`); + } else if (err.message?.includes('API key') || err.message?.includes('API error')) { + console.log(`\n LLM Error: ${err.message}\n`); + } else { + console.log(`\n Error: ${err.message}\n`); + if (process.env.DEBUG) { + console.error(err); + } + } + process.exitCode = 1; + } +}; diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index 4673b59697..0de1fea07e 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -128,12 +128,14 @@ const createVectorIndex = async ( * @param executeWithReusedStatement - Function to execute with reused prepared statement * @param onProgress - Callback for progress updates * @param config - Optional configuration override + * @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode) */ export const runEmbeddingPipeline = async ( executeQuery: (cypher: string) => Promise, executeWithReusedStatement: (cypher: string, paramsList: Array>) => Promise, onProgress: EmbeddingProgressCallback, - config: Partial = {} + config: Partial = {}, + skipNodeIds?: Set, ): Promise => { const finalConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config }; @@ -146,11 +148,10 @@ export const runEmbeddingPipeline = async ( }); await initEmbedder((modelProgress: ModelProgress) => { - // Report model download progress const downloadPercent = modelProgress.progress ?? 0; onProgress({ phase: 'loading-model', - percent: Math.round(downloadPercent * 0.2), // 0-20% for model loading + percent: Math.round(downloadPercent * 0.2), modelDownloadPercent: downloadPercent, }); }, finalConfig); @@ -166,7 +167,17 @@ export const runEmbeddingPipeline = async ( } // Phase 2: Query embeddable nodes - const nodes = await queryEmbeddableNodes(executeQuery); + let nodes = await queryEmbeddableNodes(executeQuery); + + // Incremental mode: filter out nodes that already have embeddings + if (skipNodeIds && skipNodeIds.size > 0) { + const beforeCount = nodes.length; + nodes = nodes.filter(n => !skipNodeIds.has(n.id)); + if (isDev) { + console.log(`๐Ÿ“ฆ Incremental embeddings: ${beforeCount} total, ${skipNodeIds.size} cached, ${nodes.length} to embed`); + } + } + const totalNodes = nodes.length; if (isDev) { diff --git a/gitnexus/src/core/embeddings/types.ts b/gitnexus/src/core/embeddings/types.ts index b769b950cc..f345725725 100644 --- a/gitnexus/src/core/embeddings/types.ts +++ b/gitnexus/src/core/embeddings/types.ts @@ -59,8 +59,8 @@ export interface EmbeddingConfig { batchSize: number; /** Embedding vector dimensions */ dimensions: number; - /** Device to use for inference: 'auto' tries GPU first, falls back to CPU */ - device: 'auto' | 'webgpu' | 'cuda' | 'cpu' | 'wasm'; + /** Device to use for inference: 'auto' tries GPU first (DirectML on Windows, CUDA on Linux), falls back to CPU */ + device: 'auto' | 'dml' | 'cuda' | 'cpu' | 'wasm'; /** Maximum characters of code snippet to include */ maxSnippetLength: number; } diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index bb42502698..b3766df2b9 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -7,6 +7,7 @@ import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; import { LANGUAGE_QUERIES } from './tree-sitter-queries.js'; import { generateId } from '../../lib/utils.js'; import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; +import type { ExtractedCall } from './workers/parse-worker.js'; /** * Node types that represent function/method definitions across languages. @@ -166,6 +167,8 @@ export const processCalls = async ( continue; } wasReparsed = true; + // Cache re-parsed tree so heritage phase gets hits + astCache.set(file.path, tree); } let query; @@ -176,7 +179,6 @@ export const processCalls = async ( matches = query.matches(tree.rootNode); } catch (queryError) { console.warn(`Query error for ${file.path}:`, queryError); - if (wasReparsed) (tree as any).delete?.(); continue; } @@ -225,10 +227,7 @@ export const processCalls = async ( }); }); - // Cleanup if re-parsed - if (wasReparsed) { - (tree as any).delete?.(); - } + // Tree is now owned by the LRU cache โ€” no manual delete needed } }; @@ -255,29 +254,29 @@ const resolveCallTarget = ( symbolTable: SymbolTable, importMap: ImportMap ): ResolveResult | null => { - // Strategy A: Check imported files (HIGH confidence - we know the import chain) - const importedFiles = importMap.get(currentFile); - if (importedFiles) { - for (const importedFile of importedFiles) { - const nodeId = symbolTable.lookupExact(importedFile, calledName); - if (nodeId) { - return { nodeId, confidence: 0.9, reason: 'import-resolved' }; - } - } - } - - // Strategy B: Check local file (HIGH confidence - same file definition) + // Strategy B first (cheapest โ€” single map lookup): Check local file const localNodeId = symbolTable.lookupExact(currentFile, calledName); if (localNodeId) { return { nodeId: localNodeId, confidence: 0.85, reason: 'same-file' }; } - // Strategy C: Fuzzy global search (LOW confidence - just matching by name) - const fuzzyMatches = symbolTable.lookupFuzzy(calledName); - if (fuzzyMatches.length > 0) { - // Lower confidence if multiple matches exist (more ambiguous) - const confidence = fuzzyMatches.length === 1 ? 0.5 : 0.3; - return { nodeId: fuzzyMatches[0].nodeId, confidence, reason: 'fuzzy-global' }; + // Strategy A: Check if any definition of calledName is in an imported file + // Reversed: instead of iterating all imports and checking each, get all definitions + // and check if any is imported. O(definitions) instead of O(imports). + const allDefs = symbolTable.lookupFuzzy(calledName); + if (allDefs.length > 0) { + const importedFiles = importMap.get(currentFile); + if (importedFiles) { + for (const def of allDefs) { + if (importedFiles.has(def.filePath)) { + return { nodeId: def.nodeId, confidence: 0.9, reason: 'import-resolved' }; + } + } + } + + // Strategy C: Fuzzy global (no import match found) + const confidence = allDefs.length === 1 ? 0.5 : 0.3; + return { nodeId: allDefs[0].nodeId, confidence, reason: 'fuzzy-global' }; } return null; @@ -321,3 +320,59 @@ const isBuiltInOrNoise = (name: string): boolean => { return builtIns.has(name); }; +/** + * Fast path: resolve pre-extracted call sites from workers. + * No AST parsing โ€” workers already extracted calledName + sourceId. + * This function only does symbol table lookups + graph mutations. + */ +export const processCallsFromExtracted = async ( + graph: KnowledgeGraph, + extractedCalls: ExtractedCall[], + symbolTable: SymbolTable, + importMap: ImportMap, + onProgress?: (current: number, total: number) => void +) => { + // Group by file for progress reporting + const byFile = new Map(); + for (const call of extractedCalls) { + let list = byFile.get(call.filePath); + if (!list) { + list = []; + byFile.set(call.filePath, list); + } + list.push(call); + } + + const totalFiles = byFile.size; + let filesProcessed = 0; + + for (const [_filePath, calls] of byFile) { + filesProcessed++; + if (filesProcessed % 100 === 0) { + onProgress?.(filesProcessed, totalFiles); + await yieldToEventLoop(); + } + + for (const call of calls) { + const resolved = resolveCallTarget( + call.calledName, + call.filePath, + symbolTable, + importMap + ); + if (!resolved) continue; + + const relId = generateId('CALLS', `${call.sourceId}:${call.calledName}->${resolved.nodeId}`); + graph.addRelationship({ + id: relId, + sourceId: call.sourceId, + targetId: resolved.nodeId, + type: 'CALLS', + confidence: resolved.confidence, + reason: resolved.reason, + }); + } + } + + onProgress?.(totalFiles, totalFiles); +}; diff --git a/gitnexus/src/core/ingestion/community-processor.ts b/gitnexus/src/core/ingestion/community-processor.ts index 4628df4098..369715b665 100644 --- a/gitnexus/src/core/ingestion/community-processor.ts +++ b/gitnexus/src/core/ingestion/community-processor.ts @@ -159,10 +159,22 @@ const buildGraphologyGraph = (knowledgeGraph: KnowledgeGraph): any => { // Symbol types that should be clustered const symbolTypes = new Set(['Function', 'Class', 'Method', 'Interface']); - - // Add symbol nodes + + // First pass: collect which nodes participate in clustering edges + const clusteringRelTypes = new Set(['CALLS', 'EXTENDS', 'IMPLEMENTS']); + const connectedNodes = new Set(); + + knowledgeGraph.relationships.forEach(rel => { + if (clusteringRelTypes.has(rel.type) && rel.sourceId !== rel.targetId) { + connectedNodes.add(rel.sourceId); + connectedNodes.add(rel.targetId); + } + }); + + // Only add nodes that have at least one clustering edge + // Isolated nodes would just become singletons (skipped anyway) knowledgeGraph.nodes.forEach(node => { - if (symbolTypes.has(node.label)) { + if (symbolTypes.has(node.label) && connectedNodes.has(node.id)) { graph.addNode(node.id, { name: node.properties.name, filePath: node.properties.filePath, @@ -171,16 +183,10 @@ const buildGraphologyGraph = (knowledgeGraph: KnowledgeGraph): any => { } }); - // Add CALLS edges (primary clustering signal) - // We can also include EXTENDS/IMPLEMENTS for OOP clustering - const clusteringRelTypes = new Set(['CALLS', 'EXTENDS', 'IMPLEMENTS']); - + // Add edges knowledgeGraph.relationships.forEach(rel => { if (clusteringRelTypes.has(rel.type)) { - // Only add edge if both nodes exist in our symbol graph - // Also skip self-loops (recursive calls) - not allowed in undirected graph if (graph.hasNode(rel.sourceId) && graph.hasNode(rel.targetId) && rel.sourceId !== rel.targetId) { - // Avoid duplicate edges if (!graph.hasEdge(rel.sourceId, rel.targetId)) { graph.addEdge(rel.sourceId, rel.targetId); } @@ -334,33 +340,34 @@ const findCommonPrefix = (strings: string[]): string => { // ============================================================================ /** - * Calculate cohesion score (0-1) based on internal edge density - * Higher cohesion = more internal connections relative to size + * Estimate cohesion score (0-1) based on internal edge density. + * Uses sampling for large communities to avoid O(N^2) cost. */ const calculateCohesion = (memberIds: string[], graph: any): number => { if (memberIds.length <= 1) return 1.0; const memberSet = new Set(memberIds); + + // Sample up to 50 members for large communities + const SAMPLE_SIZE = 50; + const sample = memberIds.length <= SAMPLE_SIZE + ? memberIds + : memberIds.slice(0, SAMPLE_SIZE); + let internalEdges = 0; - - // Count edges within the community - memberIds.forEach(nodeId => { - if (graph.hasNode(nodeId)) { - graph.forEachNeighbor(nodeId, neighbor => { - if (memberSet.has(neighbor)) { - internalEdges++; - } - }); - } - }); - - // Each edge is counted twice (once from each end), so divide by 2 - internalEdges = internalEdges / 2; - - // Maximum possible internal edges for n nodes: n*(n-1)/2 - const maxPossibleEdges = (memberIds.length * (memberIds.length - 1)) / 2; - - if (maxPossibleEdges === 0) return 1.0; - - return Math.min(1.0, internalEdges / maxPossibleEdges); + let totalEdges = 0; + + for (const nodeId of sample) { + if (!graph.hasNode(nodeId)) continue; + graph.forEachNeighbor(nodeId, (neighbor: string) => { + totalEdges++; + if (memberSet.has(neighbor)) { + internalEdges++; + } + }); + } + + // Cohesion = fraction of edges that stay internal + if (totalEdges === 0) return 1.0; + return Math.min(1.0, internalEdges / totalEdges); }; diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 70712d8e0f..bfad1574c7 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -8,6 +8,8 @@ export interface FileEntry { content: string; } +const READ_CONCURRENCY = 32; + export const walkRepository = async ( repoPath: string, onProgress?: (current: number, total: number, filePath: string) => void @@ -20,21 +22,27 @@ export const walkRepository = async ( const filtered = files.filter(file => !shouldIgnorePath(file)); const entries: FileEntry[] = []; + let processed = 0; - for (let i = 0; i < filtered.length; i++) { - const relativePath = filtered[i]; - const fullPath = path.join(repoPath, relativePath); - try { - const content = await fs.readFile(fullPath, 'utf-8'); - entries.push({ path: relativePath.replace(/\\/g, '/'), content }); - onProgress?.(i + 1, filtered.length, relativePath); - } catch { - onProgress?.(i + 1, filtered.length, relativePath); + for (let start = 0; start < filtered.length; start += READ_CONCURRENCY) { + const batch = filtered.slice(start, start + READ_CONCURRENCY); + const results = await Promise.allSettled( + batch.map(relativePath => + fs.readFile(path.join(repoPath, relativePath), 'utf-8') + .then(content => ({ path: relativePath.replace(/\\/g, '/'), content })) + ) + ); + + for (const result of results) { + processed++; + if (result.status === 'fulfilled') { + entries.push(result.value); + onProgress?.(processed, filtered.length, result.value.path); + } else { + onProgress?.(processed, filtered.length, batch[results.indexOf(result)]); + } } } return entries; }; - - - diff --git a/gitnexus/src/core/ingestion/heritage-processor.ts b/gitnexus/src/core/ingestion/heritage-processor.ts index 0ee2a21b64..dbb7bac8ca 100644 --- a/gitnexus/src/core/ingestion/heritage-processor.ts +++ b/gitnexus/src/core/ingestion/heritage-processor.ts @@ -14,6 +14,7 @@ import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; import { LANGUAGE_QUERIES } from './tree-sitter-queries.js'; import { generateId } from '../../lib/utils.js'; import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; +import type { ExtractedHeritage } from './workers/parse-worker.js'; export const processHeritage = async ( graph: KnowledgeGraph, @@ -52,6 +53,8 @@ export const processHeritage = async ( continue; } wasReparsed = true; + // Cache re-parsed tree for potential future use + astCache.set(file.path, tree); } let query; @@ -62,7 +65,6 @@ export const processHeritage = async ( matches = query.matches(tree.rootNode); } catch (queryError) { console.warn(`Heritage query error for ${file.path}:`, queryError); - if (wasReparsed) (tree as any).delete?.(); continue; } @@ -155,9 +157,86 @@ export const processHeritage = async ( } }); - // Cleanup - if (wasReparsed) { - (tree as any).delete?.(); - } + // Tree is now owned by the LRU cache โ€” no manual delete needed } }; + +/** + * Fast path: resolve pre-extracted heritage from workers. + * No AST parsing โ€” workers already extracted className + parentName + kind. + */ +export const processHeritageFromExtracted = async ( + graph: KnowledgeGraph, + extractedHeritage: ExtractedHeritage[], + symbolTable: SymbolTable, + onProgress?: (current: number, total: number) => void +) => { + const total = extractedHeritage.length; + + for (let i = 0; i < extractedHeritage.length; i++) { + if (i % 500 === 0) { + onProgress?.(i, total); + await yieldToEventLoop(); + } + + const h = extractedHeritage[i]; + + if (h.kind === 'extends') { + const childId = symbolTable.lookupExact(h.filePath, h.className) || + symbolTable.lookupFuzzy(h.className)[0]?.nodeId || + generateId('Class', `${h.filePath}:${h.className}`); + + const parentId = symbolTable.lookupFuzzy(h.parentName)[0]?.nodeId || + generateId('Class', `${h.parentName}`); + + if (childId && parentId && childId !== parentId) { + graph.addRelationship({ + id: generateId('EXTENDS', `${childId}->${parentId}`), + sourceId: childId, + targetId: parentId, + type: 'EXTENDS', + confidence: 1.0, + reason: '', + }); + } + } else if (h.kind === 'implements') { + const classId = symbolTable.lookupExact(h.filePath, h.className) || + symbolTable.lookupFuzzy(h.className)[0]?.nodeId || + generateId('Class', `${h.filePath}:${h.className}`); + + const interfaceId = symbolTable.lookupFuzzy(h.parentName)[0]?.nodeId || + generateId('Interface', `${h.parentName}`); + + if (classId && interfaceId) { + graph.addRelationship({ + id: generateId('IMPLEMENTS', `${classId}->${interfaceId}`), + sourceId: classId, + targetId: interfaceId, + type: 'IMPLEMENTS', + confidence: 1.0, + reason: '', + }); + } + } else if (h.kind === 'trait-impl') { + const structId = symbolTable.lookupExact(h.filePath, h.className) || + symbolTable.lookupFuzzy(h.className)[0]?.nodeId || + generateId('Struct', `${h.filePath}:${h.className}`); + + const traitId = symbolTable.lookupFuzzy(h.parentName)[0]?.nodeId || + generateId('Trait', `${h.parentName}`); + + if (structId && traitId) { + graph.addRelationship({ + id: generateId('IMPLEMENTS', `${structId}->${traitId}`), + sourceId: structId, + targetId: traitId, + type: 'IMPLEMENTS', + confidence: 1.0, + reason: 'trait-impl', + }); + } + } + } + + onProgress?.(total, total); +}; diff --git a/gitnexus/src/core/ingestion/import-processor.ts b/gitnexus/src/core/ingestion/import-processor.ts index 07c1b17141..54b5ea0709 100644 --- a/gitnexus/src/core/ingestion/import-processor.ts +++ b/gitnexus/src/core/ingestion/import-processor.ts @@ -8,6 +8,7 @@ import { LANGUAGE_QUERIES } from './tree-sitter-queries.js'; import { generateId } from '../../lib/utils.js'; import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; import { SupportedLanguages } from '../../config/supported-languages.js'; +import type { ExtractedImport } from './workers/parse-worker.js'; const isDev = process.env.NODE_ENV === 'development'; @@ -139,14 +140,101 @@ function tryResolveWithExtensions( } /** - * Suffix-based resolution: try progressively shorter suffixes against all files. - * Used for package-style imports (Java, Python, etc.). + * Build a suffix index for O(1) endsWith lookups. + * Maps every possible path suffix to its original file path. + * e.g. for "src/com/example/Foo.java": + * "Foo.java" -> "src/com/example/Foo.java" + * "example/Foo.java" -> "src/com/example/Foo.java" + * "com/example/Foo.java" -> "src/com/example/Foo.java" + * etc. + */ +export interface SuffixIndex { + /** Exact suffix lookup (case-sensitive) */ + get(suffix: string): string | undefined; + /** Case-insensitive suffix lookup */ + getInsensitive(suffix: string): string | undefined; + /** Get all files in a directory suffix */ + getFilesInDir(dirSuffix: string, extension: string): string[]; +} + +function buildSuffixIndex(normalizedFileList: string[], allFileList: string[]): SuffixIndex { + // Map: normalized suffix -> original file path + const exactMap = new Map(); + // Map: lowercase suffix -> original file path + const lowerMap = new Map(); + // Map: directory suffix -> list of file paths in that directory + const dirMap = new Map(); + + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + const original = allFileList[i]; + const parts = normalized.split('/'); + + // Index all suffixes: "a/b/c.java" -> ["c.java", "b/c.java", "a/b/c.java"] + for (let j = parts.length - 1; j >= 0; j--) { + const suffix = parts.slice(j).join('/'); + // Only store first match (longest path wins for ambiguous suffixes) + if (!exactMap.has(suffix)) { + exactMap.set(suffix, original); + } + const lower = suffix.toLowerCase(); + if (!lowerMap.has(lower)) { + lowerMap.set(lower, original); + } + } + + // Index directory membership + const lastSlash = normalized.lastIndexOf('/'); + if (lastSlash >= 0) { + // Build all directory suffixes + const dirParts = parts.slice(0, -1); + const fileName = parts[parts.length - 1]; + const ext = fileName.substring(fileName.lastIndexOf('.')); + + for (let j = dirParts.length - 1; j >= 0; j--) { + const dirSuffix = dirParts.slice(j).join('/'); + const key = `${dirSuffix}:${ext}`; + let list = dirMap.get(key); + if (!list) { + list = []; + dirMap.set(key, list); + } + list.push(original); + } + } + } + + return { + get: (suffix: string) => exactMap.get(suffix), + getInsensitive: (suffix: string) => lowerMap.get(suffix.toLowerCase()), + getFilesInDir: (dirSuffix: string, extension: string) => { + return dirMap.get(`${dirSuffix}:${extension}`) || []; + }, + }; +} + +/** + * Suffix-based resolution using index. O(1) per lookup instead of O(files). */ function suffixResolve( pathParts: string[], normalizedFileList: string[], allFileList: string[], + index?: SuffixIndex, ): string | null { + if (index) { + for (let i = 0; i < pathParts.length; i++) { + const suffix = pathParts.slice(i).join('/'); + for (const ext of EXTENSIONS) { + const suffixWithExt = suffix + ext; + const result = index.get(suffixWithExt) || index.getInsensitive(suffixWithExt); + if (result) return result; + } + } + return null; + } + + // Fallback: linear scan (for backward compatibility) for (let i = 0; i < pathParts.length; i++) { const suffix = pathParts.slice(i).join('/'); for (const ext of EXTENSIONS) { @@ -182,6 +270,7 @@ const resolveImportPath = ( resolveCache: Map, language: SupportedLanguages, tsconfigPaths: TsconfigPaths | null, + index?: SuffixIndex, ): string | null => { const cacheKey = `${currentFile}::${importPath}`; if (resolveCache.has(cacheKey)) return resolveCache.get(cacheKey) ?? null; @@ -211,7 +300,7 @@ const resolveImportPath = ( // Try suffix matching as fallback const parts = rewritten.split('/').filter(Boolean); - const suffixResult = suffixResolve(parts, normalizedFileList, allFileList); + const suffixResult = suffixResolve(parts, normalizedFileList, allFileList, index); if (suffixResult) return cache(suffixResult); } } @@ -255,7 +344,7 @@ const resolveImportPath = ( : importPath.replace(/\./g, '/'); const pathParts = pathLike.split('/').filter(Boolean); - const resolved = suffixResolve(pathParts, normalizedFileList, allFileList); + const resolved = suffixResolve(pathParts, normalizedFileList, allFileList, index); return cache(resolved); }; @@ -353,16 +442,31 @@ function resolveJavaWildcard( importPath: string, normalizedFileList: string[], allFileList: string[], + index?: SuffixIndex, ): string[] { // "com.example.util.*" -> "com/example/util" const packagePath = importPath.slice(0, -2).replace(/\./g, '/'); - const packageSuffix = '/' + packagePath + '/'; + if (index) { + // Use directory index: get all .java files in this package directory + const candidates = index.getFilesInDir(packagePath, '.java'); + // Filter to only direct children (no subdirectories) + const packageSuffix = '/' + packagePath + '/'; + return candidates.filter(f => { + const normalized = f.replace(/\\/g, '/'); + const idx = normalized.indexOf(packageSuffix); + if (idx < 0) return false; + const afterPkg = normalized.substring(idx + packageSuffix.length); + return !afterPkg.includes('/'); + }); + } + + // Fallback: linear scan + const packageSuffix = '/' + packagePath + '/'; const matches: string[] = []; for (let i = 0; i < normalizedFileList.length; i++) { const normalized = normalizedFileList[i]; if (normalized.includes(packageSuffix) && normalized.endsWith('.java')) { - // Ensure the file is directly in the package (not a subdirectory) const afterPackage = normalized.substring(normalized.indexOf(packageSuffix) + packageSuffix.length); if (!afterPackage.includes('/')) { matches.push(allFileList[i]); @@ -380,6 +484,7 @@ function resolveJavaStaticImport( importPath: string, normalizedFileList: string[], allFileList: string[], + index?: SuffixIndex, ): string | null { // Static imports look like: com.example.Constants.VALUE or com.example.Constants.* // The last segment is a member name (field/method) if it starts with lowercase or is ALL_CAPS @@ -390,10 +495,17 @@ function resolveJavaStaticImport( // If last segment is a wildcard or ALL_CAPS constant or starts with lowercase, strip it if (lastSeg === '*' || /^[a-z]/.test(lastSeg) || /^[A-Z_]+$/.test(lastSeg)) { const classPath = segments.slice(0, -1).join('/'); - const classSuffix = '/' + classPath + '.java'; + const classSuffix = classPath + '.java'; + + if (index) { + return index.get(classSuffix) || index.getInsensitive(classSuffix) || null; + } + + // Fallback: linear scan + const fullSuffix = '/' + classSuffix; for (let i = 0; i < normalizedFileList.length; i++) { - if (normalizedFileList[i].endsWith(classSuffix) || - normalizedFileList[i].toLowerCase().endsWith(classSuffix.toLowerCase())) { + if (normalizedFileList[i].endsWith(fullSuffix) || + normalizedFileList[i].toLowerCase().endsWith(fullSuffix.toLowerCase())) { return allFileList[i]; } } @@ -458,6 +570,8 @@ export const processImports = async ( const allFileList = files.map(f => f.path); // Pre-compute normalized file list once (forward slashes) const normalizedFileList = allFileList.map(p => p.replace(/\\/g, '/')); + // Build suffix index for O(1) lookups + const index = buildSuffixIndex(normalizedFileList, allFileList); // Track import statistics let totalImportsFound = 0; @@ -517,6 +631,8 @@ export const processImports = async ( continue; } wasReparsed = true; + // Cache re-parsed tree so call/heritage phases get hits + astCache.set(file.path, tree); } let query; @@ -561,7 +677,7 @@ export const processImports = async ( // ---- Java: handle wildcards and static imports specially ---- if (language === SupportedLanguages.Java) { if (rawImportPath.endsWith('.*')) { - const matchedFiles = resolveJavaWildcard(rawImportPath, normalizedFileList, allFileList); + const matchedFiles = resolveJavaWildcard(rawImportPath, normalizedFileList, allFileList, index); for (const matchedFile of matchedFiles) { addImportEdge(file.path, matchedFile); } @@ -569,7 +685,7 @@ export const processImports = async ( } // Try static import resolution (strip member name) - const staticResolved = resolveJavaStaticImport(rawImportPath, normalizedFileList, allFileList); + const staticResolved = resolveJavaStaticImport(rawImportPath, normalizedFileList, allFileList, index); if (staticResolved) { addImportEdge(file.path, staticResolved); return; @@ -599,6 +715,7 @@ export const processImports = async ( resolveCache, language, tsconfigPaths, + index, ); if (resolvedPath) { @@ -607,13 +724,160 @@ export const processImports = async ( } }); - // If re-parsed just for this, delete the tree to save memory - if (wasReparsed) { - (tree as any).delete?.(); - } + // Tree is now owned by the LRU cache โ€” no manual delete needed } if (isDev) { console.log(`๐Ÿ“Š Import processing complete: ${totalImportsResolved}/${totalImportsFound} imports resolved to graph edges`); } }; + +// ============================================================================ +// FAST PATH: Resolve pre-extracted imports (no parsing needed) +// ============================================================================ + +export const processImportsFromExtracted = async ( + graph: KnowledgeGraph, + files: { path: string; content: string }[], + extractedImports: ExtractedImport[], + importMap: ImportMap, + onProgress?: (current: number, total: number) => void, + repoRoot?: string, +) => { + const allFilePaths = new Set(files.map(f => f.path)); + const resolveCache = new Map(); + const allFileList = files.map(f => f.path); + const normalizedFileList = allFileList.map(p => p.replace(/\\/g, '/')); + // Build suffix index for O(1) lookups + const index = buildSuffixIndex(normalizedFileList, allFileList); + + let totalImportsFound = 0; + let totalImportsResolved = 0; + + const effectiveRoot = repoRoot || ''; + const tsconfigPaths = await loadTsconfigPaths(effectiveRoot); + const goModule = await loadGoModulePath(effectiveRoot); + + const addImportEdge = (filePath: string, resolvedPath: string) => { + const sourceId = generateId('File', filePath); + const targetId = generateId('File', resolvedPath); + const relId = generateId('IMPORTS', `${filePath}->${resolvedPath}`); + + totalImportsResolved++; + + graph.addRelationship({ + id: relId, + sourceId, + targetId, + type: 'IMPORTS', + confidence: 1.0, + reason: '', + }); + + if (!importMap.has(filePath)) { + importMap.set(filePath, new Set()); + } + importMap.get(filePath)!.add(resolvedPath); + }; + + // Group by file for progress reporting (users see file count, not import count) + const importsByFile = new Map(); + for (const imp of extractedImports) { + let list = importsByFile.get(imp.filePath); + if (!list) { + list = []; + importsByFile.set(imp.filePath, list); + } + list.push(imp); + } + + const totalFiles = importsByFile.size; + let filesProcessed = 0; + + // Pre-build a suffix index for O(1) suffix lookups instead of O(n) linear scans + const suffixIndex = new Map(); + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + // Index by last path segment (filename) for fast suffix matching + const lastSlash = normalized.lastIndexOf('/'); + const filename = lastSlash >= 0 ? normalized.substring(lastSlash + 1) : normalized; + let list = suffixIndex.get(filename); + if (!list) { + list = []; + suffixIndex.set(filename, list); + } + list.push(allFileList[i]); + } + + for (const [filePath, fileImports] of importsByFile) { + filesProcessed++; + if (filesProcessed % 100 === 0) { + onProgress?.(filesProcessed, totalFiles); + await yieldToEventLoop(); + } + + for (const { rawImportPath, language } of fileImports) { + totalImportsFound++; + + // Check resolve cache first + const cacheKey = `${filePath}::${rawImportPath}`; + if (resolveCache.has(cacheKey)) { + const cached = resolveCache.get(cacheKey); + if (cached) addImportEdge(filePath, cached); + continue; + } + + // Java: handle wildcards and static imports + if (language === SupportedLanguages.Java) { + if (rawImportPath.endsWith('.*')) { + const matchedFiles = resolveJavaWildcard(rawImportPath, normalizedFileList, allFileList, index); + for (const matchedFile of matchedFiles) { + addImportEdge(filePath, matchedFile); + } + continue; + } + + const staticResolved = resolveJavaStaticImport(rawImportPath, normalizedFileList, allFileList, index); + if (staticResolved) { + resolveCache.set(cacheKey, staticResolved); + addImportEdge(filePath, staticResolved); + continue; + } + } + + // Go: handle package-level imports + if (language === SupportedLanguages.Go && goModule && rawImportPath.startsWith(goModule.modulePath)) { + const pkgFiles = resolveGoPackage(rawImportPath, goModule, normalizedFileList, allFileList); + if (pkgFiles.length > 0) { + for (const pkgFile of pkgFiles) { + addImportEdge(filePath, pkgFile); + } + continue; + } + } + + // Standard resolution (has its own internal cache) + const resolvedPath = resolveImportPath( + filePath, + rawImportPath, + allFilePaths, + allFileList, + normalizedFileList, + resolveCache, + language as SupportedLanguages, + tsconfigPaths, + index, + ); + + if (resolvedPath) { + addImportEdge(filePath, resolvedPath); + } + } + } + + onProgress?.(totalFiles, totalFiles); + + if (isDev) { + console.log(`๐Ÿ“Š Import processing (fast path): ${totalImportsResolved}/${totalImportsFound} imports resolved to graph edges`); + } +}; diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index a4777562dd..4d02e3f891 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -6,9 +6,17 @@ import { generateId } from '../../lib/utils.js'; import { SymbolTable } from './symbol-table.js'; import { ASTCache } from './ast-cache.js'; import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; +import { WorkerPool } from './workers/worker-pool.js'; +import type { ParseWorkerResult, ParseWorkerInput, ExtractedImport, ExtractedCall, ExtractedHeritage } from './workers/parse-worker.js'; export type FileProgressCallback = (current: number, total: number, filePath: string) => void; +export interface WorkerExtractedData { + imports: ExtractedImport[]; + calls: ExtractedCall[]; + heritage: ExtractedHeritage[]; +} + // ============================================================================ // EXPORT DETECTION - Language-specific visibility detection // ============================================================================ @@ -16,7 +24,7 @@ export type FileProgressCallback = (current: number, total: number, filePath: st /** * Check if a symbol (function, class, etc.) is exported/public * Handles all 9 supported languages with explicit logic - * + * * @param node - The AST node for the symbol name * @param name - The symbol name * @param language - The programming language @@ -24,14 +32,14 @@ export type FileProgressCallback = (current: number, total: number, filePath: st */ const isNodeExported = (node: any, name: string, language: string): boolean => { let current = node; - + switch (language) { // JavaScript/TypeScript: Check for export keyword in ancestors case 'javascript': case 'typescript': while (current) { const type = current.type; - if (type === 'export_statement' || + if (type === 'export_statement' || type === 'export_specifier' || type === 'lexical_declaration' && current.parent?.type === 'export_statement') { return true; @@ -43,11 +51,11 @@ const isNodeExported = (node: any, name: string, language: string): boolean => { current = current.parent; } return false; - + // Python: Public if no leading underscore (convention) case 'python': return !name.startsWith('_'); - + // Java: Check for 'public' modifier // In tree-sitter Java, modifiers are siblings of the name node, not parents case 'java': @@ -72,7 +80,7 @@ const isNodeExported = (node: any, name: string, language: string): boolean => { current = current.parent; } return false; - + // C#: Check for 'public' modifier in ancestors case 'csharp': while (current) { @@ -82,14 +90,14 @@ const isNodeExported = (node: any, name: string, language: string): boolean => { current = current.parent; } return false; - + // Go: Uppercase first letter = exported case 'go': if (name.length === 0) return false; const first = name[0]; // Must be uppercase letter (not a number or symbol) return first === first.toUpperCase() && first !== first.toLowerCase(); - + // Rust: Check for 'pub' visibility modifier case 'rust': while (current) { @@ -99,66 +107,125 @@ const isNodeExported = (node: any, name: string, language: string): boolean => { current = current.parent; } return false; - + // C/C++: No native export concept at language level // Entry points will be detected via name patterns (main, etc.) case 'c': case 'cpp': return false; - + default: return false; } }; -export const processParsing = async ( - graph: KnowledgeGraph, +// ============================================================================ +// Worker-based parallel parsing +// ============================================================================ + +const processParsingWithWorkers = async ( + graph: KnowledgeGraph, + files: { path: string; content: string }[], + symbolTable: SymbolTable, + astCache: ASTCache, + workerPool: WorkerPool, + onFileProgress?: FileProgressCallback +): Promise => { + // Filter to parseable files only + const parseableFiles: ParseWorkerInput[] = []; + for (const file of files) { + const lang = getLanguageFromFilename(file.path); + if (lang) { + parseableFiles.push({ path: file.path, content: file.content }); + } + } + + if (parseableFiles.length === 0) return { imports: [], calls: [], heritage: [] }; + + const total = files.length; + + // Dispatch to worker pool โ€” pool handles splitting into chunks + // Workers send progress messages during parsing so the bar updates smoothly + const chunkResults = await workerPool.dispatch( + parseableFiles, + (filesProcessed) => { + onFileProgress?.(Math.min(filesProcessed, total), total, 'Parsing...'); + } + ); + + // Merge results from all workers into graph and symbol table + const allImports: ExtractedImport[] = []; + const allCalls: ExtractedCall[] = []; + const allHeritage: ExtractedHeritage[] = []; + for (const result of chunkResults) { + for (const node of result.nodes) { + graph.addNode({ + id: node.id, + label: node.label as any, + properties: node.properties, + }); + } + + for (const rel of result.relationships) { + graph.addRelationship(rel); + } + + for (const sym of result.symbols) { + symbolTable.add(sym.filePath, sym.name, sym.nodeId, sym.type); + } + + allImports.push(...result.imports); + allCalls.push(...result.calls); + allHeritage.push(...result.heritage); + } + + // Final progress + onFileProgress?.(total, total, 'done'); + return { imports: allImports, calls: allCalls, heritage: allHeritage }; +}; + +// ============================================================================ +// Sequential fallback (original implementation) +// ============================================================================ + +const processParsingSequential = async ( + graph: KnowledgeGraph, files: { path: string; content: string }[], symbolTable: SymbolTable, astCache: ASTCache, onFileProgress?: FileProgressCallback ) => { - const parser = await loadParser(); const total = files.length; for (let i = 0; i < files.length; i++) { const file = files[i]; - - // Report progress for each file + onFileProgress?.(i + 1, total, file.path); - - // Yield to event loop periodically so spinner can update + if (i % 20 === 0) await yieldToEventLoop(); - + const language = getLanguageFromFilename(file.path); if (!language) continue; await loadLanguage(language, file.path); - - // 3. Parse the text content into an AST - // Use larger bufferSize for files > 32KB (default limit) + let tree; try { tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 }); } catch (parseError) { - // Skip files that can't be parsed (binary, encoding issues, etc.) console.warn(`Skipping unparseable file: ${file.path}`); continue; } - - // Store in cache immediately (this might evict an old one) + astCache.set(file.path, tree); - - // 4. Get the specific query string for this language + const queryString = LANGUAGE_QUERIES[language]; if (!queryString) { continue; } - // 5. Run the query against the AST root node - // This looks for patterns like (function_declaration) let query; let matches; try { @@ -170,21 +237,17 @@ export const processParsing = async ( continue; } - // 6. Process every match found matches.forEach(match => { const captureMap: Record = {}; - + match.captures.forEach(c => { captureMap[c.name] = c.node; }); - // Skip imports here - they are handled by import-processor.ts - // which creates proper File -> IMPORTS -> File relationships if (captureMap['import']) { return; } - // Skip call expressions - they are handled by call-processor.ts if (captureMap['call']) { return; } @@ -193,43 +256,34 @@ export const processParsing = async ( if (!nameNode) return; const nodeName = nameNode.text; - + let nodeLabel = 'CodeElement'; - - // Core types + if (captureMap['definition.function']) nodeLabel = 'Function'; else if (captureMap['definition.class']) nodeLabel = 'Class'; else if (captureMap['definition.interface']) nodeLabel = 'Interface'; else if (captureMap['definition.method']) nodeLabel = 'Method'; - // Struct types (C, C++, Go, Rust, C#) else if (captureMap['definition.struct']) nodeLabel = 'Struct'; - // Enum types else if (captureMap['definition.enum']) nodeLabel = 'Enum'; - // Namespace/Module (C++, C#, Rust) else if (captureMap['definition.namespace']) nodeLabel = 'Namespace'; else if (captureMap['definition.module']) nodeLabel = 'Module'; - // Rust-specific else if (captureMap['definition.trait']) nodeLabel = 'Trait'; else if (captureMap['definition.impl']) nodeLabel = 'Impl'; else if (captureMap['definition.type']) nodeLabel = 'TypeAlias'; else if (captureMap['definition.const']) nodeLabel = 'Const'; else if (captureMap['definition.static']) nodeLabel = 'Static'; - // C-specific else if (captureMap['definition.typedef']) nodeLabel = 'Typedef'; else if (captureMap['definition.macro']) nodeLabel = 'Macro'; else if (captureMap['definition.union']) nodeLabel = 'Union'; - // C#-specific else if (captureMap['definition.property']) nodeLabel = 'Property'; else if (captureMap['definition.record']) nodeLabel = 'Record'; else if (captureMap['definition.delegate']) nodeLabel = 'Delegate'; - // Java-specific else if (captureMap['definition.annotation']) nodeLabel = 'Annotation'; else if (captureMap['definition.constructor']) nodeLabel = 'Constructor'; - // C++ template else if (captureMap['definition.template']) nodeLabel = 'Template'; const nodeId = generateId(nodeLabel, `${file.path}:${nodeName}`); - + const node: GraphNode = { id: nodeId, label: nodeLabel as any, @@ -245,13 +299,12 @@ export const processParsing = async ( graph.addNode(node); - // Register in Symbol Table (only definitions, not imports) symbolTable.add(file.path, nodeName, nodeId, nodeLabel); const fileId = generateId('File', file.path); - + const relId = generateId('DEFINES', `${fileId}->${nodeId}`); - + const relationship: GraphRelationship = { id: relId, sourceId: fileId, @@ -263,7 +316,30 @@ export const processParsing = async ( graph.addRelationship(relationship); }); - - // Don't delete tree here - LRU cache handles cleanup when evicted } }; + +// ============================================================================ +// Public API +// ============================================================================ + +export const processParsing = async ( + graph: KnowledgeGraph, + files: { path: string; content: string }[], + symbolTable: SymbolTable, + astCache: ASTCache, + onFileProgress?: FileProgressCallback, + workerPool?: WorkerPool, +): Promise => { + if (workerPool) { + try { + return await processParsingWithWorkers(graph, files, symbolTable, astCache, workerPool, onFileProgress); + } catch (err) { + console.warn('Worker pool parsing failed, falling back to sequential:', err); + } + } + + // Fallback: sequential parsing (no pre-extracted data) + await processParsingSequential(graph, files, symbolTable, astCache, onFileProgress); + return null; +}; diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts new file mode 100644 index 0000000000..573884f36c --- /dev/null +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -0,0 +1,535 @@ +import { parentPort } from 'node:worker_threads'; +import Parser from 'tree-sitter'; +import JavaScript from 'tree-sitter-javascript'; +import TypeScript from 'tree-sitter-typescript'; +import Python from 'tree-sitter-python'; +import Java from 'tree-sitter-java'; +import C from 'tree-sitter-c'; +import CPP from 'tree-sitter-cpp'; +import CSharp from 'tree-sitter-c-sharp'; +import Go from 'tree-sitter-go'; +import Rust from 'tree-sitter-rust'; +import { SupportedLanguages } from '../../../config/supported-languages.js'; +import { LANGUAGE_QUERIES } from '../tree-sitter-queries.js'; +import { getLanguageFromFilename } from '../utils.js'; +import { generateId } from '../../../lib/utils.js'; + +// ============================================================================ +// Types for serializable results +// ============================================================================ + +interface ParsedNode { + id: string; + label: string; + properties: { + name: string; + filePath: string; + startLine: number; + endLine: number; + language: string; + isExported: boolean; + }; +} + +interface ParsedRelationship { + id: string; + sourceId: string; + targetId: string; + type: 'DEFINES'; + confidence: number; + reason: string; +} + +interface ParsedSymbol { + filePath: string; + name: string; + nodeId: string; + type: string; +} + +export interface ExtractedImport { + filePath: string; + rawImportPath: string; + language: string; +} + +export interface ExtractedCall { + filePath: string; + calledName: string; + /** generateId of enclosing function, or generateId('File', filePath) for top-level */ + sourceId: string; +} + +export interface ExtractedHeritage { + filePath: string; + className: string; + parentName: string; + /** 'extends' | 'implements' | 'trait-impl' */ + kind: string; +} + +export interface ParseWorkerResult { + nodes: ParsedNode[]; + relationships: ParsedRelationship[]; + symbols: ParsedSymbol[]; + imports: ExtractedImport[]; + calls: ExtractedCall[]; + heritage: ExtractedHeritage[]; + fileCount: number; +} + +export interface ParseWorkerInput { + path: string; + content: string; +} + +// ============================================================================ +// Worker-local parser + language map +// ============================================================================ + +const parser = new Parser(); + +const languageMap: Record = { + [SupportedLanguages.JavaScript]: JavaScript, + [SupportedLanguages.TypeScript]: TypeScript.typescript, + [`${SupportedLanguages.TypeScript}:tsx`]: TypeScript.tsx, + [SupportedLanguages.Python]: Python, + [SupportedLanguages.Java]: Java, + [SupportedLanguages.C]: C, + [SupportedLanguages.CPlusPlus]: CPP, + [SupportedLanguages.CSharp]: CSharp, + [SupportedLanguages.Go]: Go, + [SupportedLanguages.Rust]: Rust, +}; + +const setLanguage = (language: SupportedLanguages, filePath: string): void => { + const key = language === SupportedLanguages.TypeScript && filePath.endsWith('.tsx') + ? `${language}:tsx` + : language; + const lang = languageMap[key]; + if (!lang) throw new Error(`Unsupported language: ${language}`); + parser.setLanguage(lang); +}; + +// ============================================================================ +// Export detection (copied โ€” needs AST parent traversal, can't cross threads) +// ============================================================================ + +const isNodeExported = (node: any, name: string, language: string): boolean => { + let current = node; + + switch (language) { + case 'javascript': + case 'typescript': + while (current) { + const type = current.type; + if (type === 'export_statement' || + type === 'export_specifier' || + type === 'lexical_declaration' && current.parent?.type === 'export_statement') { + return true; + } + if (current.text?.startsWith('export ')) { + return true; + } + current = current.parent; + } + return false; + + case 'python': + return !name.startsWith('_'); + + case 'java': + while (current) { + if (current.parent) { + const parent = current.parent; + for (let i = 0; i < parent.childCount; i++) { + const child = parent.child(i); + if (child?.type === 'modifiers' && child.text?.includes('public')) { + return true; + } + } + if (parent.type === 'method_declaration' || parent.type === 'constructor_declaration') { + if (parent.text?.trimStart().startsWith('public')) { + return true; + } + } + } + current = current.parent; + } + return false; + + case 'csharp': + while (current) { + if (current.type === 'modifier' || current.type === 'modifiers') { + if (current.text?.includes('public')) return true; + } + current = current.parent; + } + return false; + + case 'go': + if (name.length === 0) return false; + const first = name[0]; + return first === first.toUpperCase() && first !== first.toLowerCase(); + + case 'rust': + while (current) { + if (current.type === 'visibility_modifier') { + if (current.text?.includes('pub')) return true; + } + current = current.parent; + } + return false; + + case 'c': + case 'cpp': + return false; + + default: + return false; + } +}; + +// ============================================================================ +// Enclosing function detection (for call extraction) +// ============================================================================ + +const FUNCTION_NODE_TYPES = new Set([ + 'function_declaration', 'arrow_function', 'function_expression', + 'method_definition', 'generator_function_declaration', + 'function_definition', 'async_function_declaration', 'async_arrow_function', + 'method_declaration', 'constructor_declaration', + 'local_function_statement', 'function_item', 'impl_item', +]); + +/** Walk up AST to find enclosing function, return its generateId or null for top-level */ +const findEnclosingFunctionId = (node: any, filePath: string): string | null => { + let current = node.parent; + while (current) { + if (FUNCTION_NODE_TYPES.has(current.type)) { + let funcName: string | null = null; + let label = 'Function'; + + if (['function_declaration', 'function_definition', 'async_function_declaration', + 'generator_function_declaration', 'function_item'].includes(current.type)) { + const nameNode = current.childForFieldName?.('name') || + current.children?.find((c: any) => c.type === 'identifier' || c.type === 'property_identifier'); + funcName = nameNode?.text; + } else if (current.type === 'impl_item') { + const funcItem = current.children?.find((c: any) => c.type === 'function_item'); + if (funcItem) { + const nameNode = funcItem.childForFieldName?.('name') || + funcItem.children?.find((c: any) => c.type === 'identifier'); + funcName = nameNode?.text; + label = 'Method'; + } + } else if (current.type === 'method_definition') { + const nameNode = current.childForFieldName?.('name') || + current.children?.find((c: any) => c.type === 'property_identifier'); + funcName = nameNode?.text; + label = 'Method'; + } else if (current.type === 'method_declaration' || current.type === 'constructor_declaration') { + const nameNode = current.childForFieldName?.('name') || + current.children?.find((c: any) => c.type === 'identifier'); + funcName = nameNode?.text; + label = 'Method'; + } else if (current.type === 'arrow_function' || current.type === 'function_expression') { + const parent = current.parent; + if (parent?.type === 'variable_declarator') { + const nameNode = parent.childForFieldName?.('name') || + parent.children?.find((c: any) => c.type === 'identifier'); + funcName = nameNode?.text; + } + } + + if (funcName) { + return generateId(label, `${filePath}:${funcName}`); + } + } + current = current.parent; + } + return null; +}; + +const BUILT_INS = new Set([ + 'console', 'log', 'warn', 'error', 'info', 'debug', + 'setTimeout', 'setInterval', 'clearTimeout', 'clearInterval', + 'parseInt', 'parseFloat', 'isNaN', 'isFinite', + 'encodeURI', 'decodeURI', 'encodeURIComponent', 'decodeURIComponent', + 'JSON', 'parse', 'stringify', + 'Object', 'Array', 'String', 'Number', 'Boolean', 'Symbol', 'BigInt', + 'Map', 'Set', 'WeakMap', 'WeakSet', + 'Promise', 'resolve', 'reject', 'then', 'catch', 'finally', + 'Math', 'Date', 'RegExp', 'Error', + 'require', 'import', 'export', 'fetch', 'Response', 'Request', + 'useState', 'useEffect', 'useCallback', 'useMemo', 'useRef', 'useContext', + 'useReducer', 'useLayoutEffect', 'useImperativeHandle', 'useDebugValue', + 'createElement', 'createContext', 'createRef', 'forwardRef', 'memo', 'lazy', + 'map', 'filter', 'reduce', 'forEach', 'find', 'findIndex', 'some', 'every', + 'includes', 'indexOf', 'slice', 'splice', 'concat', 'join', 'split', + 'push', 'pop', 'shift', 'unshift', 'sort', 'reverse', + 'keys', 'values', 'entries', 'assign', 'freeze', 'seal', + 'hasOwnProperty', 'toString', 'valueOf', + 'print', 'len', 'range', 'str', 'int', 'float', 'list', 'dict', 'set', 'tuple', + 'open', 'read', 'write', 'close', 'append', 'extend', 'update', + 'super', 'type', 'isinstance', 'issubclass', 'getattr', 'setattr', 'hasattr', + 'enumerate', 'zip', 'sorted', 'reversed', 'min', 'max', 'sum', 'abs', +]); + +// ============================================================================ +// Label detection from capture map +// ============================================================================ + +const getLabelFromCaptures = (captureMap: Record): string | null => { + // Skip imports (handled separately) and calls + if (captureMap['import'] || captureMap['call']) return null; + if (!captureMap['name']) return null; + + if (captureMap['definition.function']) return 'Function'; + if (captureMap['definition.class']) return 'Class'; + if (captureMap['definition.interface']) return 'Interface'; + if (captureMap['definition.method']) return 'Method'; + if (captureMap['definition.struct']) return 'Struct'; + if (captureMap['definition.enum']) return 'Enum'; + if (captureMap['definition.namespace']) return 'Namespace'; + if (captureMap['definition.module']) return 'Module'; + if (captureMap['definition.trait']) return 'Trait'; + if (captureMap['definition.impl']) return 'Impl'; + if (captureMap['definition.type']) return 'TypeAlias'; + if (captureMap['definition.const']) return 'Const'; + if (captureMap['definition.static']) return 'Static'; + if (captureMap['definition.typedef']) return 'Typedef'; + if (captureMap['definition.macro']) return 'Macro'; + if (captureMap['definition.union']) return 'Union'; + if (captureMap['definition.property']) return 'Property'; + if (captureMap['definition.record']) return 'Record'; + if (captureMap['definition.delegate']) return 'Delegate'; + if (captureMap['definition.annotation']) return 'Annotation'; + if (captureMap['definition.constructor']) return 'Constructor'; + if (captureMap['definition.template']) return 'Template'; + return 'CodeElement'; +}; + +// ============================================================================ +// Process a batch of files +// ============================================================================ + +const processBatch = (files: ParseWorkerInput[], onProgress?: (filesProcessed: number) => void): ParseWorkerResult => { + const result: ParseWorkerResult = { + nodes: [], + relationships: [], + symbols: [], + imports: [], + calls: [], + heritage: [], + fileCount: 0, + }; + + // Group by language to minimize setLanguage calls + const byLanguage = new Map(); + for (const file of files) { + const lang = getLanguageFromFilename(file.path); + if (!lang) continue; + let list = byLanguage.get(lang); + if (!list) { + list = []; + byLanguage.set(lang, list); + } + list.push(file); + } + + let totalProcessed = 0; + let lastReported = 0; + const PROGRESS_INTERVAL = 100; // report every 100 files + + const onFileProcessed = onProgress ? () => { + totalProcessed++; + if (totalProcessed - lastReported >= PROGRESS_INTERVAL) { + lastReported = totalProcessed; + onProgress(totalProcessed); + } + } : undefined; + + for (const [language, langFiles] of byLanguage) { + const queryString = LANGUAGE_QUERIES[language]; + if (!queryString) continue; + + // Track if we need to handle tsx separately + const tsxFiles: ParseWorkerInput[] = []; + const regularFiles: ParseWorkerInput[] = []; + + if (language === SupportedLanguages.TypeScript) { + for (const f of langFiles) { + if (f.path.endsWith('.tsx')) { + tsxFiles.push(f); + } else { + regularFiles.push(f); + } + } + } else { + regularFiles.push(...langFiles); + } + + // Process regular files for this language + if (regularFiles.length > 0) { + setLanguage(language, regularFiles[0].path); + processFileGroup(regularFiles, language, queryString, result, onFileProcessed); + } + + // Process tsx files separately (different grammar) + if (tsxFiles.length > 0) { + setLanguage(language, tsxFiles[0].path); + processFileGroup(tsxFiles, language, queryString, result, onFileProcessed); + } + } + + return result; +}; + +const processFileGroup = ( + files: ParseWorkerInput[], + language: SupportedLanguages, + queryString: string, + result: ParseWorkerResult, + onFileProcessed?: () => void, +): void => { + let query: any; + try { + const lang = parser.getLanguage(); + query = new Parser.Query(lang, queryString); + } catch { + return; + } + + for (const file of files) { + let tree; + try { + tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 }); + } catch { + continue; + } + + result.fileCount++; + onFileProcessed?.(); + + let matches; + try { + matches = query.matches(tree.rootNode); + } catch { + continue; + } + + for (const match of matches) { + const captureMap: Record = {}; + for (const c of match.captures) { + captureMap[c.name] = c.node; + } + + // Extract import paths before skipping + if (captureMap['import'] && captureMap['import.source']) { + const rawImportPath = captureMap['import.source'].text.replace(/['"<>]/g, ''); + result.imports.push({ + filePath: file.path, + rawImportPath, + language: language, + }); + continue; + } + + // Extract call sites + if (captureMap['call']) { + const callNameNode = captureMap['call.name']; + if (callNameNode) { + const calledName = callNameNode.text; + if (!BUILT_INS.has(calledName)) { + const callNode = captureMap['call']; + const sourceId = findEnclosingFunctionId(callNode, file.path) + || generateId('File', file.path); + result.calls.push({ filePath: file.path, calledName, sourceId }); + } + } + continue; + } + + // Extract heritage (extends/implements) + if (captureMap['heritage.class']) { + if (captureMap['heritage.extends']) { + result.heritage.push({ + filePath: file.path, + className: captureMap['heritage.class'].text, + parentName: captureMap['heritage.extends'].text, + kind: 'extends', + }); + } + if (captureMap['heritage.implements']) { + result.heritage.push({ + filePath: file.path, + className: captureMap['heritage.class'].text, + parentName: captureMap['heritage.implements'].text, + kind: 'implements', + }); + } + if (captureMap['heritage.trait']) { + result.heritage.push({ + filePath: file.path, + className: captureMap['heritage.class'].text, + parentName: captureMap['heritage.trait'].text, + kind: 'trait-impl', + }); + } + if (captureMap['heritage.extends'] || captureMap['heritage.implements'] || captureMap['heritage.trait']) { + continue; + } + } + + const nodeLabel = getLabelFromCaptures(captureMap); + if (!nodeLabel) continue; + + const nameNode = captureMap['name']; + const nodeName = nameNode.text; + const nodeId = generateId(nodeLabel, `${file.path}:${nodeName}`); + + result.nodes.push({ + id: nodeId, + label: nodeLabel, + properties: { + name: nodeName, + filePath: file.path, + startLine: nameNode.startPosition.row, + endLine: nameNode.endPosition.row, + language: language, + isExported: isNodeExported(nameNode, nodeName, language), + }, + }); + + result.symbols.push({ + filePath: file.path, + name: nodeName, + nodeId, + type: nodeLabel, + }); + + const fileId = generateId('File', file.path); + const relId = generateId('DEFINES', `${fileId}->${nodeId}`); + result.relationships.push({ + id: relId, + sourceId: fileId, + targetId: nodeId, + type: 'DEFINES', + confidence: 1.0, + reason: '', + }); + } + } +}; + +// ============================================================================ +// Worker message handler +// ============================================================================ + +parentPort!.on('message', (files: ParseWorkerInput[]) => { + const result = processBatch(files, (filesProcessed) => { + parentPort!.postMessage({ type: 'progress', filesProcessed }); + }); + parentPort!.postMessage({ type: 'result', data: result }); +}); diff --git a/gitnexus/src/core/ingestion/workers/worker-pool.ts b/gitnexus/src/core/ingestion/workers/worker-pool.ts new file mode 100644 index 0000000000..aad23b178e --- /dev/null +++ b/gitnexus/src/core/ingestion/workers/worker-pool.ts @@ -0,0 +1,89 @@ +import { Worker } from 'node:worker_threads'; +import os from 'node:os'; + +export interface WorkerPool { + /** + * Dispatch items across workers. Items are split into chunks (one per worker), + * each worker processes its chunk, and results are concatenated back in order. + * + * @param onProgress - Called with cumulative files processed across all workers + */ + dispatch(items: TInput[], onProgress?: (filesProcessed: number) => void): Promise; + + /** + * Terminate all workers. Must be called when done. + */ + terminate(): Promise; + + /** Number of workers in the pool */ + readonly size: number; +} + +/** + * Create a pool of worker threads. + * + * @param workerUrl - URL to the worker script (use `new URL('./parse-worker.js', import.meta.url)`) + * @param poolSize - Number of workers (defaults to cpus - 1, minimum 1) + */ +export const createWorkerPool = (workerUrl: URL, poolSize?: number): WorkerPool => { + const size = poolSize ?? Math.max(1, os.cpus().length - 1); + const workers: Worker[] = []; + + for (let i = 0; i < size; i++) { + workers.push(new Worker(workerUrl)); + } + + const dispatch = (items: TInput[], onProgress?: (filesProcessed: number) => void): Promise => { + if (items.length === 0) return Promise.resolve([]); + + // Split items into one chunk per worker + const chunkSize = Math.ceil(items.length / size); + const chunks: TInput[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + + // Track per-worker progress for cumulative reporting + const workerProgress = new Array(chunks.length).fill(0); + + // Send one chunk to each worker, collect results + const promises = chunks.map((chunk, i) => { + const worker = workers[i]; + return new Promise((resolve, reject) => { + const handler = (msg: any) => { + if (msg && msg.type === 'progress') { + // Intermediate progress from worker + workerProgress[i] = msg.filesProcessed; + if (onProgress) { + const total = workerProgress.reduce((a, b) => a + b, 0); + onProgress(total); + } + } else if (msg && msg.type === 'result') { + // Final result + worker.removeListener('message', handler); + resolve(msg.data); + } else { + // Legacy: treat any non-typed message as result (backward compat) + worker.removeListener('message', handler); + resolve(msg); + } + }; + worker.on('message', handler); + worker.once('error', (err) => { + worker.removeListener('message', handler); + reject(err); + }); + worker.postMessage(chunk); + }); + }); + + return Promise.all(promises); + }; + + const terminate = async (): Promise => { + await Promise.all(workers.map(w => w.terminate())); + workers.length = 0; + }; + + return { dispatch, terminate, size }; +}; diff --git a/gitnexus/src/core/kuzu/kuzu-adapter.ts b/gitnexus/src/core/kuzu/kuzu-adapter.ts index 972e8631cd..9988b56ede 100644 --- a/gitnexus/src/core/kuzu/kuzu-adapter.ts +++ b/gitnexus/src/core/kuzu/kuzu-adapter.ts @@ -11,8 +11,6 @@ import { } from './schema.js'; import { generateAllCSVs } from './csv-generator.js'; -const isDev = process.env.NODE_ENV === 'development'; - let db: kuzu.Database | null = null; let conn: kuzu.Connection | null = null; diff --git a/gitnexus/src/core/kuzu/schema.ts b/gitnexus/src/core/kuzu/schema.ts index d74545c069..c0f3b394a0 100644 --- a/gitnexus/src/core/kuzu/schema.ts +++ b/gitnexus/src/core/kuzu/schema.ts @@ -241,6 +241,7 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM Class TO \`TypeAlias\`, FROM Class TO \`Struct\`, FROM Class TO \`Enum\`, + FROM Class TO \`Annotation\`, FROM Class TO \`Constructor\`, FROM Method TO Function, FROM Method TO Method, @@ -310,8 +311,10 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM \`Constructor\` TO \`Template\`, FROM \`Constructor\` TO \`TypeAlias\`, FROM \`Constructor\` TO \`Enum\`, + FROM \`Constructor\` TO \`Annotation\`, FROM \`Constructor\` TO \`Impl\`, FROM \`Constructor\` TO \`Namespace\`, + FROM \`Constructor\` TO \`Module\`, FROM \`Template\` TO Community, FROM \`Module\` TO Community, FROM Function TO Process, diff --git a/gitnexus/src/core/wiki/generator.ts b/gitnexus/src/core/wiki/generator.ts new file mode 100644 index 0000000000..092ce2e531 --- /dev/null +++ b/gitnexus/src/core/wiki/generator.ts @@ -0,0 +1,819 @@ +/** + * Wiki Generator + * + * Orchestrates the full wiki generation pipeline: + * Phase 0: Validate prerequisites + gather graph structure + * Phase 1: Build module tree (one LLM call) + * Phase 2: Generate module pages (one LLM call per module, bottom-up) + * Phase 3: Generate overview page + * + * Supports incremental updates via git diff + module-file mapping. + */ + +import fs from 'fs/promises'; +import path from 'path'; +import { execSync } from 'child_process'; + +import { + initWikiDb, + closeWikiDb, + getFilesWithExports, + getAllFiles, + getInterFileCallEdges, + getIntraModuleCallEdges, + getInterModuleCallEdges, + getProcessesForFiles, + getAllProcesses, + getInterModuleEdgesForOverview, + type FileWithExports, +} from './graph-queries.js'; + +import { + callLLM, + estimateTokens, + type LLMConfig, +} from './llm-client.js'; + +import { + GROUPING_SYSTEM_PROMPT, + GROUPING_USER_PROMPT, + MODULE_SYSTEM_PROMPT, + MODULE_USER_PROMPT, + PARENT_SYSTEM_PROMPT, + PARENT_USER_PROMPT, + OVERVIEW_SYSTEM_PROMPT, + OVERVIEW_USER_PROMPT, + fillTemplate, + formatFileListForGrouping, + formatDirectoryTree, + formatCallEdges, + formatProcesses, +} from './prompts.js'; + +import { shouldIgnorePath } from '../../config/ignore-service.js'; + +// โ”€โ”€โ”€ Types โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +export interface WikiOptions { + force?: boolean; + model?: string; + baseUrl?: string; + apiKey?: string; + maxTokensPerModule?: number; +} + +export interface WikiMeta { + fromCommit: string; + generatedAt: string; + model: string; + moduleFiles: Record; + moduleTree: ModuleTreeNode[]; +} + +export interface ModuleTreeNode { + name: string; + slug: string; + files: string[]; + children?: ModuleTreeNode[]; +} + +export type ProgressCallback = (phase: string, percent: number, detail?: string) => void; + +// โ”€โ”€โ”€ Constants โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +const DEFAULT_MAX_TOKENS_PER_MODULE = 30_000; +const WIKI_DIR = 'wiki'; + +// โ”€โ”€โ”€ Generator Class โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +export class WikiGenerator { + private repoPath: string; + private storagePath: string; + private wikiDir: string; + private kuzuPath: string; + private llmConfig: LLMConfig; + private maxTokensPerModule: number; + private options: WikiOptions; + private onProgress: ProgressCallback; + private failedModules: string[] = []; + + constructor( + repoPath: string, + storagePath: string, + kuzuPath: string, + llmConfig: LLMConfig, + options: WikiOptions = {}, + onProgress?: ProgressCallback, + ) { + this.repoPath = repoPath; + this.storagePath = storagePath; + this.wikiDir = path.join(storagePath, WIKI_DIR); + this.kuzuPath = kuzuPath; + this.options = options; + this.llmConfig = llmConfig; + this.maxTokensPerModule = options.maxTokensPerModule ?? DEFAULT_MAX_TOKENS_PER_MODULE; + this.onProgress = onProgress || (() => {}); + } + + /** + * Main entry point. Runs the full pipeline or incremental update. + */ + async run(): Promise<{ pagesGenerated: number; mode: 'full' | 'incremental' | 'up-to-date'; failedModules: string[] }> { + await fs.mkdir(this.wikiDir, { recursive: true }); + + const existingMeta = await this.loadWikiMeta(); + const currentCommit = this.getCurrentCommit(); + const forceMode = this.options.force; + + // Up-to-date check (skip if --force) + if (!forceMode && existingMeta && existingMeta.fromCommit === currentCommit) { + return { pagesGenerated: 0, mode: 'up-to-date', failedModules: [] }; + } + + // Force mode: delete snapshot to force full re-grouping + if (forceMode) { + try { await fs.unlink(path.join(this.wikiDir, 'first_module_tree.json')); } catch {} + // Delete existing module pages so they get regenerated + const existingFiles = await fs.readdir(this.wikiDir).catch(() => [] as string[]); + for (const f of existingFiles) { + if (f.endsWith('.md')) { + try { await fs.unlink(path.join(this.wikiDir, f)); } catch {} + } + } + } + + // Init graph + this.onProgress('init', 2, 'Connecting to knowledge graph...'); + await initWikiDb(this.kuzuPath); + + try { + if (!forceMode && existingMeta && existingMeta.fromCommit) { + return await this.incrementalUpdate(existingMeta, currentCommit); + } + return await this.fullGeneration(currentCommit); + } finally { + await closeWikiDb(); + } + } + + // โ”€โ”€โ”€ Full Generation โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + private async fullGeneration(currentCommit: string): Promise<{ pagesGenerated: number; mode: 'full'; failedModules: string[] }> { + let pagesGenerated = 0; + + // Phase 0: Gather structure + this.onProgress('gather', 5, 'Querying graph for file structure...'); + const filesWithExports = await getFilesWithExports(); + const allFiles = await getAllFiles(); + + // Filter to source files only + const sourceFiles = allFiles.filter(f => !shouldIgnorePath(f)); + if (sourceFiles.length === 0) { + throw new Error('No source files found in the knowledge graph. Nothing to document.'); + } + + // Build enriched file list (merge exports into all source files) + const exportMap = new Map(filesWithExports.map(f => [f.filePath, f])); + const enrichedFiles: FileWithExports[] = sourceFiles.map(fp => { + return exportMap.get(fp) || { filePath: fp, symbols: [] }; + }); + + this.onProgress('gather', 10, `Found ${sourceFiles.length} source files`); + + // Phase 1: Build module tree + const moduleTree = await this.buildModuleTree(enrichedFiles); + pagesGenerated = 0; + + // Phase 2: Generate module pages (bottom-up) + const totalModules = this.countModules(moduleTree); + let modulesProcessed = 0; + + for (const node of moduleTree) { + const generated = await this.generateModulePage(node, () => { + modulesProcessed++; + const percent = 30 + Math.round((modulesProcessed / totalModules) * 55); + this.onProgress('modules', percent, `${modulesProcessed}/${totalModules} modules`); + }); + pagesGenerated += generated; + } + + // Phase 3: Generate overview + this.onProgress('overview', 88, 'Generating overview page...'); + await this.generateOverview(moduleTree); + pagesGenerated++; + + // Save metadata + this.onProgress('finalize', 95, 'Saving metadata...'); + const moduleFiles = this.extractModuleFiles(moduleTree); + await this.saveModuleTree(moduleTree); + await this.saveWikiMeta({ + fromCommit: currentCommit, + generatedAt: new Date().toISOString(), + model: this.llmConfig.model, + moduleFiles, + moduleTree, + }); + + this.onProgress('done', 100, 'Wiki generation complete'); + return { pagesGenerated, mode: 'full', failedModules: [...this.failedModules] }; + } + + // โ”€โ”€โ”€ Phase 1: Build Module Tree โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + private async buildModuleTree(files: FileWithExports[]): Promise { + // Check for existing immutable snapshot (resumability) + const snapshotPath = path.join(this.wikiDir, 'first_module_tree.json'); + try { + const existing = await fs.readFile(snapshotPath, 'utf-8'); + const parsed = JSON.parse(existing); + if (Array.isArray(parsed) && parsed.length > 0) { + this.onProgress('grouping', 25, 'Using existing module tree (resuming)'); + return parsed; + } + } catch { + // No snapshot, generate new + } + + this.onProgress('grouping', 15, 'Grouping files into modules (LLM)...'); + + const fileList = formatFileListForGrouping(files); + const dirTree = formatDirectoryTree(files.map(f => f.filePath)); + + const prompt = fillTemplate(GROUPING_USER_PROMPT, { + FILE_LIST: fileList, + DIRECTORY_TREE: dirTree, + }); + + const response = await callLLM(prompt, this.llmConfig, GROUPING_SYSTEM_PROMPT); + const grouping = this.parseGroupingResponse(response.content, files); + + // Convert to tree nodes + const tree: ModuleTreeNode[] = []; + for (const [moduleName, modulePaths] of Object.entries(grouping)) { + const slug = this.slugify(moduleName); + const node: ModuleTreeNode = { name: moduleName, slug, files: modulePaths }; + + // Token budget check โ€” split if too large + const totalTokens = await this.estimateModuleTokens(modulePaths); + if (totalTokens > this.maxTokensPerModule && modulePaths.length > 3) { + node.children = this.splitBySubdirectory(moduleName, modulePaths); + node.files = []; // Parent doesn't own files directly when split + } + + tree.push(node); + } + + // Save immutable snapshot for resumability + await fs.writeFile(snapshotPath, JSON.stringify(tree, null, 2), 'utf-8'); + this.onProgress('grouping', 28, `Created ${tree.length} modules`); + + return tree; + } + + /** + * Parse LLM grouping response. Validates all files are assigned. + */ + private parseGroupingResponse( + content: string, + files: FileWithExports[], + ): Record { + // Extract JSON from response (handle markdown fences) + let jsonStr = content.trim(); + const fenceMatch = jsonStr.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/); + if (fenceMatch) { + jsonStr = fenceMatch[1].trim(); + } + + let parsed: Record; + try { + parsed = JSON.parse(jsonStr); + } catch { + // Fallback: group by top-level directory + return this.fallbackGrouping(files); + } + + if (typeof parsed !== 'object' || Array.isArray(parsed)) { + return this.fallbackGrouping(files); + } + + // Validate โ€” ensure all files are assigned + const allFilePaths = new Set(files.map(f => f.filePath)); + const assignedFiles = new Set(); + const validGrouping: Record = {}; + + for (const [mod, paths] of Object.entries(parsed)) { + if (!Array.isArray(paths)) continue; + const validPaths = paths.filter(p => { + if (allFilePaths.has(p) && !assignedFiles.has(p)) { + assignedFiles.add(p); + return true; + } + return false; + }); + if (validPaths.length > 0) { + validGrouping[mod] = validPaths; + } + } + + // Assign unassigned files to a "Miscellaneous" module + const unassigned = files + .map(f => f.filePath) + .filter(fp => !assignedFiles.has(fp)); + if (unassigned.length > 0) { + validGrouping['Other'] = unassigned; + } + + return Object.keys(validGrouping).length > 0 + ? validGrouping + : this.fallbackGrouping(files); + } + + /** + * Fallback grouping by top-level directory when LLM parsing fails. + */ + private fallbackGrouping(files: FileWithExports[]): Record { + const groups = new Map(); + for (const f of files) { + const parts = f.filePath.replace(/\\/g, '/').split('/'); + const topDir = parts.length > 1 ? parts[0] : 'Root'; + let group = groups.get(topDir); + if (!group) { group = []; groups.set(topDir, group); } + group.push(f.filePath); + } + return Object.fromEntries(groups); + } + + /** + * Split a large module into sub-modules by subdirectory. + */ + private splitBySubdirectory(moduleName: string, files: string[]): ModuleTreeNode[] { + const subGroups = new Map(); + for (const fp of files) { + const parts = fp.replace(/\\/g, '/').split('/'); + // Use the deepest common-ish directory + const subDir = parts.length > 2 ? parts.slice(0, 2).join('/') : parts[0]; + let group = subGroups.get(subDir); + if (!group) { group = []; subGroups.set(subDir, group); } + group.push(fp); + } + + return Array.from(subGroups.entries()).map(([subDir, subFiles]) => ({ + name: `${moduleName} โ€” ${path.basename(subDir)}`, + slug: this.slugify(`${moduleName}-${path.basename(subDir)}`), + files: subFiles, + })); + } + + // โ”€โ”€โ”€ Phase 2: Generate Module Pages โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + /** + * Recursively generate pages for a module tree node. + * Returns count of pages generated. + */ + private async generateModulePage( + node: ModuleTreeNode, + onPageDone: () => void, + ): Promise { + let count = 0; + + // If node has children, generate children first (bottom-up) + if (node.children && node.children.length > 0) { + for (const child of node.children) { + count += await this.generateModulePage(child, onPageDone); + } + + // Then generate parent page from children docs + const pagePath = path.join(this.wikiDir, `${node.slug}.md`); + + // Resumability: skip if page already exists + if (await this.fileExists(pagePath)) { + onPageDone(); + return count; + } + + try { + await this.generateParentPage(node); + count++; + } catch (err: any) { + this.failedModules.push(node.name); + this.onProgress('modules', 0, `Failed: ${node.name} โ€” ${err.message?.slice(0, 80)}`); + } + onPageDone(); + return count; + } + + // Leaf module โ€” generate from source code + const pagePath = path.join(this.wikiDir, `${node.slug}.md`); + + // Resumability: skip if page already exists + if (await this.fileExists(pagePath)) { + onPageDone(); + return count; + } + + try { + await this.generateLeafPage(node); + count++; + } catch (err: any) { + this.failedModules.push(node.name); + this.onProgress('modules', 0, `Failed: ${node.name} โ€” ${err.message?.slice(0, 80)}`); + } + onPageDone(); + return count; + } + + /** + * Generate a leaf module page from source code + graph data. + */ + private async generateLeafPage(node: ModuleTreeNode): Promise { + const filePaths = node.files; + + // Read source files from disk + const sourceCode = await this.readSourceFiles(filePaths); + + // Token budget check โ€” if too large, summarize in batches + const totalTokens = estimateTokens(sourceCode); + let finalSourceCode = sourceCode; + if (totalTokens > this.maxTokensPerModule) { + finalSourceCode = this.truncateSource(sourceCode, this.maxTokensPerModule); + } + + // Get graph data + const [intraCalls, interCalls, processes] = await Promise.all([ + getIntraModuleCallEdges(filePaths), + getInterModuleCallEdges(filePaths), + getProcessesForFiles(filePaths, 5), + ]); + + const prompt = fillTemplate(MODULE_USER_PROMPT, { + MODULE_NAME: node.name, + SOURCE_CODE: finalSourceCode, + INTRA_CALLS: formatCallEdges(intraCalls), + OUTGOING_CALLS: formatCallEdges(interCalls.outgoing), + INCOMING_CALLS: formatCallEdges(interCalls.incoming), + PROCESSES: formatProcesses(processes), + }); + + const response = await callLLM(prompt, this.llmConfig, MODULE_SYSTEM_PROMPT); + + // Write page with front matter + const pageContent = `# ${node.name}\n\n${response.content}`; + await fs.writeFile(path.join(this.wikiDir, `${node.slug}.md`), pageContent, 'utf-8'); + } + + /** + * Generate a parent module page from children's documentation. + */ + private async generateParentPage(node: ModuleTreeNode): Promise { + if (!node.children || node.children.length === 0) return; + + // Read children's overview sections + const childDocs: string[] = []; + for (const child of node.children) { + const childPage = path.join(this.wikiDir, `${child.slug}.md`); + try { + const content = await fs.readFile(childPage, 'utf-8'); + // Extract overview section (first ~500 chars or up to "### Architecture") + const overviewEnd = content.indexOf('### Architecture'); + const overview = overviewEnd > 0 ? content.slice(0, overviewEnd).trim() : content.slice(0, 800).trim(); + childDocs.push(`#### ${child.name}\n${overview}`); + } catch { + childDocs.push(`#### ${child.name}\n(Documentation not yet generated)`); + } + } + + // Get cross-child call edges + const allChildFiles = node.children.flatMap(c => c.files); + const crossCalls = await getIntraModuleCallEdges(allChildFiles); + const processes = await getProcessesForFiles(allChildFiles, 3); + + const prompt = fillTemplate(PARENT_USER_PROMPT, { + MODULE_NAME: node.name, + CHILDREN_DOCS: childDocs.join('\n\n'), + CROSS_MODULE_CALLS: formatCallEdges(crossCalls), + CROSS_PROCESSES: formatProcesses(processes), + }); + + const response = await callLLM(prompt, this.llmConfig, PARENT_SYSTEM_PROMPT); + + const pageContent = `# ${node.name}\n\n${response.content}`; + await fs.writeFile(path.join(this.wikiDir, `${node.slug}.md`), pageContent, 'utf-8'); + } + + // โ”€โ”€โ”€ Phase 3: Generate Overview โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + private async generateOverview(moduleTree: ModuleTreeNode[]): Promise { + // Read module overview sections + const moduleSummaries: string[] = []; + for (const node of moduleTree) { + const pagePath = path.join(this.wikiDir, `${node.slug}.md`); + try { + const content = await fs.readFile(pagePath, 'utf-8'); + const overviewEnd = content.indexOf('### Architecture'); + const overview = overviewEnd > 0 ? content.slice(0, overviewEnd).trim() : content.slice(0, 600).trim(); + moduleSummaries.push(`#### ${node.name}\n${overview}`); + } catch { + moduleSummaries.push(`#### ${node.name}\n(Documentation pending)`); + } + } + + // Get inter-module edges for architecture diagram + const moduleFiles = this.extractModuleFiles(moduleTree); + const moduleEdges = await getInterModuleEdgesForOverview(moduleFiles); + + // Get top processes for key workflows + const topProcesses = await getAllProcesses(5); + + // Read project config + const projectInfo = await this.readProjectInfo(); + + const edgesText = moduleEdges.length > 0 + ? moduleEdges.map(e => `${e.from} โ†’ ${e.to} (${e.count} calls)`).join('\n') + : 'No inter-module call edges detected'; + + const prompt = fillTemplate(OVERVIEW_USER_PROMPT, { + PROJECT_INFO: projectInfo, + MODULE_SUMMARIES: moduleSummaries.join('\n\n'), + MODULE_EDGES: edgesText, + TOP_PROCESSES: formatProcesses(topProcesses), + }); + + const response = await callLLM(prompt, this.llmConfig, OVERVIEW_SYSTEM_PROMPT); + + const pageContent = `# ${path.basename(this.repoPath)} โ€” Wiki\n\n${response.content}`; + await fs.writeFile(path.join(this.wikiDir, 'overview.md'), pageContent, 'utf-8'); + } + + // โ”€โ”€โ”€ Incremental Updates โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + private async incrementalUpdate( + existingMeta: WikiMeta, + currentCommit: string, + ): Promise<{ pagesGenerated: number; mode: 'incremental'; failedModules: string[] }> { + this.onProgress('incremental', 5, 'Detecting changes...'); + + // Get changed files since last generation + const changedFiles = this.getChangedFiles(existingMeta.fromCommit, currentCommit); + if (changedFiles.length === 0) { + // No file changes but commit differs (e.g. merge commit) + await this.saveWikiMeta({ + ...existingMeta, + fromCommit: currentCommit, + generatedAt: new Date().toISOString(), + }); + return { pagesGenerated: 0, mode: 'incremental', failedModules: [] }; + } + + this.onProgress('incremental', 10, `${changedFiles.length} files changed`); + + // Determine affected modules + const affectedModules = new Set(); + const newFiles: string[] = []; + + for (const fp of changedFiles) { + let found = false; + for (const [mod, files] of Object.entries(existingMeta.moduleFiles)) { + if (files.includes(fp)) { + affectedModules.add(mod); + found = true; + break; + } + } + if (!found && !shouldIgnorePath(fp)) { + newFiles.push(fp); + } + } + + // If significant new files exist, re-run full grouping + if (newFiles.length > 5) { + this.onProgress('incremental', 15, 'Significant new files detected, running full generation...'); + // Delete old snapshot to force re-grouping + try { await fs.unlink(path.join(this.wikiDir, 'first_module_tree.json')); } catch {} + const fullResult = await this.fullGeneration(currentCommit); + return { ...fullResult, mode: 'incremental' }; + } + + // Add new files to nearest module or "Other" + if (newFiles.length > 0) { + if (!existingMeta.moduleFiles['Other']) { + existingMeta.moduleFiles['Other'] = []; + } + existingMeta.moduleFiles['Other'].push(...newFiles); + affectedModules.add('Other'); + } + + // Regenerate affected module pages + let pagesGenerated = 0; + const moduleTree = existingMeta.moduleTree; + const affectedArray = Array.from(affectedModules); + + this.onProgress('incremental', 20, `Regenerating ${affectedArray.length} module(s)...`); + + for (let i = 0; i < affectedArray.length; i++) { + const modSlug = this.slugify(affectedArray[i]); + const node = this.findNodeBySlug(moduleTree, modSlug); + + if (node) { + // Delete existing page to force re-generation + try { await fs.unlink(path.join(this.wikiDir, `${node.slug}.md`)); } catch {} + + await this.generateModulePage(node, () => {}); + pagesGenerated++; + } + + const percent = 20 + Math.round(((i + 1) / affectedArray.length) * 60); + this.onProgress('incremental', percent, `${i + 1}/${affectedArray.length} modules`); + } + + // Regenerate overview if any pages changed + if (pagesGenerated > 0) { + this.onProgress('incremental', 85, 'Updating overview...'); + await this.generateOverview(moduleTree); + pagesGenerated++; + } + + // Save updated metadata + this.onProgress('incremental', 95, 'Saving metadata...'); + await this.saveWikiMeta({ + ...existingMeta, + fromCommit: currentCommit, + generatedAt: new Date().toISOString(), + model: this.llmConfig.model, + }); + + this.onProgress('done', 100, 'Incremental update complete'); + return { pagesGenerated, mode: 'incremental', failedModules: [...this.failedModules] }; + } + + // โ”€โ”€โ”€ Helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + private getCurrentCommit(): string { + try { + return execSync('git rev-parse HEAD', { cwd: this.repoPath }).toString().trim(); + } catch { + return ''; + } + } + + private getChangedFiles(fromCommit: string, toCommit: string): string[] { + try { + const output = execSync( + `git diff ${fromCommit}..${toCommit} --name-only`, + { cwd: this.repoPath }, + ).toString().trim(); + return output ? output.split('\n').filter(Boolean) : []; + } catch { + return []; + } + } + + private async readSourceFiles(filePaths: string[]): Promise { + const parts: string[] = []; + for (const fp of filePaths) { + const fullPath = path.join(this.repoPath, fp); + try { + const content = await fs.readFile(fullPath, 'utf-8'); + parts.push(`\n--- ${fp} ---\n${content}`); + } catch { + parts.push(`\n--- ${fp} ---\n(file not readable)`); + } + } + return parts.join('\n'); + } + + private truncateSource(source: string, maxTokens: number): string { + // Rough truncation: keep first maxTokens*4 chars and add notice + const maxChars = maxTokens * 4; + if (source.length <= maxChars) return source; + return source.slice(0, maxChars) + '\n\n... (source truncated for context window limits)'; + } + + private async estimateModuleTokens(filePaths: string[]): Promise { + let total = 0; + for (const fp of filePaths) { + try { + const content = await fs.readFile(path.join(this.repoPath, fp), 'utf-8'); + total += estimateTokens(content); + } catch { + // File not readable, skip + } + } + return total; + } + + private async readProjectInfo(): Promise { + const candidates = ['package.json', 'Cargo.toml', 'pyproject.toml', 'go.mod', 'pom.xml', 'build.gradle']; + const lines: string[] = [`Project: ${path.basename(this.repoPath)}`]; + + for (const file of candidates) { + const fullPath = path.join(this.repoPath, file); + try { + const content = await fs.readFile(fullPath, 'utf-8'); + if (file === 'package.json') { + const pkg = JSON.parse(content); + if (pkg.name) lines.push(`Name: ${pkg.name}`); + if (pkg.description) lines.push(`Description: ${pkg.description}`); + if (pkg.scripts) lines.push(`Scripts: ${Object.keys(pkg.scripts).join(', ')}`); + } else { + // Include first 500 chars of other config files + lines.push(`\n${file}:\n${content.slice(0, 500)}`); + } + break; // Use first config found + } catch { + continue; + } + } + + // Read README excerpt + for (const readme of ['README.md', 'readme.md', 'README.txt']) { + try { + const content = await fs.readFile(path.join(this.repoPath, readme), 'utf-8'); + lines.push(`\nREADME excerpt:\n${content.slice(0, 1000)}`); + break; + } catch { + continue; + } + } + + return lines.join('\n'); + } + + private extractModuleFiles(tree: ModuleTreeNode[]): Record { + const result: Record = {}; + for (const node of tree) { + if (node.children && node.children.length > 0) { + result[node.name] = node.children.flatMap(c => c.files); + for (const child of node.children) { + result[child.name] = child.files; + } + } else { + result[node.name] = node.files; + } + } + return result; + } + + private countModules(tree: ModuleTreeNode[]): number { + let count = 0; + for (const node of tree) { + count++; + if (node.children) { + count += node.children.length; + } + } + return count; + } + + private findNodeBySlug(tree: ModuleTreeNode[], slug: string): ModuleTreeNode | null { + for (const node of tree) { + if (node.slug === slug) return node; + if (node.children) { + const found = this.findNodeBySlug(node.children, slug); + if (found) return found; + } + } + return null; + } + + private slugify(name: string): string { + return name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 60); + } + + private async fileExists(fp: string): Promise { + try { + await fs.access(fp); + return true; + } catch { + return false; + } + } + + private async loadWikiMeta(): Promise { + try { + const raw = await fs.readFile(path.join(this.wikiDir, 'meta.json'), 'utf-8'); + return JSON.parse(raw) as WikiMeta; + } catch { + return null; + } + } + + private async saveWikiMeta(meta: WikiMeta): Promise { + await fs.writeFile( + path.join(this.wikiDir, 'meta.json'), + JSON.stringify(meta, null, 2), + 'utf-8', + ); + } + + private async saveModuleTree(tree: ModuleTreeNode[]): Promise { + await fs.writeFile( + path.join(this.wikiDir, 'module_tree.json'), + JSON.stringify(tree, null, 2), + 'utf-8', + ); + } +} diff --git a/gitnexus/src/core/wiki/graph-queries.ts b/gitnexus/src/core/wiki/graph-queries.ts new file mode 100644 index 0000000000..f32161e167 --- /dev/null +++ b/gitnexus/src/core/wiki/graph-queries.ts @@ -0,0 +1,300 @@ +/** + * Graph Queries for Wiki Generation + * + * Encapsulated Cypher queries against the GitNexus knowledge graph. + * Uses the MCP-style pooled kuzu-adapter for connection management. + */ + +import { initKuzu, executeQuery, closeKuzu } from '../../mcp/core/kuzu-adapter.js'; + +const REPO_ID = '__wiki__'; + +export interface FileWithExports { + filePath: string; + symbols: Array<{ name: string; type: string }>; +} + +export interface CallEdge { + fromFile: string; + fromName: string; + toFile: string; + toName: string; +} + +export interface ProcessInfo { + id: string; + label: string; + type: string; + stepCount: number; + steps: Array<{ + step: number; + name: string; + filePath: string; + type: string; + }>; +} + +/** + * Initialize the KuzuDB connection for wiki generation. + */ +export async function initWikiDb(kuzuPath: string): Promise { + await initKuzu(REPO_ID, kuzuPath); +} + +/** + * Close the KuzuDB connection. + */ +export async function closeWikiDb(): Promise { + await closeKuzu(REPO_ID); +} + +/** + * Get all source files with their exported symbol names and types. + */ +export async function getFilesWithExports(): Promise { + const rows = await executeQuery(REPO_ID, ` + MATCH (f:File)-[:CodeRelation {type: 'DEFINES'}]->(n) + WHERE n.isExported = true + RETURN f.filePath AS filePath, n.name AS name, labels(n)[0] AS type + ORDER BY f.filePath + `); + + const fileMap = new Map(); + for (const row of rows) { + const fp = row.filePath || row[0]; + const name = row.name || row[1]; + const type = row.type || row[2]; + + let entry = fileMap.get(fp); + if (!entry) { + entry = { filePath: fp, symbols: [] }; + fileMap.set(fp, entry); + } + entry.symbols.push({ name, type }); + } + + return Array.from(fileMap.values()); +} + +/** + * Get all files tracked in the graph (including those with no exports). + */ +export async function getAllFiles(): Promise { + const rows = await executeQuery(REPO_ID, ` + MATCH (f:File) + RETURN f.filePath AS filePath + ORDER BY f.filePath + `); + return rows.map(r => r.filePath || r[0]); +} + +/** + * Get inter-file call edges (calls between different files). + */ +export async function getInterFileCallEdges(): Promise { + const rows = await executeQuery(REPO_ID, ` + MATCH (a)-[:CodeRelation {type: 'CALLS'}]->(b) + WHERE a.filePath <> b.filePath + RETURN DISTINCT a.filePath AS fromFile, a.name AS fromName, + b.filePath AS toFile, b.name AS toName + `); + + return rows.map(r => ({ + fromFile: r.fromFile || r[0], + fromName: r.fromName || r[1], + toFile: r.toFile || r[2], + toName: r.toName || r[3], + })); +} + +/** + * Get call edges between files within a specific set (intra-module). + */ +export async function getIntraModuleCallEdges(filePaths: string[]): Promise { + if (filePaths.length === 0) return []; + + const fileList = filePaths.map(f => `'${f.replace(/'/g, "''")}'`).join(', '); + const rows = await executeQuery(REPO_ID, ` + MATCH (a)-[:CodeRelation {type: 'CALLS'}]->(b) + WHERE a.filePath IN [${fileList}] AND b.filePath IN [${fileList}] + RETURN DISTINCT a.filePath AS fromFile, a.name AS fromName, + b.filePath AS toFile, b.name AS toName + `); + + return rows.map(r => ({ + fromFile: r.fromFile || r[0], + fromName: r.fromName || r[1], + toFile: r.toFile || r[2], + toName: r.toName || r[3], + })); +} + +/** + * Get call edges crossing module boundaries (external calls from/to module files). + */ +export async function getInterModuleCallEdges(filePaths: string[]): Promise<{ + outgoing: CallEdge[]; + incoming: CallEdge[]; +}> { + if (filePaths.length === 0) return { outgoing: [], incoming: [] }; + + const fileList = filePaths.map(f => `'${f.replace(/'/g, "''")}'`).join(', '); + + const outRows = await executeQuery(REPO_ID, ` + MATCH (a)-[:CodeRelation {type: 'CALLS'}]->(b) + WHERE a.filePath IN [${fileList}] AND NOT b.filePath IN [${fileList}] + RETURN DISTINCT a.filePath AS fromFile, a.name AS fromName, + b.filePath AS toFile, b.name AS toName + LIMIT 30 + `); + + const inRows = await executeQuery(REPO_ID, ` + MATCH (a)-[:CodeRelation {type: 'CALLS'}]->(b) + WHERE NOT a.filePath IN [${fileList}] AND b.filePath IN [${fileList}] + RETURN DISTINCT a.filePath AS fromFile, a.name AS fromName, + b.filePath AS toFile, b.name AS toName + LIMIT 30 + `); + + return { + outgoing: outRows.map(r => ({ + fromFile: r.fromFile || r[0], + fromName: r.fromName || r[1], + toFile: r.toFile || r[2], + toName: r.toName || r[3], + })), + incoming: inRows.map(r => ({ + fromFile: r.fromFile || r[0], + fromName: r.fromName || r[1], + toFile: r.toFile || r[2], + toName: r.toName || r[3], + })), + }; +} + +/** + * Get processes (execution flows) that pass through a set of files. + * Returns top N by step count. + */ +export async function getProcessesForFiles(filePaths: string[], limit = 5): Promise { + if (filePaths.length === 0) return []; + + const fileList = filePaths.map(f => `'${f.replace(/'/g, "''")}'`).join(', '); + + // Find processes that have steps in the given files + const procRows = await executeQuery(REPO_ID, ` + MATCH (s)-[r:CodeRelation {type: 'STEP_IN_PROCESS'}]->(p:Process) + WHERE s.filePath IN [${fileList}] + RETURN DISTINCT p.id AS id, p.heuristicLabel AS label, + p.processType AS type, p.stepCount AS stepCount + ORDER BY stepCount DESC + LIMIT ${limit} + `); + + const processes: ProcessInfo[] = []; + for (const row of procRows) { + const procId = row.id || row[0]; + const label = row.label || row[1] || procId; + const type = row.type || row[2] || 'unknown'; + const stepCount = row.stepCount || row[3] || 0; + + // Get the full step trace for this process + const stepRows = await executeQuery(REPO_ID, ` + MATCH (s)-[r:CodeRelation {type: 'STEP_IN_PROCESS'}]->(p:Process {id: '${procId.replace(/'/g, "''")}'}) + RETURN s.name AS name, s.filePath AS filePath, labels(s)[0] AS type, r.step AS step + ORDER BY r.step + `); + + processes.push({ + id: procId, + label, + type, + stepCount, + steps: stepRows.map(s => ({ + step: s.step || s[3] || 0, + name: s.name || s[0], + filePath: s.filePath || s[1], + type: s.type || s[2], + })), + }); + } + + return processes; +} + +/** + * Get all processes in the graph (for overview page). + */ +export async function getAllProcesses(limit = 20): Promise { + const procRows = await executeQuery(REPO_ID, ` + MATCH (p:Process) + RETURN p.id AS id, p.heuristicLabel AS label, + p.processType AS type, p.stepCount AS stepCount + ORDER BY stepCount DESC + LIMIT ${limit} + `); + + const processes: ProcessInfo[] = []; + for (const row of procRows) { + const procId = row.id || row[0]; + const label = row.label || row[1] || procId; + const type = row.type || row[2] || 'unknown'; + const stepCount = row.stepCount || row[3] || 0; + + const stepRows = await executeQuery(REPO_ID, ` + MATCH (s)-[r:CodeRelation {type: 'STEP_IN_PROCESS'}]->(p:Process {id: '${procId.replace(/'/g, "''")}'}) + RETURN s.name AS name, s.filePath AS filePath, labels(s)[0] AS type, r.step AS step + ORDER BY r.step + `); + + processes.push({ + id: procId, + label, + type, + stepCount, + steps: stepRows.map(s => ({ + step: s.step || s[3] || 0, + name: s.name || s[0], + filePath: s.filePath || s[1], + type: s.type || s[2], + })), + }); + } + + return processes; +} + +/** + * Get inter-module edges for overview architecture diagram. + * Groups call edges by source/target module. + */ +export async function getInterModuleEdgesForOverview( + moduleFiles: Record +): Promise> { + // Build file-to-module lookup + const fileToModule = new Map(); + for (const [mod, files] of Object.entries(moduleFiles)) { + for (const f of files) { + fileToModule.set(f, mod); + } + } + + const allEdges = await getInterFileCallEdges(); + const moduleEdgeCounts = new Map(); + + for (const edge of allEdges) { + const fromMod = fileToModule.get(edge.fromFile); + const toMod = fileToModule.get(edge.toFile); + if (fromMod && toMod && fromMod !== toMod) { + const key = `${fromMod}|||${toMod}`; + moduleEdgeCounts.set(key, (moduleEdgeCounts.get(key) || 0) + 1); + } + } + + return Array.from(moduleEdgeCounts.entries()) + .map(([key, count]) => { + const [from, to] = key.split('|||'); + return { from, to, count }; + }) + .sort((a, b) => b.count - a.count); +} diff --git a/gitnexus/src/core/wiki/llm-client.ts b/gitnexus/src/core/wiki/llm-client.ts new file mode 100644 index 0000000000..7306c4a0fd --- /dev/null +++ b/gitnexus/src/core/wiki/llm-client.ts @@ -0,0 +1,147 @@ +/** + * LLM Client for Wiki Generation + * + * OpenAI-compatible API client using native fetch. + * Supports OpenAI, Azure, LiteLLM, Ollama, and any OpenAI-compatible endpoint. + * + * Config priority: CLI flags > env vars > defaults + */ + +export interface LLMConfig { + apiKey: string; + baseUrl: string; + model: string; + maxTokens: number; + temperature: number; +} + +export interface LLMResponse { + content: string; + promptTokens?: number; + completionTokens?: number; +} + +/** + * Resolve LLM configuration from env vars, saved config, and optional overrides. + * Priority: overrides (CLI flags) > env vars > ~/.gitnexus/config.json > error + * + * If no API key is found, returns config with empty apiKey (caller should handle). + */ +export async function resolveLLMConfig(overrides?: Partial): Promise { + const { loadCLIConfig } = await import('../../storage/repo-manager.js'); + const savedConfig = await loadCLIConfig(); + + const apiKey = overrides?.apiKey + || process.env.GITNEXUS_API_KEY + || process.env.OPENAI_API_KEY + || savedConfig.apiKey + || ''; + + return { + apiKey, + baseUrl: overrides?.baseUrl + || process.env.GITNEXUS_LLM_BASE_URL + || savedConfig.baseUrl + || 'https://api.openai.com/v1', + model: overrides?.model + || process.env.GITNEXUS_MODEL + || savedConfig.model + || 'gpt-4o-mini', + maxTokens: overrides?.maxTokens ?? 16_384, + temperature: overrides?.temperature ?? 0, + }; +} + +/** + * Estimate token count from text (rough heuristic: ~4 chars per token). + */ +export function estimateTokens(text: string): number { + return Math.ceil(text.length / 4); +} + +/** + * Call an OpenAI-compatible LLM API. + * Retries once on transient failures (5xx, network errors). + */ +export async function callLLM( + prompt: string, + config: LLMConfig, + systemPrompt?: string, +): Promise { + const messages: Array<{ role: string; content: string }> = []; + if (systemPrompt) { + messages.push({ role: 'system', content: systemPrompt }); + } + messages.push({ role: 'user', content: prompt }); + + const url = `${config.baseUrl.replace(/\/+$/, '')}/chat/completions`; + + const body = { + model: config.model, + messages, + max_tokens: config.maxTokens, + temperature: config.temperature, + }; + + let lastError: Error | null = null; + + for (let attempt = 0; attempt < 2; attempt++) { + try { + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${config.apiKey}`, + }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const errorText = await response.text().catch(() => 'unknown error'); + + // Rate limit โ€” wait and retry + if (response.status === 429 && attempt === 0) { + const retryAfter = parseInt(response.headers.get('retry-after') || '5', 10); + await sleep(retryAfter * 1000); + continue; + } + + // Server error โ€” retry once + if (response.status >= 500 && attempt === 0) { + await sleep(2000); + continue; + } + + throw new Error(`LLM API error (${response.status}): ${errorText.slice(0, 500)}`); + } + + const json = await response.json() as any; + const choice = json.choices?.[0]; + if (!choice?.message?.content) { + throw new Error('LLM returned empty response'); + } + + return { + content: choice.message.content, + promptTokens: json.usage?.prompt_tokens, + completionTokens: json.usage?.completion_tokens, + }; + } catch (err: any) { + lastError = err; + + // Network error โ€” retry once + if (attempt === 0 && (err.code === 'ECONNREFUSED' || err.code === 'ETIMEDOUT' || err.message?.includes('fetch'))) { + await sleep(3000); + continue; + } + + throw err; + } + } + + throw lastError || new Error('LLM call failed after retries'); +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} diff --git a/gitnexus/src/core/wiki/prompts.ts b/gitnexus/src/core/wiki/prompts.ts new file mode 100644 index 0000000000..8e6bb3b633 --- /dev/null +++ b/gitnexus/src/core/wiki/prompts.ts @@ -0,0 +1,207 @@ +/** + * LLM Prompt Templates for Wiki Generation + * + * All prompts produce deterministic, source-grounded documentation. + * Templates use {{PLACEHOLDER}} substitution. + */ + +// โ”€โ”€โ”€ Grouping Prompt โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +export const GROUPING_SYSTEM_PROMPT = `You are a documentation architect. Given a list of source files with their exported symbols, group them into logical documentation modules. + +Rules: +- Each module should represent a cohesive feature, layer, or domain +- Every file must appear in exactly one module +- Module names should be human-readable (e.g. "Authentication", "Database Layer", "API Routes") +- Aim for 5-15 modules for a typical project. Fewer for small projects, more for large ones +- Group by functionality, not by file type or directory structure alone +- Do NOT create modules for tests, configs, or non-source files`; + +export const GROUPING_USER_PROMPT = `Group these source files into documentation modules. + +**Files and their exports:** +{{FILE_LIST}} + +**Directory structure:** +{{DIRECTORY_TREE}} + +Respond with ONLY a JSON object mapping module names to file path arrays. No markdown, no explanation. +Example format: +{ + "Authentication": ["src/auth/login.ts", "src/auth/session.ts"], + "Database": ["src/db/connection.ts", "src/db/models.ts"] +}`; + +// โ”€โ”€โ”€ Leaf Module Prompt โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +export const MODULE_SYSTEM_PROMPT = `You are a technical documentation writer. Write clear, developer-focused documentation for a code module. + +Rules: +- Reference actual function names, class names, and code patterns โ€” do NOT invent APIs +- Use the call graph and execution flow data for accuracy, but do NOT mechanically list every edge +- Include Mermaid diagrams only when they genuinely help understanding. Keep them small (5-10 nodes max) +- Structure the document however makes sense for this module โ€” there is no mandatory format +- Write for a developer who needs to understand and contribute to this code`; + +export const MODULE_USER_PROMPT = `Write documentation for the **{{MODULE_NAME}}** module. + +## Source Code + +{{SOURCE_CODE}} + +## Call Graph & Execution Flows (reference for accuracy) + +Internal calls: {{INTRA_CALLS}} +Outgoing calls: {{OUTGOING_CALLS}} +Incoming calls: {{INCOMING_CALLS}} +Execution flows: {{PROCESSES}} + +--- + +Write comprehensive documentation for this module. Cover its purpose, how it works, its key components, and how it connects to the rest of the codebase. Use whatever structure best fits this module โ€” you decide the sections and headings. Include a Mermaid diagram only if it genuinely clarifies the architecture.`; + +// โ”€โ”€โ”€ Parent Module Prompt โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +export const PARENT_SYSTEM_PROMPT = `You are a technical documentation writer. Write a summary page for a module that contains sub-modules. Synthesize the children's documentation โ€” do not re-read source code. + +Rules: +- Reference actual components from the child modules +- Focus on how the sub-modules work together, not repeating their individual docs +- Keep it concise โ€” the reader can click through to child pages for detail +- Include a Mermaid diagram only if it genuinely clarifies how the sub-modules relate`; + +export const PARENT_USER_PROMPT = `Write documentation for the **{{MODULE_NAME}}** module, which contains these sub-modules: + +{{CHILDREN_DOCS}} + +Cross-module calls: {{CROSS_MODULE_CALLS}} +Shared execution flows: {{CROSS_PROCESSES}} + +--- + +Write a concise overview of this module group. Explain its purpose, how the sub-modules fit together, and the key workflows that span them. Link to sub-module pages (e.g. \`[Sub-module Name](sub-module-slug.md)\`) rather than repeating their content. Use whatever structure fits best.`; + +// โ”€โ”€โ”€ Overview Prompt โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +export const OVERVIEW_SYSTEM_PROMPT = `You are a technical documentation writer. Write the top-level overview page for a repository wiki. This is the first page a new developer sees. + +Rules: +- Be clear and welcoming โ€” this is the entry point to the entire codebase +- Reference actual module names so readers can navigate to their docs +- Include a high-level Mermaid architecture diagram showing only the most important modules and their relationships (max 10 nodes). A new dev should grasp it in 10 seconds +- Do NOT create module index tables or list every module with descriptions โ€” just link to module pages naturally within the text +- Use the inter-module edges and execution flow data for accuracy, but do NOT dump them raw`; + +export const OVERVIEW_USER_PROMPT = `Write the overview page for this repository's wiki. + +## Project Info + +{{PROJECT_INFO}} + +## Module Summaries + +{{MODULE_SUMMARIES}} + +## Reference Data (for accuracy โ€” do not reproduce verbatim) + +Inter-module call edges: {{MODULE_EDGES}} +Key system flows: {{TOP_PROCESSES}} + +--- + +Write a clear overview of this project: what it does, how it's architected, and the key end-to-end flows. Include a simple Mermaid architecture diagram (max 10 nodes, big-picture only). Link to module pages (e.g. \`[Module Name](module-slug.md)\`) naturally in the text rather than listing them in a table. If project config was provided, include brief setup instructions. Structure the page however reads best.`; + +// โ”€โ”€โ”€ Template Substitution Helper โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +/** + * Replace {{PLACEHOLDER}} tokens in a template string. + */ +export function fillTemplate( + template: string, + vars: Record, +): string { + let result = template; + for (const [key, value] of Object.entries(vars)) { + result = result.replaceAll(`{{${key}}}`, value); + } + return result; +} + +// โ”€โ”€โ”€ Formatting Helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +/** + * Format file list with exports for the grouping prompt. + */ +export function formatFileListForGrouping( + files: Array<{ filePath: string; symbols: Array<{ name: string; type: string }> }>, +): string { + return files + .map(f => { + const exports = f.symbols.length > 0 + ? f.symbols.map(s => `${s.name} (${s.type})`).join(', ') + : 'no exports'; + return `- ${f.filePath}: ${exports}`; + }) + .join('\n'); +} + +/** + * Build a directory tree string from file paths. + */ +export function formatDirectoryTree(filePaths: string[]): string { + const dirs = new Set(); + for (const fp of filePaths) { + const parts = fp.replace(/\\/g, '/').split('/'); + for (let i = 1; i < parts.length; i++) { + dirs.add(parts.slice(0, i).join('/')); + } + } + + const sorted = Array.from(dirs).sort(); + if (sorted.length === 0) return '(flat structure)'; + + return sorted.slice(0, 50).join('\n') + (sorted.length > 50 ? `\n... and ${sorted.length - 50} more directories` : ''); +} + +/** + * Format call edges as readable text. + */ +export function formatCallEdges( + edges: Array<{ fromFile: string; fromName: string; toFile: string; toName: string }>, +): string { + if (edges.length === 0) return 'None'; + return edges + .slice(0, 30) + .map(e => `${e.fromName} (${shortPath(e.fromFile)}) โ†’ ${e.toName} (${shortPath(e.toFile)})`) + .join('\n'); +} + +/** + * Format process traces as readable text. + */ +export function formatProcesses( + processes: Array<{ + label: string; + type: string; + steps: Array<{ step: number; name: string; filePath: string }>; + }>, +): string { + if (processes.length === 0) return 'No execution flows detected for this module.'; + + return processes + .map(p => { + const stepsText = p.steps + .map(s => ` ${s.step}. ${s.name} (${shortPath(s.filePath)})`) + .join('\n'); + return `**${p.label}** (${p.type}):\n${stepsText}`; + }) + .join('\n\n'); +} + +/** + * Shorten a file path for readability. + */ +function shortPath(fp: string): string { + const parts = fp.replace(/\\/g, '/').split('/'); + return parts.length > 3 ? parts.slice(-3).join('/') : fp; +} diff --git a/gitnexus/src/mcp/core/embedder.ts b/gitnexus/src/mcp/core/embedder.ts index b254f52bf7..097b13fd73 100644 --- a/gitnexus/src/mcp/core/embedder.ts +++ b/gitnexus/src/mcp/core/embedder.ts @@ -36,8 +36,10 @@ export const initEmbedder = async (): Promise => { console.error('GitNexus: Loading embedding model (first search may take a moment)...'); - // Try WebGPU first (Windows DirectX12), fall back to CPU - const devicesToTry: Array<'webgpu' | 'cpu'> = ['webgpu', 'cpu']; + // Try GPU first (DirectML on Windows, CUDA on Linux), fall back to CPU + const isWindows = process.platform === 'win32'; + const gpuDevice = isWindows ? 'dml' : 'cuda'; + const devicesToTry: Array<'dml' | 'cuda' | 'cpu'> = [gpuDevice, 'cpu']; for (const device of devicesToTry) { try { diff --git a/gitnexus/src/mcp/core/kuzu-adapter.ts b/gitnexus/src/mcp/core/kuzu-adapter.ts index 06a7ba14d1..fc0dfb4b88 100644 --- a/gitnexus/src/mcp/core/kuzu-adapter.ts +++ b/gitnexus/src/mcp/core/kuzu-adapter.ts @@ -71,8 +71,12 @@ function closeOne(repoId: string): void { pool.delete(repoId); } +const LOCK_RETRY_ATTEMPTS = 3; +const LOCK_RETRY_DELAY_MS = 2000; + /** - * Initialize (or reuse) a connection for a specific repo + * Initialize (or reuse) a connection for a specific repo. + * Retries on lock errors (e.g., when `gitnexus analyze` is running). */ export const initKuzu = async (repoId: string, dbPath: string): Promise => { const existing = pool.get(repoId); @@ -90,21 +94,42 @@ export const initKuzu = async (repoId: string, dbPath: string): Promise => evictLRU(); - // Silence stdout during KuzuDB init โ€” native module may write to stdout - // which corrupts the MCP stdio protocol. - const origWrite = process.stdout.write; - process.stdout.write = (() => true) as any; - let db: kuzu.Database; - let conn: kuzu.Connection; - try { - db = new kuzu.Database(dbPath); - conn = new kuzu.Connection(db); - } finally { - process.stdout.write = origWrite; + // Open in read-only mode โ€” MCP server never writes to the database. + // This allows multiple MCP server instances to read concurrently, and + // avoids lock conflicts when `gitnexus analyze` is writing. + let lastError: Error | null = null; + for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) { + // Silence stdout during KuzuDB init โ€” native module may write to stdout + // which corrupts the MCP stdio protocol. + const origWrite = process.stdout.write; + process.stdout.write = (() => true) as any; + try { + const db = new kuzu.Database( + dbPath, + 0, // bufferManagerSize (default) + false, // enableCompression (default) + true, // readOnly + ); + const conn = new kuzu.Connection(db); + process.stdout.write = origWrite; + pool.set(repoId, { db, conn, lastUsed: Date.now(), dbPath }); + ensureIdleTimer(); + return; + } catch (err: any) { + process.stdout.write = origWrite; + lastError = err instanceof Error ? err : new Error(String(err)); + const isLockError = lastError.message.includes('Could not set lock') + || lastError.message.includes('lock'); + if (!isLockError || attempt === LOCK_RETRY_ATTEMPTS) break; + // Wait before retrying โ€” analyze may be mid-rebuild + await new Promise(resolve => setTimeout(resolve, LOCK_RETRY_DELAY_MS * attempt)); + } } - pool.set(repoId, { db, conn, lastUsed: Date.now(), dbPath }); - ensureIdleTimer(); + throw new Error( + `KuzuDB unavailable for ${repoId}. Another process may be rebuilding the index. ` + + `Retry later. (${lastError?.message || 'unknown error'})` + ); }; /** diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index c7ab45a88a..7048fd1764 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -183,8 +183,14 @@ export class LocalBackend { const handle = this.repos.get(repoId); if (!handle) throw new Error(`Unknown repo: ${repoId}`); - await initKuzu(repoId, handle.kuzuPath); - this.initializedRepos.add(repoId); + try { + await initKuzu(repoId, handle.kuzuPath); + this.initializedRepos.add(repoId); + } catch (err: any) { + // If lock error, mark as not initialized so next call retries + this.initializedRepos.delete(repoId); + throw err; + } } // โ”€โ”€โ”€ Public Getters โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ diff --git a/gitnexus/src/mcp/server.ts b/gitnexus/src/mcp/server.ts index ae4820acd8..5775856241 100644 --- a/gitnexus/src/mcp/server.ts +++ b/gitnexus/src/mcp/server.ts @@ -79,7 +79,7 @@ export async function startMCPServer(backend: LocalBackend): Promise { const server = new Server( { name: 'gitnexus', - version: '1.1.0', + version: '1.1.9', }, { capabilities: { diff --git a/gitnexus/src/storage/git.ts b/gitnexus/src/storage/git.ts index dd127f2f98..95deca1109 100644 --- a/gitnexus/src/storage/git.ts +++ b/gitnexus/src/storage/git.ts @@ -19,14 +19,6 @@ export const getCurrentCommit = (repoPath: string): string => { } }; -export const getStatusPorcelain = (repoPath: string): string => { - try { - return execSync('git status --porcelain', { cwd: repoPath }).toString(); - } catch { - return ''; - } -}; - /** * Find the git repository root from any path inside the repo */ @@ -39,53 +31,3 @@ export const getGitRoot = (fromPath: string): string | null => { return null; } }; - -/** - * Get files that were added, modified, copied, or renamed between two commits. - * Returns relative paths (forward-slash normalized). - */ -export const getChangedFiles = (fromCommit: string, toCommit: string, repoPath: string): string[] => { - try { - const output = execSync( - `git diff ${fromCommit}..${toCommit} --name-only --diff-filter=ACMR`, - { cwd: repoPath, encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 } - ).trim(); - return output ? output.split('\n').filter(Boolean).map(f => f.replace(/\\/g, '/')) : []; - } catch { - return []; - } -}; - -/** - * Get files that were deleted between two commits. - * Returns relative paths (forward-slash normalized). - */ -export const getDeletedFiles = (fromCommit: string, toCommit: string, repoPath: string): string[] => { - try { - const output = execSync( - `git diff ${fromCommit}..${toCommit} --name-only --diff-filter=D`, - { cwd: repoPath, encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 } - ).trim(); - return output ? output.split('\n').filter(Boolean).map(f => f.replace(/\\/g, '/')) : []; - } catch { - return []; - } -}; - -/** - * Get files with uncommitted changes (working tree vs HEAD). - * This catches staged + unstaged modifications that aren't in any commit yet. - * Returns relative paths (forward-slash normalized). - */ -export const getUncommittedChanges = (repoPath: string): string[] => { - try { - const output = execSync( - 'git diff HEAD --name-only --diff-filter=ACMR', - { cwd: repoPath, encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 } - ).trim(); - return output ? output.split('\n').filter(Boolean).map(f => f.replace(/\\/g, '/')) : []; - } catch { - return []; - } -}; - diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index 2630a577b1..5ec4006be8 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -262,3 +262,39 @@ export const listRegisteredRepos = async (opts?: { validate?: boolean }): Promis return valid; }; + +// โ”€โ”€โ”€ Global CLI Config (~/.gitnexus/config.json) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +export interface CLIConfig { + apiKey?: string; + model?: string; + baseUrl?: string; +} + +/** + * Get the path to the global CLI config file + */ +export const getGlobalConfigPath = (): string => { + return path.join(getGlobalDir(), 'config.json'); +}; + +/** + * Load CLI config from ~/.gitnexus/config.json + */ +export const loadCLIConfig = async (): Promise => { + try { + const raw = await fs.readFile(getGlobalConfigPath(), 'utf-8'); + return JSON.parse(raw) as CLIConfig; + } catch { + return {}; + } +}; + +/** + * Save CLI config to ~/.gitnexus/config.json + */ +export const saveCLIConfig = async (config: CLIConfig): Promise => { + const dir = getGlobalDir(); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(getGlobalConfigPath(), JSON.stringify(config, null, 2), 'utf-8'); +}; diff --git a/todo_live_check_feature.md b/todo_live_check_feature.md deleted file mode 100644 index fceab3bd91..0000000000 --- a/todo_live_check_feature.md +++ /dev/null @@ -1,171 +0,0 @@ -# Feature Specification: GitNexus "Guardian" (Live Impact Check) - -## 1. Overview -The "Guardian" is an active, background monitoring system that provides real-time feedback to developers as they modify code. It leverages the deterministic Knowledge Graph (KuzuDB) to perform instant "Impact Analysis" and "Architecture Linting" without incurring LLM token costs. - -**Goal:** Provide a "Safety Net" that catches breaking changes, side effects, and architectural violations *before* a commit is made. - -## 2. Core Capabilities - -### A. Live "Blast Radius" Detection -* **Trigger:** - * **Manual:** File Save / Debounced Keystroke. - * **AI-Aware Heuristic:** "Burst Write Cooldown" (See Section 3). -* **Logic:** - 1. Identify modified symbols (Functions, Classes) via incremental Tree-sitter parsing. - 2. Execute Graph Query (Cypher) to find dependents. - * `MATCH (modified)<-[:CALLS*1..5]-(affected) RETURN affected` - 3. Filter "affected" nodes that are outside the current file. -* **User Experience:** - * **Toast/Status Bar:** "โš ๏ธ Modification affects 12 external files." - * **Panel:** List of affected files/functions (e.g., "Breaks `PaymentService.process()`"). - * **"Fix Prompt" Generator:** Button to copy a prompt for the AI agent (e.g., "Check `AuthService.ts` for regressions caused by my changes to `Login.tsx`"). -* **Cost:** **Zero Tokens.** (Pure Graph Traversal). - -### B. Architecture "Linting" -* **Trigger:** File Save / New Import Added. -* **Logic:** - 1. Detect new `IMPORTS` or `CALLS` edges in the graph. - 2. Check against defined "Layer Rules" (e.g., defined in `.gitnexus/rules.yaml`). - * *Rule Example:* `Frontend` cannot import `Database`. - 3. Execute Graph Query: - * `MATCH (source)-[:IMPORTS]->(target) WHERE source.layer = 'Frontend' AND target.layer = 'Database' RETURN source, target` -* **User Experience:** - * **Inline Warning:** "โŒ Architectural Violation: UI component cannot directly access Database types." -* **Cost:** **Zero Tokens.** (Rule-based Graph Matching). - -### C. "Smart" Explanation (On Demand) -* **Trigger:** User clicks "Explain Risk" on a warning. -* **Logic:** - 1. Gather context: Source code of the change + Signatures of affected functions. - 2. Send structured prompt to LLM (Small model: GPT-4o-mini / Local Llama). - 3. *Prompt:* "The user modified `calculateTotal()`. This function is called by `InvoiceGenerator`. Explain potential risks." -* **User Experience:** - * Natural language summary: "Changing the return type of `calculateTotal` will cause a compilation error in `InvoiceGenerator` which expects a number." -* **Cost:** **Low.** (Only on user request, highly targeted context). - -## 3. Technical Architecture - -### The "Watcher" Loop -1. **File Watcher:** Listens for `change` events in the workspace. -2. **AI Detection Heuristic (Burst Mode):** - * *Logic:* Detect rapid file writes (3+ files in <2s) typical of AI Agents. - * *Action:* Suppress immediate checks (don't spam). Wait for "Cooldown" (e.g., 5s silence). - * *Trigger:* After cooldown, run "Full Impact Scan" and present summary. -3. **Incremental Parser:** Updates the KuzuDB graph for the specific file (milliseconds). - * *Note:* Uses Tree-sitter's incremental parsing to be extremely fast. -3. **Graph Engine:** Runs pre-compiled Cypher queries against the updated graph. -4. **Client Interface (MCP/Extension):** - * Push notifications to VS Code / Cursor. - * Update "Impact" panel in the browser UI. - -## 4. Agent Integration (Mandatory Workflow) - -To ensure AI agents (Cursor, Claude, etc.) utilize GitNexus, we leverage "Prompt Engineering via Context" and Tool Definitions. - -### A. The `.cursorrules` Protocol -* **Mechanism:** A `.cursorrules` file in the project root enforces agent behavior. -* **Content:** - ```markdown - # GitNexus Safety Protocol - You have access to `@gitnexus` tools. - - **MANDATORY WORKFLOW:** - 1. **PLANNING:** Before writing code, use `gitnexus_get_dependencies` to map the blast radius. - 2. **VERIFICATION:** After EVERY code modification, you MUST run `gitnexus_detect_impact`. - 3. **REPORTING:** If risks are found, fix them or report to user. - ``` - -### B. "Bossy" Tool Descriptions -* **Mechanism:** Update MCP Tool definitions to be imperative. -* **Description:** "CRITICAL SAFETY TOOL. MUST be used after modifying code. Scans graph for broken dependencies. Returns affected files." - -### C. "Fix Prompt" Generator -* **Mechanism:** If the Agent ignores the tools and the "Guardian" detects a break, the popup offers a "Copy Fix Prompt" button. -* **Prompt:** "Your changes to X broke Y. Use `gitnexus_detect_impact` to verify and fix." - -## 5. Distributed Knowledge Graph (Git-Native Architecture) - -To enable "B2B / Team" features without a central server, we use Git itself as the synchronization mechanism for the Knowledge Graph. This is the **"Git-Native Knowledge Graph"** architecture. - -### A. The Core Concept: "Graph Manifest" -You cannot commit the raw KuzuDB database files (binaries) to Git. Instead, we use a lightweight, diff-friendly **Graph Manifest**. - -* **File Path:** `.gitnexus/graph-state.jsonl.gz` -* **Content:** A compressed JSON Lines dump of the Nodes and Edges (semantic data only). -* **Purpose:** Acts as the "Transport Layer" for the graph between machines. - -### B. The Workflow - -#### Step 1: The "Write" Op (Local Dev) -1. **Code Change:** Developer modifies `User.ts`. -2. **Local Indexing:** GitNexus CLI updates local KuzuDB instantly (Incremental Update). -3. **Pre-Commit Hook:** - * Trigger: `git commit` - * Action: GitNexus dumps the current KuzuDB state to `.gitnexus/graph-state.jsonl.gz`. - * Optimization: Only dumps semantic data (e.g., "Func A calls Func B"), not the full AST, keeping it small. - -#### Step 2: The "Transport" (Git Sync) -* `git push` uploads the Code + Graph Manifest. -* **Crucial:** The graph version is now cryptographically tied to the commit hash. No "drift" between code and graph. - -#### Step 3: The "Read" Op (Teammate Pull) -1. **Git Pull:** Teammate receives new code + new manifest. -2. **Post-Merge Hook / Hydration:** - * Trigger: Git detects change in `.gitnexus`. - * Action: GitNexus CLI reads the manifest and **bulk-inserts** it into the local KuzuDB. - * **Result:** Teammate has a fully queried graph in seconds (vs. minutes of re-parsing). - -### C. Conflict Resolution: "Discard and Rebuild" -What happens if two devs change the graph simultaneously? - -* **Scenario:** Merge conflict in `.gitnexus/graph-state.jsonl.gz`. -* **Strategy:** - 1. GitNexus detects the conflict in the manifest file. - 2. It **discards** the conflicted manifest. - 3. It runs the **Parser** locally on the *merged* source code (Source of Truth). - 4. It generates a **fresh, correct** manifest. -* **Philosophy:** The Graph is a *derivative* of the Code. We never manually merge the graph; we regenerate it from the source. - -### D. Architecture Diagram (Mermaid) - -```mermaid -graph TD - subgraph "Developer A (Write)" - CodeA[User.ts] -->|Parser| DB_A[(Local KuzuDB)] - DB_A -->|Pre-Commit Export| ManifestA[.gitnexus/graph-state.gnx] - ManifestA -->|git push| GitHub - end - - subgraph "GitHub / Git Server" - GitHub -->|git pull| DevB_Repo - end - - subgraph "Developer B (Read)" - DevB_Repo[Code + Manifest] -->|Hydration Hook| DB_B[(Local KuzuDB)] - ManifestA -->|Bulk Insert| DB_B - DB_B -->|Instant Query| Cursor_B[Cursor / IDE] - end - - subgraph "Enterprise Hub (Monetization)" - GitHub -->|Webhook| HubServer[Node.js Hub] - HubServer -->|Download Manifest| CentralDB[(Neo4j / Postgres)] - CentralDB -->|Analytics API| Dashboard[CTO Dashboard] - end -``` - -### E. B2B / Enterprise "Hub" Integration -The "Hub" is a lightweight server that monetizes this architecture. -1. **Action:** Subscribes to the repo's webhooks. -2. **Ingestion:** Downloads *only* the manifest file (not the source code). -3. **Storage:** Loads it into a centralized DB (Neo4j/Postgres) for organization-wide queries. -4. **Security:** "We don't see your code, only your graph structure." - - -## 7. Implementation Roadmap -1. **Phase 1:** Implement `FileWatcher` in CLI + Incremental Graph Update. -2. **Phase 2:** Create `ImpactQuery` engine (Cypher queries for dependents). -3. **Phase 3:** Build MCP Tool `get_live_impact` for Editor integration and "Bossy" descriptions. -4. **Phase 4:** Implement Git-Native Graph Sync (Manifest generation + Pre-commit hook). -5. **Phase 5:** Add Architecture Rule definition schema (`.gitnexus/rules.yaml`). -