From 62f86ea0685b4f20c2dbbbb27f15a05fdbcb2059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sun, 18 Oct 2026 19:32:09 +0300 Subject: [PATCH] feat(mcp): reduce response size and query overhead (#3621) * docs(mcp): document focused response filters * feat(mcp): add opt-in response section and relation filters * test(mcp): verify response filters and compatibility * test(mcp): add deterministic response-size benchmark * docs(skills): teach focused MCP calls and safe filter use * docs(skills): update debugging response-filter guidance * docs(skills): update exploration response-filter guidance * docs(skills): explain focused MCP response options * docs(plugin): mirror focused debugging guidance * docs(plugin): mirror focused exploration guidance * docs(plugin): mirror focused MCP tool guide * docs(cursor): mirror focused debugging guidance * docs(cursor): mirror focused exploration guidance * test(mcp): give filter cases unique identities Use named cases to keep array inputs intact and let the CI execution audit distinguish every malformed-filter test. * perf(mcp): batch BM25 symbol hydration * test(mcp): serialize native hydration coverage * test(eval): publish child PID atomically * test(mcp): cover hydration on native platforms --------- Co-authored-by: Gergo Magyar Co-authored-by: Abhigyan Patwari <126312502+abhigyanpatwari@users.noreply.github.com> --- .claude/skills/gitnexus-debugging/SKILL.md | 33 ++- .claude/skills/gitnexus-exploring/SKILL.md | 95 +++++--- .claude/skills/gitnexus-guide/SKILL.md | 35 ++- README.md | 16 ++ eval/tests/test_process_control.py | 5 +- .../skills/gitnexus-debugging/SKILL.md | 33 ++- .../skills/gitnexus-exploring/SKILL.md | 95 +++++--- .../skills/gitnexus-guide/SKILL.md | 35 ++- .../skills/gitnexus-debugging/SKILL.md | 33 ++- .../skills/gitnexus-exploring/SKILL.md | 95 +++++--- .../scripts/bench/mcp-response-filters.ts | 121 +++++++++++ gitnexus/scripts/cross-platform-tests.ts | 1 + gitnexus/skills/gitnexus-debugging.md | 33 ++- gitnexus/skills/gitnexus-exploring.md | 95 +++++--- gitnexus/skills/gitnexus-guide.md | 35 ++- gitnexus/src/mcp/local/local-backend.ts | 79 +++++-- gitnexus/src/mcp/response-filter.ts | 143 ++++++++++++ gitnexus/src/mcp/server.ts | 15 +- gitnexus/src/mcp/tools.ts | 33 ++- .../integration/bm25-symbol-hydration.test.ts | 65 ++++++ .../unit/local-backend-bm25-hydration.test.ts | 177 +++++++++++++++ .../test/unit/mcp-response-filter.test.ts | 205 ++++++++++++++++++ gitnexus/test/unit/server.test.ts | 94 ++++++++ gitnexus/vitest.config.ts | 2 + 24 files changed, 1415 insertions(+), 158 deletions(-) create mode 100644 gitnexus/scripts/bench/mcp-response-filters.ts create mode 100644 gitnexus/src/mcp/response-filter.ts create mode 100644 gitnexus/test/integration/bm25-symbol-hydration.test.ts create mode 100644 gitnexus/test/unit/local-backend-bm25-hydration.test.ts create mode 100644 gitnexus/test/unit/mcp-response-filter.test.ts diff --git a/.claude/skills/gitnexus-debugging/SKILL.md b/.claude/skills/gitnexus-debugging/SKILL.md index 6051c33c4..2bca79663 100644 --- a/.claude/skills/gitnexus-debugging/SKILL.md +++ b/.claude/skills/gitnexus-debugging/SKILL.md @@ -17,7 +17,8 @@ description: "Use when the user is debugging a bug, tracing an error, or asking A root cause traced in the wrong repository is a wrong root cause. -Call `list_repos {}` before the first tool call. With one indexed repository, +Bind once using a confirmed registry name/path or configured MCP default. Call +`list_repos {}` if the target is not yet known; reuse that binding across calls. With one indexed repository, use the examples below as written. With more than one, pass `repo` on every call: an omitted `repo` normally errors, but under an MCP policy with a configured default it resolves to that default silently. If you cannot tell @@ -34,10 +35,10 @@ diagnosis. ## Workflow ``` -0. list_repos {} → Bind repo -1. query({search_query: ""}) → Find related execution flows +0. Bind the repo once (list_repos only if not already known) +1. If the suspect is unknown, query for the error/symptom with small result limits 2. context({name: ""}) → See callers/callees/processes -3. READ gitnexus://repo/{name}/process/{name} → Trace execution flow +3. Read a process resource only if its trace resolves an open question 4. cypher({statement: "MATCH path..."}) → Custom traces if needed ``` @@ -47,7 +48,7 @@ diagnosis. ## Checklist ``` -- [ ] list_repos {} — bind repo; explicit repo when >1 indexed, ask if ambiguous +- [ ] Once per session: bind repo; explicit repo when >1 indexed, ask if ambiguous - [ ] Understand the symptom (error message, unexpected behavior) - [ ] query for error text or related code - [ ] Identify the suspect function from returned processes @@ -58,6 +59,28 @@ diagnosis. - [ ] State the repository and index freshness with the diagnosis ``` +## Keep investigations focused + +Start with a known symbol's `context`, or a small `query` (`limit: 3`, +`max_symbols: 3`, `include_content: false`, `chain_depth: 0`) to discover one. +When the server advertises these single-repository filters, request only the +needed evidence: + +```text +context({uid: "", repo: "my-app", include_content: true, + response_sections: ["outgoing"], relationTypes: ["CALLS"]}) +trace({from_uid: "", to_uid: "", repo: "my-app"}) +``` + +`symbol` and completeness/staleness diagnostics remain in a filtered context. +`relationTypes` filters the returned incoming/outgoing categories only; it does +not expand their existing bounded window or filter an explicitly requested chain. +For callback registration, include `USES`; for imports or inheritance, request +those relation types or omit the filter. Omitted references are not absent +references. Broaden when the hypothesis needs it, and preserve UNKNOWN/partial +warnings. Confirm source and tests before claiming a root cause. A useful 2–3 +call investigation is a goal, not a reason to skip verification or pre-edit impact. + ## Debugging Patterns | Symptom | GitNexus Approach | diff --git a/.claude/skills/gitnexus-exploring/SKILL.md b/.claude/skills/gitnexus-exploring/SKILL.md index 89f9a3884..8a6ee3ba3 100644 --- a/.claude/skills/gitnexus-exploring/SKILL.md +++ b/.claude/skills/gitnexus-exploring/SKILL.md @@ -15,9 +15,10 @@ description: "Use when the user asks how code works, wants to understand archite ## Bind the repository first -Step 1 discovers what is indexed; every call after it must say which of those -it means. With one indexed repository, use the examples below as written. With -more than one, pass `repo` on every call: an omitted `repo` normally errors, +Bind once per session using a confirmed registry name/path or configured MCP +default; reuse it. Call `list_repos {}` when the target is not yet known, not before +every question. Never guess the registry name from the checkout directory. With +one indexed repository, use the examples below as written. With more than one, pass `repo` on every call: an omitted `repo` normally errors, but under an MCP policy with a configured default it resolves to that default silently. If you cannot tell which repository is meant, stop and ask. Report the bound repository and index freshness alongside your explanation. @@ -28,26 +29,64 @@ the bound repository and index freshness alongside your explanation. ## Workflow ``` -1. list_repos {} or READ gitnexus://repos → Discover indexed repos -2. READ gitnexus://repo/{name}/context → Codebase overview, check staleness -3. query({search_query: ""}) → Find related execution flows -4. context({name: ""}) → Deep dive on specific symbol -5. READ gitnexus://repo/{name}/process/{name} → Trace full execution flow +1. Bind the repository once; use hot-tool staleness on subsequent calls +2. If no symbol is known: query({search_query: "", limit: 3, max_symbols: 3}) +3. context({uid: ""}) → Inspect the selected symbol +4. Read a process resource only when its full trace answers an unresolved question ``` -> If step 2 says "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. +> If the index is stale → run `node .gitnexus/run.cjs analyze` in terminal. > Hot-tool `staleness` names which index answered (`branch`/`lastCommit`) and how fresh it is (`status`). Re-analyze only for `behind` or `diverged` — `current` is identity, `unknown` is unmeasurable. +## Ask for only the evidence needed + +Use options advertised by the connected server. `response_sections` and context +`relationTypes` are opt-in views for a single repository, not `@group` routing. +On an older server, use the existing limits and content/chain options instead. +Use filters when unwanted sections are bulky; their omission metadata can make +an already-small response slightly larger. +Aim for 2–3 focused calls after repository binding; stop when the question is +answered, but expand if evidence or a required safety check is still missing. + +```text +query({search_query: "payment retry", repo: "my-app", limit: 3, max_symbols: 3, + include_content: false, chain_depth: 0, + response_sections: ["process_symbols", "definitions"]}) +context({uid: "", repo: "my-app", include_content: true, + response_sections: ["incoming", "outgoing"], relationTypes: ["CALLS"]}) +// Before editing, assess impact using the normal traversal and risk calculation: +impact({target: "", repo: "my-app", direction: "upstream", summaryOnly: true}) +``` + +If the symbol is already known, start with `context`; use `uid`, or `name` plus +`file_path`/`kind`, to avoid ambiguous results. To answer only “who calls it?”, +request `response_sections: ["incoming"]`, `relationTypes: ["CALLS"]` and leave +`include_content` false. Keep `processes` in a query when the question is about +flows, rather than requesting a symbols-only view. + +Do not request source and multi-hop chains for every search hit. Fetch source +for the selected symbol only; for a large class/file, prefer `read_file` with +a small 0-based `startLine`/`endLine` window (subtract 1 from graph line numbers), +or `maxLines` for a whole-file read, if that tool is available. Use +`trace` for a known A→B path instead of repeatedly expanding `context`. + +Filters retain requested sections/categories and hide the others; they do not prove other references +are absent or make the existing bounded search exhaustive. Read `response_filter`, +`warning`, `partial`, `truncated`, `epistemic` and `staleness`. Remove filters when +broader evidence is needed. `maxTokens` is a last-resort byte cutoff that can cut +JSON: narrow the request rather than treating a cut response as complete. Never +use a filtered caller view to waive the required impact check or an UNKNOWN risk. + ## Checklist ``` -- [ ] list_repos {} — bind repo; explicit repo when >1 indexed, ask if ambiguous -- [ ] READ gitnexus://repo/{name}/context -- [ ] query for the concept you want to understand -- [ ] Review returned processes (execution flows) -- [ ] context on key symbols for callers/callees -- [ ] READ process resource for full execution traces -- [ ] Read source files for implementation details +- [ ] Once per session: bind repo; explicit repo when >1 indexed, ask if ambiguous +- [ ] Check hot-tool staleness; read the overview only if needed +- [ ] Query for the concept only if the relevant symbol is not already known +- [ ] Review returned symbols; include processes when execution flow matters +- [ ] context on selected symbols; request only relevant sections/relations +- [ ] Read a process resource only if its full trace is needed +- [ ] Read targeted source only when implementation details are needed - [ ] State the repository and index freshness with the explanation ``` @@ -84,18 +123,16 @@ with a single one. ## Example: "How does payment processing work?" -``` -1. list_repos {} → total: 1 (my-app) — bind it - READ gitnexus://repo/my-app/context → 918 symbols, 45 processes -2. query({search_query: "payment processing"}) - → CheckoutFlow: processPayment → validateCard → chargeStripe - → RefundFlow: initiateRefund → calculateRefund → processRefund -3. context({name: "processPayment"}) - → Incoming: checkoutHandler, webhookHandler - → Outgoing: validateCard, chargeStripe, saveTransaction -4. Read src/payments/processor.ts for implementation details -5. Answer, noting: Repository my-app, index current +With `my-app` already bound and the index current: + +```text +1. query({search_query: "payment processing", repo: "my-app", limit: 3, max_symbols: 3}) + → CheckoutFlow and RefundFlow, with symbol IDs and locations +2. context({uid: "", repo: "my-app", include_content: true, + response_sections: ["outgoing", "processes"]}) + → Selected source, callees, process participation and completeness diagnostics +3. Answer from that evidence; read one process resource only if its trace is still needed ``` -Had step 1 returned two repositories, every call above would carry -`repo: "my-app"`. +Report the repository and index freshness with the explanation. If `my-app` +had not yet been bound, discover it once with `list_repos` first. diff --git a/.claude/skills/gitnexus-guide/SKILL.md b/.claude/skills/gitnexus-guide/SKILL.md index 5f595e21a..e1adb6e91 100644 --- a/.claude/skills/gitnexus-guide/SKILL.md +++ b/.claude/skills/gitnexus-guide/SKILL.md @@ -11,7 +11,7 @@ Quick reference for all GitNexus MCP tools, resources, and the knowledge graph s For any task involving code understanding, debugging, impact analysis, or refactoring: -1. **Read `gitnexus://repo/{name}/context`** — codebase overview + check index freshness +1. **Bind the repository once** — use a confirmed registry name/path or configured default; call `list_repos` if unknown. Read hot-tool `staleness`; use `gitnexus://repo/{name}/context` when an overview is needed. 2. **Match your task to a skill below** and **read that skill file** 3. **Follow the skill's workflow and checklist** @@ -51,6 +51,39 @@ For any task involving code understanding, debugging, impact analysis, or refact | `group_sync` | Rebuild a group's Contract Registry (cross-repo HTTP contract links); run after `group.yaml` changes or member re-index | | `list_repos` | Discover indexed repos (paginated — `limit`/`offset`) | +### Choose a focused response + +Use the connected server's schema; older servers may not advertise the new +filters. All options below are explicit: omitting them preserves existing output. +Use filters for bulky unwanted sections; on an already-small response, omission +metadata can outweigh the bytes saved. + +| Question | Options | +| --- | --- | +| Which symbols implement this? | `query`: `limit: 3`, `max_symbols: 3`, `include_content: false`, `chain_depth: 0`, `response_sections: ["process_symbols", "definitions"]` | +| Which execution flows match? | `query`: keep `processes` in `response_sections`; join its `id` to `process_symbols[].process_id` | +| Who calls this exact symbol? | `context`: `uid`, `response_sections: ["incoming"]`, `relationTypes: ["CALLS"]` | +| What does it call, with source? | `context`: `uid`, `include_content: true`, `response_sections: ["outgoing"]`, `relationTypes: ["CALLS"]` | +| What is the impact summary? | `impact`: `direction: "upstream"`, `summaryOnly: true`; risk calculation and totals are retained | +| Which affected symbols need inspection? | `impact`: `limit: 10`, `offset: 0`; page as indicated by the response | +| How does A reach B? | `trace`: `from_uid`, `to_uid` | + +`response_sections` selects data sections for single-repository MCP `query` or +`context` (not `@group` routes). Query choices: `processes`, `process_symbols`, +`definitions`. Context choices: `incoming`, `outgoing`, `processes`, `chain`, +`routes`, `typed_properties`; the selected symbol always remains. Context +`relationTypes` accepts uppercase relationship names advertised by its schema +and filters only the existing incoming/outgoing categories. It does not change +ranking, traversal, source retention or the bounded candidate/reference windows. + +Filters add `response_filter` with the selection and omitted fields/categories. +Warnings, ambiguity, staleness and completeness metadata always remain. An empty +filtered view is not proof of absence. Omit filters to expand, and never waive +impact checks because a selected category is empty. `maxTokens` is a byte cutoff, +not pagination: it can cut JSON. Prefer these filters, small result counts, and +source/chain expansion for a selected symbol rather than a large search result. +Stop once enough evidence answers the question; do not run every tool by default. + ### Paginating `list_repos` `list_repos` is paginated so a large registry is not truncated by MCP/LLM token limits. It takes optional `limit` (default **50**, max **200**) and `offset`, and returns: diff --git a/README.md b/README.md index cd0d7a6cd..4b99ae36e 100644 --- a/README.md +++ b/README.md @@ -390,6 +390,22 @@ The default repository must resolve to an allowed repository. Invalid, ambiguous
MCP response budgets +Prefer explicit response filters before a byte cutoff. Single-repository MCP +`query` and `context` accept `response_sections`: choose query sections +(`processes`, `process_symbols`, `definitions`) or context sections (`incoming`, +`outgoing`, `processes`, `chain`, `routes`, `typed_properties`). Context always +keeps the selected symbol and also accepts `relationTypes: ["CALLS"]` to show +only that category in the existing incoming/outgoing reference window. These +options do not change graph search, ranking or traversal. They are not supported +for `@group` routing; use a member's indexed repository name/path instead. + +For example, `context({uid: "", response_sections: ["incoming"], +relationTypes: ["CALLS"]})` returns the symbol, its returned callers, and all +completeness/staleness diagnostics. The `response_filter` field identifies omitted +data; omission does not mean absence. Remove filters for a broader view. For +impact, existing `summaryOnly`, `limit` and `offset` already provide focused views. +Omitting these filters preserves the existing response. + The `query`, `context`, and `impact` tools accept an optional positive-integer `maxTokens` argument. It bounds the complete formatted MCP response, including hints and error text, using a deterministic four-UTF-8-bytes-per-token estimate. When truncation is required, the response ends with `…` and remains valid UTF-8. Set `GITNEXUS_MCP_DEFAULT_MAX_TOKENS` to apply the same guardrail when callers do not send `maxTokens`. An explicit tool argument takes precedence. Leaving both unset preserves the existing response byte-for-byte; this is a transport guardrail, not semantic pagination or an exact model-specific tokenizer limit. diff --git a/eval/tests/test_process_control.py b/eval/tests/test_process_control.py index 1184feea0..0bce8414c 100644 --- a/eval/tests/test_process_control.py +++ b/eval/tests/test_process_control.py @@ -63,8 +63,11 @@ def test_signal_cancels_active_wave_before_assets_are_released(tmp_path, signum) ready = tmp_path / "child-pid" assets = tmp_path / "assets" assets.write_text("shared") + # Publish readiness only after the complete PID is visible to the parent. command = ( - f"import os,time; from pathlib import Path; Path({str(ready)!r}).write_text(str(os.getpid())); time.sleep(60)" + f"import os,time; from pathlib import Path; ready=Path({str(ready)!r}); " + "staged=ready.with_suffix('.tmp'); staged.write_text(str(os.getpid())); " + "staged.replace(ready); time.sleep(60)" ) script = f""" import json,sys diff --git a/gitnexus-claude-plugin/skills/gitnexus-debugging/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-debugging/SKILL.md index 6051c33c4..2bca79663 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-debugging/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-debugging/SKILL.md @@ -17,7 +17,8 @@ description: "Use when the user is debugging a bug, tracing an error, or asking A root cause traced in the wrong repository is a wrong root cause. -Call `list_repos {}` before the first tool call. With one indexed repository, +Bind once using a confirmed registry name/path or configured MCP default. Call +`list_repos {}` if the target is not yet known; reuse that binding across calls. With one indexed repository, use the examples below as written. With more than one, pass `repo` on every call: an omitted `repo` normally errors, but under an MCP policy with a configured default it resolves to that default silently. If you cannot tell @@ -34,10 +35,10 @@ diagnosis. ## Workflow ``` -0. list_repos {} → Bind repo -1. query({search_query: ""}) → Find related execution flows +0. Bind the repo once (list_repos only if not already known) +1. If the suspect is unknown, query for the error/symptom with small result limits 2. context({name: ""}) → See callers/callees/processes -3. READ gitnexus://repo/{name}/process/{name} → Trace execution flow +3. Read a process resource only if its trace resolves an open question 4. cypher({statement: "MATCH path..."}) → Custom traces if needed ``` @@ -47,7 +48,7 @@ diagnosis. ## Checklist ``` -- [ ] list_repos {} — bind repo; explicit repo when >1 indexed, ask if ambiguous +- [ ] Once per session: bind repo; explicit repo when >1 indexed, ask if ambiguous - [ ] Understand the symptom (error message, unexpected behavior) - [ ] query for error text or related code - [ ] Identify the suspect function from returned processes @@ -58,6 +59,28 @@ diagnosis. - [ ] State the repository and index freshness with the diagnosis ``` +## Keep investigations focused + +Start with a known symbol's `context`, or a small `query` (`limit: 3`, +`max_symbols: 3`, `include_content: false`, `chain_depth: 0`) to discover one. +When the server advertises these single-repository filters, request only the +needed evidence: + +```text +context({uid: "", repo: "my-app", include_content: true, + response_sections: ["outgoing"], relationTypes: ["CALLS"]}) +trace({from_uid: "", to_uid: "", repo: "my-app"}) +``` + +`symbol` and completeness/staleness diagnostics remain in a filtered context. +`relationTypes` filters the returned incoming/outgoing categories only; it does +not expand their existing bounded window or filter an explicitly requested chain. +For callback registration, include `USES`; for imports or inheritance, request +those relation types or omit the filter. Omitted references are not absent +references. Broaden when the hypothesis needs it, and preserve UNKNOWN/partial +warnings. Confirm source and tests before claiming a root cause. A useful 2–3 +call investigation is a goal, not a reason to skip verification or pre-edit impact. + ## Debugging Patterns | Symptom | GitNexus Approach | diff --git a/gitnexus-claude-plugin/skills/gitnexus-exploring/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-exploring/SKILL.md index 89f9a3884..8a6ee3ba3 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-exploring/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-exploring/SKILL.md @@ -15,9 +15,10 @@ description: "Use when the user asks how code works, wants to understand archite ## Bind the repository first -Step 1 discovers what is indexed; every call after it must say which of those -it means. With one indexed repository, use the examples below as written. With -more than one, pass `repo` on every call: an omitted `repo` normally errors, +Bind once per session using a confirmed registry name/path or configured MCP +default; reuse it. Call `list_repos {}` when the target is not yet known, not before +every question. Never guess the registry name from the checkout directory. With +one indexed repository, use the examples below as written. With more than one, pass `repo` on every call: an omitted `repo` normally errors, but under an MCP policy with a configured default it resolves to that default silently. If you cannot tell which repository is meant, stop and ask. Report the bound repository and index freshness alongside your explanation. @@ -28,26 +29,64 @@ the bound repository and index freshness alongside your explanation. ## Workflow ``` -1. list_repos {} or READ gitnexus://repos → Discover indexed repos -2. READ gitnexus://repo/{name}/context → Codebase overview, check staleness -3. query({search_query: ""}) → Find related execution flows -4. context({name: ""}) → Deep dive on specific symbol -5. READ gitnexus://repo/{name}/process/{name} → Trace full execution flow +1. Bind the repository once; use hot-tool staleness on subsequent calls +2. If no symbol is known: query({search_query: "", limit: 3, max_symbols: 3}) +3. context({uid: ""}) → Inspect the selected symbol +4. Read a process resource only when its full trace answers an unresolved question ``` -> If step 2 says "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. +> If the index is stale → run `node .gitnexus/run.cjs analyze` in terminal. > Hot-tool `staleness` names which index answered (`branch`/`lastCommit`) and how fresh it is (`status`). Re-analyze only for `behind` or `diverged` — `current` is identity, `unknown` is unmeasurable. +## Ask for only the evidence needed + +Use options advertised by the connected server. `response_sections` and context +`relationTypes` are opt-in views for a single repository, not `@group` routing. +On an older server, use the existing limits and content/chain options instead. +Use filters when unwanted sections are bulky; their omission metadata can make +an already-small response slightly larger. +Aim for 2–3 focused calls after repository binding; stop when the question is +answered, but expand if evidence or a required safety check is still missing. + +```text +query({search_query: "payment retry", repo: "my-app", limit: 3, max_symbols: 3, + include_content: false, chain_depth: 0, + response_sections: ["process_symbols", "definitions"]}) +context({uid: "", repo: "my-app", include_content: true, + response_sections: ["incoming", "outgoing"], relationTypes: ["CALLS"]}) +// Before editing, assess impact using the normal traversal and risk calculation: +impact({target: "", repo: "my-app", direction: "upstream", summaryOnly: true}) +``` + +If the symbol is already known, start with `context`; use `uid`, or `name` plus +`file_path`/`kind`, to avoid ambiguous results. To answer only “who calls it?”, +request `response_sections: ["incoming"]`, `relationTypes: ["CALLS"]` and leave +`include_content` false. Keep `processes` in a query when the question is about +flows, rather than requesting a symbols-only view. + +Do not request source and multi-hop chains for every search hit. Fetch source +for the selected symbol only; for a large class/file, prefer `read_file` with +a small 0-based `startLine`/`endLine` window (subtract 1 from graph line numbers), +or `maxLines` for a whole-file read, if that tool is available. Use +`trace` for a known A→B path instead of repeatedly expanding `context`. + +Filters retain requested sections/categories and hide the others; they do not prove other references +are absent or make the existing bounded search exhaustive. Read `response_filter`, +`warning`, `partial`, `truncated`, `epistemic` and `staleness`. Remove filters when +broader evidence is needed. `maxTokens` is a last-resort byte cutoff that can cut +JSON: narrow the request rather than treating a cut response as complete. Never +use a filtered caller view to waive the required impact check or an UNKNOWN risk. + ## Checklist ``` -- [ ] list_repos {} — bind repo; explicit repo when >1 indexed, ask if ambiguous -- [ ] READ gitnexus://repo/{name}/context -- [ ] query for the concept you want to understand -- [ ] Review returned processes (execution flows) -- [ ] context on key symbols for callers/callees -- [ ] READ process resource for full execution traces -- [ ] Read source files for implementation details +- [ ] Once per session: bind repo; explicit repo when >1 indexed, ask if ambiguous +- [ ] Check hot-tool staleness; read the overview only if needed +- [ ] Query for the concept only if the relevant symbol is not already known +- [ ] Review returned symbols; include processes when execution flow matters +- [ ] context on selected symbols; request only relevant sections/relations +- [ ] Read a process resource only if its full trace is needed +- [ ] Read targeted source only when implementation details are needed - [ ] State the repository and index freshness with the explanation ``` @@ -84,18 +123,16 @@ with a single one. ## Example: "How does payment processing work?" -``` -1. list_repos {} → total: 1 (my-app) — bind it - READ gitnexus://repo/my-app/context → 918 symbols, 45 processes -2. query({search_query: "payment processing"}) - → CheckoutFlow: processPayment → validateCard → chargeStripe - → RefundFlow: initiateRefund → calculateRefund → processRefund -3. context({name: "processPayment"}) - → Incoming: checkoutHandler, webhookHandler - → Outgoing: validateCard, chargeStripe, saveTransaction -4. Read src/payments/processor.ts for implementation details -5. Answer, noting: Repository my-app, index current +With `my-app` already bound and the index current: + +```text +1. query({search_query: "payment processing", repo: "my-app", limit: 3, max_symbols: 3}) + → CheckoutFlow and RefundFlow, with symbol IDs and locations +2. context({uid: "", repo: "my-app", include_content: true, + response_sections: ["outgoing", "processes"]}) + → Selected source, callees, process participation and completeness diagnostics +3. Answer from that evidence; read one process resource only if its trace is still needed ``` -Had step 1 returned two repositories, every call above would carry -`repo: "my-app"`. +Report the repository and index freshness with the explanation. If `my-app` +had not yet been bound, discover it once with `list_repos` first. diff --git a/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md index 5f595e21a..e1adb6e91 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md @@ -11,7 +11,7 @@ Quick reference for all GitNexus MCP tools, resources, and the knowledge graph s For any task involving code understanding, debugging, impact analysis, or refactoring: -1. **Read `gitnexus://repo/{name}/context`** — codebase overview + check index freshness +1. **Bind the repository once** — use a confirmed registry name/path or configured default; call `list_repos` if unknown. Read hot-tool `staleness`; use `gitnexus://repo/{name}/context` when an overview is needed. 2. **Match your task to a skill below** and **read that skill file** 3. **Follow the skill's workflow and checklist** @@ -51,6 +51,39 @@ For any task involving code understanding, debugging, impact analysis, or refact | `group_sync` | Rebuild a group's Contract Registry (cross-repo HTTP contract links); run after `group.yaml` changes or member re-index | | `list_repos` | Discover indexed repos (paginated — `limit`/`offset`) | +### Choose a focused response + +Use the connected server's schema; older servers may not advertise the new +filters. All options below are explicit: omitting them preserves existing output. +Use filters for bulky unwanted sections; on an already-small response, omission +metadata can outweigh the bytes saved. + +| Question | Options | +| --- | --- | +| Which symbols implement this? | `query`: `limit: 3`, `max_symbols: 3`, `include_content: false`, `chain_depth: 0`, `response_sections: ["process_symbols", "definitions"]` | +| Which execution flows match? | `query`: keep `processes` in `response_sections`; join its `id` to `process_symbols[].process_id` | +| Who calls this exact symbol? | `context`: `uid`, `response_sections: ["incoming"]`, `relationTypes: ["CALLS"]` | +| What does it call, with source? | `context`: `uid`, `include_content: true`, `response_sections: ["outgoing"]`, `relationTypes: ["CALLS"]` | +| What is the impact summary? | `impact`: `direction: "upstream"`, `summaryOnly: true`; risk calculation and totals are retained | +| Which affected symbols need inspection? | `impact`: `limit: 10`, `offset: 0`; page as indicated by the response | +| How does A reach B? | `trace`: `from_uid`, `to_uid` | + +`response_sections` selects data sections for single-repository MCP `query` or +`context` (not `@group` routes). Query choices: `processes`, `process_symbols`, +`definitions`. Context choices: `incoming`, `outgoing`, `processes`, `chain`, +`routes`, `typed_properties`; the selected symbol always remains. Context +`relationTypes` accepts uppercase relationship names advertised by its schema +and filters only the existing incoming/outgoing categories. It does not change +ranking, traversal, source retention or the bounded candidate/reference windows. + +Filters add `response_filter` with the selection and omitted fields/categories. +Warnings, ambiguity, staleness and completeness metadata always remain. An empty +filtered view is not proof of absence. Omit filters to expand, and never waive +impact checks because a selected category is empty. `maxTokens` is a byte cutoff, +not pagination: it can cut JSON. Prefer these filters, small result counts, and +source/chain expansion for a selected symbol rather than a large search result. +Stop once enough evidence answers the question; do not run every tool by default. + ### Paginating `list_repos` `list_repos` is paginated so a large registry is not truncated by MCP/LLM token limits. It takes optional `limit` (default **50**, max **200**) and `offset`, and returns: diff --git a/gitnexus-cursor-integration/skills/gitnexus-debugging/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-debugging/SKILL.md index 6051c33c4..2bca79663 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-debugging/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-debugging/SKILL.md @@ -17,7 +17,8 @@ description: "Use when the user is debugging a bug, tracing an error, or asking A root cause traced in the wrong repository is a wrong root cause. -Call `list_repos {}` before the first tool call. With one indexed repository, +Bind once using a confirmed registry name/path or configured MCP default. Call +`list_repos {}` if the target is not yet known; reuse that binding across calls. With one indexed repository, use the examples below as written. With more than one, pass `repo` on every call: an omitted `repo` normally errors, but under an MCP policy with a configured default it resolves to that default silently. If you cannot tell @@ -34,10 +35,10 @@ diagnosis. ## Workflow ``` -0. list_repos {} → Bind repo -1. query({search_query: ""}) → Find related execution flows +0. Bind the repo once (list_repos only if not already known) +1. If the suspect is unknown, query for the error/symptom with small result limits 2. context({name: ""}) → See callers/callees/processes -3. READ gitnexus://repo/{name}/process/{name} → Trace execution flow +3. Read a process resource only if its trace resolves an open question 4. cypher({statement: "MATCH path..."}) → Custom traces if needed ``` @@ -47,7 +48,7 @@ diagnosis. ## Checklist ``` -- [ ] list_repos {} — bind repo; explicit repo when >1 indexed, ask if ambiguous +- [ ] Once per session: bind repo; explicit repo when >1 indexed, ask if ambiguous - [ ] Understand the symptom (error message, unexpected behavior) - [ ] query for error text or related code - [ ] Identify the suspect function from returned processes @@ -58,6 +59,28 @@ diagnosis. - [ ] State the repository and index freshness with the diagnosis ``` +## Keep investigations focused + +Start with a known symbol's `context`, or a small `query` (`limit: 3`, +`max_symbols: 3`, `include_content: false`, `chain_depth: 0`) to discover one. +When the server advertises these single-repository filters, request only the +needed evidence: + +```text +context({uid: "", repo: "my-app", include_content: true, + response_sections: ["outgoing"], relationTypes: ["CALLS"]}) +trace({from_uid: "", to_uid: "", repo: "my-app"}) +``` + +`symbol` and completeness/staleness diagnostics remain in a filtered context. +`relationTypes` filters the returned incoming/outgoing categories only; it does +not expand their existing bounded window or filter an explicitly requested chain. +For callback registration, include `USES`; for imports or inheritance, request +those relation types or omit the filter. Omitted references are not absent +references. Broaden when the hypothesis needs it, and preserve UNKNOWN/partial +warnings. Confirm source and tests before claiming a root cause. A useful 2–3 +call investigation is a goal, not a reason to skip verification or pre-edit impact. + ## Debugging Patterns | Symptom | GitNexus Approach | diff --git a/gitnexus-cursor-integration/skills/gitnexus-exploring/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-exploring/SKILL.md index 89f9a3884..8a6ee3ba3 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-exploring/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-exploring/SKILL.md @@ -15,9 +15,10 @@ description: "Use when the user asks how code works, wants to understand archite ## Bind the repository first -Step 1 discovers what is indexed; every call after it must say which of those -it means. With one indexed repository, use the examples below as written. With -more than one, pass `repo` on every call: an omitted `repo` normally errors, +Bind once per session using a confirmed registry name/path or configured MCP +default; reuse it. Call `list_repos {}` when the target is not yet known, not before +every question. Never guess the registry name from the checkout directory. With +one indexed repository, use the examples below as written. With more than one, pass `repo` on every call: an omitted `repo` normally errors, but under an MCP policy with a configured default it resolves to that default silently. If you cannot tell which repository is meant, stop and ask. Report the bound repository and index freshness alongside your explanation. @@ -28,26 +29,64 @@ the bound repository and index freshness alongside your explanation. ## Workflow ``` -1. list_repos {} or READ gitnexus://repos → Discover indexed repos -2. READ gitnexus://repo/{name}/context → Codebase overview, check staleness -3. query({search_query: ""}) → Find related execution flows -4. context({name: ""}) → Deep dive on specific symbol -5. READ gitnexus://repo/{name}/process/{name} → Trace full execution flow +1. Bind the repository once; use hot-tool staleness on subsequent calls +2. If no symbol is known: query({search_query: "", limit: 3, max_symbols: 3}) +3. context({uid: ""}) → Inspect the selected symbol +4. Read a process resource only when its full trace answers an unresolved question ``` -> If step 2 says "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. +> If the index is stale → run `node .gitnexus/run.cjs analyze` in terminal. > Hot-tool `staleness` names which index answered (`branch`/`lastCommit`) and how fresh it is (`status`). Re-analyze only for `behind` or `diverged` — `current` is identity, `unknown` is unmeasurable. +## Ask for only the evidence needed + +Use options advertised by the connected server. `response_sections` and context +`relationTypes` are opt-in views for a single repository, not `@group` routing. +On an older server, use the existing limits and content/chain options instead. +Use filters when unwanted sections are bulky; their omission metadata can make +an already-small response slightly larger. +Aim for 2–3 focused calls after repository binding; stop when the question is +answered, but expand if evidence or a required safety check is still missing. + +```text +query({search_query: "payment retry", repo: "my-app", limit: 3, max_symbols: 3, + include_content: false, chain_depth: 0, + response_sections: ["process_symbols", "definitions"]}) +context({uid: "", repo: "my-app", include_content: true, + response_sections: ["incoming", "outgoing"], relationTypes: ["CALLS"]}) +// Before editing, assess impact using the normal traversal and risk calculation: +impact({target: "", repo: "my-app", direction: "upstream", summaryOnly: true}) +``` + +If the symbol is already known, start with `context`; use `uid`, or `name` plus +`file_path`/`kind`, to avoid ambiguous results. To answer only “who calls it?”, +request `response_sections: ["incoming"]`, `relationTypes: ["CALLS"]` and leave +`include_content` false. Keep `processes` in a query when the question is about +flows, rather than requesting a symbols-only view. + +Do not request source and multi-hop chains for every search hit. Fetch source +for the selected symbol only; for a large class/file, prefer `read_file` with +a small 0-based `startLine`/`endLine` window (subtract 1 from graph line numbers), +or `maxLines` for a whole-file read, if that tool is available. Use +`trace` for a known A→B path instead of repeatedly expanding `context`. + +Filters retain requested sections/categories and hide the others; they do not prove other references +are absent or make the existing bounded search exhaustive. Read `response_filter`, +`warning`, `partial`, `truncated`, `epistemic` and `staleness`. Remove filters when +broader evidence is needed. `maxTokens` is a last-resort byte cutoff that can cut +JSON: narrow the request rather than treating a cut response as complete. Never +use a filtered caller view to waive the required impact check or an UNKNOWN risk. + ## Checklist ``` -- [ ] list_repos {} — bind repo; explicit repo when >1 indexed, ask if ambiguous -- [ ] READ gitnexus://repo/{name}/context -- [ ] query for the concept you want to understand -- [ ] Review returned processes (execution flows) -- [ ] context on key symbols for callers/callees -- [ ] READ process resource for full execution traces -- [ ] Read source files for implementation details +- [ ] Once per session: bind repo; explicit repo when >1 indexed, ask if ambiguous +- [ ] Check hot-tool staleness; read the overview only if needed +- [ ] Query for the concept only if the relevant symbol is not already known +- [ ] Review returned symbols; include processes when execution flow matters +- [ ] context on selected symbols; request only relevant sections/relations +- [ ] Read a process resource only if its full trace is needed +- [ ] Read targeted source only when implementation details are needed - [ ] State the repository and index freshness with the explanation ``` @@ -84,18 +123,16 @@ with a single one. ## Example: "How does payment processing work?" -``` -1. list_repos {} → total: 1 (my-app) — bind it - READ gitnexus://repo/my-app/context → 918 symbols, 45 processes -2. query({search_query: "payment processing"}) - → CheckoutFlow: processPayment → validateCard → chargeStripe - → RefundFlow: initiateRefund → calculateRefund → processRefund -3. context({name: "processPayment"}) - → Incoming: checkoutHandler, webhookHandler - → Outgoing: validateCard, chargeStripe, saveTransaction -4. Read src/payments/processor.ts for implementation details -5. Answer, noting: Repository my-app, index current +With `my-app` already bound and the index current: + +```text +1. query({search_query: "payment processing", repo: "my-app", limit: 3, max_symbols: 3}) + → CheckoutFlow and RefundFlow, with symbol IDs and locations +2. context({uid: "", repo: "my-app", include_content: true, + response_sections: ["outgoing", "processes"]}) + → Selected source, callees, process participation and completeness diagnostics +3. Answer from that evidence; read one process resource only if its trace is still needed ``` -Had step 1 returned two repositories, every call above would carry -`repo: "my-app"`. +Report the repository and index freshness with the explanation. If `my-app` +had not yet been bound, discover it once with `list_repos` first. diff --git a/gitnexus/scripts/bench/mcp-response-filters.ts b/gitnexus/scripts/bench/mcp-response-filters.ts new file mode 100644 index 000000000..53c3051aa --- /dev/null +++ b/gitnexus/scripts/bench/mcp-response-filters.ts @@ -0,0 +1,121 @@ +/** Deterministic response-size check; no model calls, DB, or network. + * Run: node --import tsx scripts/bench/mcp-response-filters.ts + * Fixtures exercise wide graph responses, not task-solving or model billing. + */ +import assert from 'node:assert/strict'; +import { applyMcpResponseFilter, resolveMcpResponseFilter } from '../../src/mcp/response-filter.js'; +import { getNextStepHint } from '../../src/mcp/server.js'; + +const symbol = (index: number) => ({ + uid: `Function:src/payments/payment-service.ts:handlePayment${index}`, + name: `handlePayment${index}`, + kind: 'Function', + filePath: 'src/payments/payment-service.ts', + startLine: index * 10 + 1, +}); +const refs = Array.from({ length: 30 }, (_, i) => symbol(i)); +const chain = Array.from({ length: 2 }, (_, i) => ({ + depth: i + 1, + upstream: Array.from({ length: 50 }, (_, j) => symbol(i * 100 + j)), + downstream: Array.from({ length: 50 }, (_, j) => symbol(i * 100 + j + 50)), +})); +const processes = Array.from({ length: 5 }, (_, i) => ({ + id: `proc:payment-${i}`, + summary: `Payment workflow ${i}`, + priority: 0.8 - i / 10, + symbol_count: 3, + step_count: 10, + chain, +})); +const processSymbols = processes.flatMap((p, i) => + Array.from({ length: 3 }, (_, j) => ({ + id: symbol(i * 3 + j).uid, + name: symbol(i * 3 + j).name, + filePath: symbol(i * 3 + j).filePath, + process_id: p.id, + startLine: (i * 3 + j) * 10 + 1, + })), +); +const diagnostics = { + warning: 'Index has an unresolved dispatch boundary', + epistemic: 'lower-bound', + boundaries: ['Dynamic dispatch is unresolved'], + causes: { dispatchBoundary: 1 }, + staleness: { status: 'current', lastCommit: 'fixture' }, +}; +const query = { processes, process_symbols: processSymbols, definitions: [], ...diagnostics }; +const context = { + status: 'found', + symbol: symbol(0), + incoming: { calls: refs.slice(0, 10), uses: refs.slice(10) }, + outgoing: { calls: refs.slice(0, 10), imports: refs.slice(10) }, + processes: processes.map(({ chain: _chain, ...process }) => process), + chain, + ...diagnostics, +}; +const cases = [ + { + name: 'query symbols, expanded flows available', + tool: 'query', + result: query, + args: { response_sections: ['process_symbols', 'definitions'] }, + keep: ['process_symbols', 'definitions'], + }, + { + name: 'query symbols, no chain requested', + tool: 'query', + result: { ...query, processes: processes.map(({ chain: _chain, ...p }) => p) }, + args: { response_sections: ['process_symbols', 'definitions'] }, + keep: ['process_symbols', 'definitions'], + }, + { + name: 'context callers, expanded chain available', + tool: 'context', + result: context, + args: { response_sections: ['incoming'], relationTypes: ['CALLS'] }, + keep: ['symbol'], + }, + { + name: 'context callers, no chain requested', + tool: 'context', + result: Object.fromEntries(Object.entries(context).filter(([key]) => key !== 'chain')), + args: { response_sections: ['incoming'], relationTypes: ['CALLS'] }, + keep: ['symbol'], + }, +]; +const measurements = cases.map(({ name, tool, result, args, keep }) => { + const filtered = applyMcpResponseFilter(result, resolveMcpResponseFilter(tool, args)) as Record< + string, + unknown + >; + assert.equal(applyMcpResponseFilter(result, undefined), result); + for (const key of [...keep, ...Object.keys(diagnostics)]) { + assert.deepEqual(filtered[key], (result as Record)[key]); + } + if (tool === 'context') assert.deepEqual(filtered.incoming, { calls: context.incoming.calls }); + const format = (value: unknown) => + JSON.stringify(value, null, 2) + getNextStepHint(tool, { repo: 'fixture' }); + const baselineBytes = Buffer.byteLength(format(result)); + const filteredBytes = Buffer.byteLength(format(filtered)); + assert.deepEqual(JSON.parse(JSON.stringify(filtered)), filtered); + return { + name, + baselineBytes, + filteredBytes, + savedBytes: baselineBytes - filteredBytes, + reductionPercent: Number((100 * (1 - filteredBytes / baselineBytes)).toFixed(1)), + baselineEstimatedTokens: Math.ceil(baselineBytes / 4), + filteredEstimatedTokens: Math.ceil(filteredBytes / 4), + }; +}); +console.log( + JSON.stringify( + { + workload: 'synthetic graph responses; complete MCP text including hints', + estimator: 'UTF-8 bytes / 4, not model-specific billing tokens', + measurements, + }, + null, + 2, + ), +); diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index d9b5a5c15..685a4fafb 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -220,6 +220,7 @@ const LBUG_NATIVE = [ 'test/integration/group/cross-trace-e2e.test.ts', 'test/integration/group/graphql-resolve-symbol.test.ts', 'test/integration/local-backend.test.ts', + 'test/integration/bm25-symbol-hydration.test.ts', 'test/integration/local-backend-calltool.test.ts', 'test/integration/search-core.test.ts', 'test/integration/search-pool.test.ts', diff --git a/gitnexus/skills/gitnexus-debugging.md b/gitnexus/skills/gitnexus-debugging.md index 6051c33c4..2bca79663 100644 --- a/gitnexus/skills/gitnexus-debugging.md +++ b/gitnexus/skills/gitnexus-debugging.md @@ -17,7 +17,8 @@ description: "Use when the user is debugging a bug, tracing an error, or asking A root cause traced in the wrong repository is a wrong root cause. -Call `list_repos {}` before the first tool call. With one indexed repository, +Bind once using a confirmed registry name/path or configured MCP default. Call +`list_repos {}` if the target is not yet known; reuse that binding across calls. With one indexed repository, use the examples below as written. With more than one, pass `repo` on every call: an omitted `repo` normally errors, but under an MCP policy with a configured default it resolves to that default silently. If you cannot tell @@ -34,10 +35,10 @@ diagnosis. ## Workflow ``` -0. list_repos {} → Bind repo -1. query({search_query: ""}) → Find related execution flows +0. Bind the repo once (list_repos only if not already known) +1. If the suspect is unknown, query for the error/symptom with small result limits 2. context({name: ""}) → See callers/callees/processes -3. READ gitnexus://repo/{name}/process/{name} → Trace execution flow +3. Read a process resource only if its trace resolves an open question 4. cypher({statement: "MATCH path..."}) → Custom traces if needed ``` @@ -47,7 +48,7 @@ diagnosis. ## Checklist ``` -- [ ] list_repos {} — bind repo; explicit repo when >1 indexed, ask if ambiguous +- [ ] Once per session: bind repo; explicit repo when >1 indexed, ask if ambiguous - [ ] Understand the symptom (error message, unexpected behavior) - [ ] query for error text or related code - [ ] Identify the suspect function from returned processes @@ -58,6 +59,28 @@ diagnosis. - [ ] State the repository and index freshness with the diagnosis ``` +## Keep investigations focused + +Start with a known symbol's `context`, or a small `query` (`limit: 3`, +`max_symbols: 3`, `include_content: false`, `chain_depth: 0`) to discover one. +When the server advertises these single-repository filters, request only the +needed evidence: + +```text +context({uid: "", repo: "my-app", include_content: true, + response_sections: ["outgoing"], relationTypes: ["CALLS"]}) +trace({from_uid: "", to_uid: "", repo: "my-app"}) +``` + +`symbol` and completeness/staleness diagnostics remain in a filtered context. +`relationTypes` filters the returned incoming/outgoing categories only; it does +not expand their existing bounded window or filter an explicitly requested chain. +For callback registration, include `USES`; for imports or inheritance, request +those relation types or omit the filter. Omitted references are not absent +references. Broaden when the hypothesis needs it, and preserve UNKNOWN/partial +warnings. Confirm source and tests before claiming a root cause. A useful 2–3 +call investigation is a goal, not a reason to skip verification or pre-edit impact. + ## Debugging Patterns | Symptom | GitNexus Approach | diff --git a/gitnexus/skills/gitnexus-exploring.md b/gitnexus/skills/gitnexus-exploring.md index 89f9a3884..8a6ee3ba3 100644 --- a/gitnexus/skills/gitnexus-exploring.md +++ b/gitnexus/skills/gitnexus-exploring.md @@ -15,9 +15,10 @@ description: "Use when the user asks how code works, wants to understand archite ## Bind the repository first -Step 1 discovers what is indexed; every call after it must say which of those -it means. With one indexed repository, use the examples below as written. With -more than one, pass `repo` on every call: an omitted `repo` normally errors, +Bind once per session using a confirmed registry name/path or configured MCP +default; reuse it. Call `list_repos {}` when the target is not yet known, not before +every question. Never guess the registry name from the checkout directory. With +one indexed repository, use the examples below as written. With more than one, pass `repo` on every call: an omitted `repo` normally errors, but under an MCP policy with a configured default it resolves to that default silently. If you cannot tell which repository is meant, stop and ask. Report the bound repository and index freshness alongside your explanation. @@ -28,26 +29,64 @@ the bound repository and index freshness alongside your explanation. ## Workflow ``` -1. list_repos {} or READ gitnexus://repos → Discover indexed repos -2. READ gitnexus://repo/{name}/context → Codebase overview, check staleness -3. query({search_query: ""}) → Find related execution flows -4. context({name: ""}) → Deep dive on specific symbol -5. READ gitnexus://repo/{name}/process/{name} → Trace full execution flow +1. Bind the repository once; use hot-tool staleness on subsequent calls +2. If no symbol is known: query({search_query: "", limit: 3, max_symbols: 3}) +3. context({uid: ""}) → Inspect the selected symbol +4. Read a process resource only when its full trace answers an unresolved question ``` -> If step 2 says "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. +> If the index is stale → run `node .gitnexus/run.cjs analyze` in terminal. > Hot-tool `staleness` names which index answered (`branch`/`lastCommit`) and how fresh it is (`status`). Re-analyze only for `behind` or `diverged` — `current` is identity, `unknown` is unmeasurable. +## Ask for only the evidence needed + +Use options advertised by the connected server. `response_sections` and context +`relationTypes` are opt-in views for a single repository, not `@group` routing. +On an older server, use the existing limits and content/chain options instead. +Use filters when unwanted sections are bulky; their omission metadata can make +an already-small response slightly larger. +Aim for 2–3 focused calls after repository binding; stop when the question is +answered, but expand if evidence or a required safety check is still missing. + +```text +query({search_query: "payment retry", repo: "my-app", limit: 3, max_symbols: 3, + include_content: false, chain_depth: 0, + response_sections: ["process_symbols", "definitions"]}) +context({uid: "", repo: "my-app", include_content: true, + response_sections: ["incoming", "outgoing"], relationTypes: ["CALLS"]}) +// Before editing, assess impact using the normal traversal and risk calculation: +impact({target: "", repo: "my-app", direction: "upstream", summaryOnly: true}) +``` + +If the symbol is already known, start with `context`; use `uid`, or `name` plus +`file_path`/`kind`, to avoid ambiguous results. To answer only “who calls it?”, +request `response_sections: ["incoming"]`, `relationTypes: ["CALLS"]` and leave +`include_content` false. Keep `processes` in a query when the question is about +flows, rather than requesting a symbols-only view. + +Do not request source and multi-hop chains for every search hit. Fetch source +for the selected symbol only; for a large class/file, prefer `read_file` with +a small 0-based `startLine`/`endLine` window (subtract 1 from graph line numbers), +or `maxLines` for a whole-file read, if that tool is available. Use +`trace` for a known A→B path instead of repeatedly expanding `context`. + +Filters retain requested sections/categories and hide the others; they do not prove other references +are absent or make the existing bounded search exhaustive. Read `response_filter`, +`warning`, `partial`, `truncated`, `epistemic` and `staleness`. Remove filters when +broader evidence is needed. `maxTokens` is a last-resort byte cutoff that can cut +JSON: narrow the request rather than treating a cut response as complete. Never +use a filtered caller view to waive the required impact check or an UNKNOWN risk. + ## Checklist ``` -- [ ] list_repos {} — bind repo; explicit repo when >1 indexed, ask if ambiguous -- [ ] READ gitnexus://repo/{name}/context -- [ ] query for the concept you want to understand -- [ ] Review returned processes (execution flows) -- [ ] context on key symbols for callers/callees -- [ ] READ process resource for full execution traces -- [ ] Read source files for implementation details +- [ ] Once per session: bind repo; explicit repo when >1 indexed, ask if ambiguous +- [ ] Check hot-tool staleness; read the overview only if needed +- [ ] Query for the concept only if the relevant symbol is not already known +- [ ] Review returned symbols; include processes when execution flow matters +- [ ] context on selected symbols; request only relevant sections/relations +- [ ] Read a process resource only if its full trace is needed +- [ ] Read targeted source only when implementation details are needed - [ ] State the repository and index freshness with the explanation ``` @@ -84,18 +123,16 @@ with a single one. ## Example: "How does payment processing work?" -``` -1. list_repos {} → total: 1 (my-app) — bind it - READ gitnexus://repo/my-app/context → 918 symbols, 45 processes -2. query({search_query: "payment processing"}) - → CheckoutFlow: processPayment → validateCard → chargeStripe - → RefundFlow: initiateRefund → calculateRefund → processRefund -3. context({name: "processPayment"}) - → Incoming: checkoutHandler, webhookHandler - → Outgoing: validateCard, chargeStripe, saveTransaction -4. Read src/payments/processor.ts for implementation details -5. Answer, noting: Repository my-app, index current +With `my-app` already bound and the index current: + +```text +1. query({search_query: "payment processing", repo: "my-app", limit: 3, max_symbols: 3}) + → CheckoutFlow and RefundFlow, with symbol IDs and locations +2. context({uid: "", repo: "my-app", include_content: true, + response_sections: ["outgoing", "processes"]}) + → Selected source, callees, process participation and completeness diagnostics +3. Answer from that evidence; read one process resource only if its trace is still needed ``` -Had step 1 returned two repositories, every call above would carry -`repo: "my-app"`. +Report the repository and index freshness with the explanation. If `my-app` +had not yet been bound, discover it once with `list_repos` first. diff --git a/gitnexus/skills/gitnexus-guide.md b/gitnexus/skills/gitnexus-guide.md index 5f595e21a..e1adb6e91 100644 --- a/gitnexus/skills/gitnexus-guide.md +++ b/gitnexus/skills/gitnexus-guide.md @@ -11,7 +11,7 @@ Quick reference for all GitNexus MCP tools, resources, and the knowledge graph s For any task involving code understanding, debugging, impact analysis, or refactoring: -1. **Read `gitnexus://repo/{name}/context`** — codebase overview + check index freshness +1. **Bind the repository once** — use a confirmed registry name/path or configured default; call `list_repos` if unknown. Read hot-tool `staleness`; use `gitnexus://repo/{name}/context` when an overview is needed. 2. **Match your task to a skill below** and **read that skill file** 3. **Follow the skill's workflow and checklist** @@ -51,6 +51,39 @@ For any task involving code understanding, debugging, impact analysis, or refact | `group_sync` | Rebuild a group's Contract Registry (cross-repo HTTP contract links); run after `group.yaml` changes or member re-index | | `list_repos` | Discover indexed repos (paginated — `limit`/`offset`) | +### Choose a focused response + +Use the connected server's schema; older servers may not advertise the new +filters. All options below are explicit: omitting them preserves existing output. +Use filters for bulky unwanted sections; on an already-small response, omission +metadata can outweigh the bytes saved. + +| Question | Options | +| --- | --- | +| Which symbols implement this? | `query`: `limit: 3`, `max_symbols: 3`, `include_content: false`, `chain_depth: 0`, `response_sections: ["process_symbols", "definitions"]` | +| Which execution flows match? | `query`: keep `processes` in `response_sections`; join its `id` to `process_symbols[].process_id` | +| Who calls this exact symbol? | `context`: `uid`, `response_sections: ["incoming"]`, `relationTypes: ["CALLS"]` | +| What does it call, with source? | `context`: `uid`, `include_content: true`, `response_sections: ["outgoing"]`, `relationTypes: ["CALLS"]` | +| What is the impact summary? | `impact`: `direction: "upstream"`, `summaryOnly: true`; risk calculation and totals are retained | +| Which affected symbols need inspection? | `impact`: `limit: 10`, `offset: 0`; page as indicated by the response | +| How does A reach B? | `trace`: `from_uid`, `to_uid` | + +`response_sections` selects data sections for single-repository MCP `query` or +`context` (not `@group` routes). Query choices: `processes`, `process_symbols`, +`definitions`. Context choices: `incoming`, `outgoing`, `processes`, `chain`, +`routes`, `typed_properties`; the selected symbol always remains. Context +`relationTypes` accepts uppercase relationship names advertised by its schema +and filters only the existing incoming/outgoing categories. It does not change +ranking, traversal, source retention or the bounded candidate/reference windows. + +Filters add `response_filter` with the selection and omitted fields/categories. +Warnings, ambiguity, staleness and completeness metadata always remain. An empty +filtered view is not proof of absence. Omit filters to expand, and never waive +impact checks because a selected category is empty. `maxTokens` is a byte cutoff, +not pagination: it can cut JSON. Prefer these filters, small result counts, and +source/chain expansion for a selected symbol rather than a large search result. +Stop once enough evidence answers the question; do not run every tool by default. + ### Paginating `list_repos` `list_repos` is paginated so a large registry is not truncated by MCP/LLM token limits. It takes optional `limit` (default **50**, max **200**) and `offset`, and returns: diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index dc1ba40fa..af5ad07f6 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -47,6 +47,7 @@ import { isValidQueryParams } from '../../core/lbug/query-params.js'; import { toDisplayLine } from './line-display.js'; import { shapeQueryProcessAttaches } from './query-process-attaches.js'; import { LBUG_ID_PROBE_BATCH_SIZE, LBUG_QUERY_BATCH_SIZE } from '../../core/lbug/query-batch.js'; +import type { BM25SearchResult } from '../../core/search/bm25-index.js'; import { chunk, mapConcurrent } from '../../lib/utils.js'; import { pathSuffixOf } from './path-predicate.js'; import { isCobolFile, isJclFile } from '../../core/ingestion/cobol/file-types.js'; @@ -4119,13 +4120,57 @@ export class LocalBackend { // Guard against unexpected response shape (#1489) — ftsResponse.results // could be undefined when the FTS extension is unavailable in the MCP process. - const bm25Results = ftsResponse?.results ?? []; + const bm25Results: BM25SearchResult[] = ftsResponse?.results ?? []; const ftsUsed = ftsResponse?.ftsAvailable ?? false; const nonBenignErrors = ftsResponse?.nonBenignErrors; const missingIndexes = ftsResponse?.missingIndexes; const results: any[] = []; + // FTS supplies at most three exact symbol IDs per file. Fetch a bounded + // batch of files together instead of scanning the graph once per file. + // Keep the database's ORDER BY for each file, including null line values. + const hydratedSymbols = new Map(); + for (const hit of bm25Results) { + assertIdentityFields(hit.filePath); + for (const nodeId of hit.nodeIds ?? []) assertSymbolIdentity(nodeId); + } + for (const hits of chunk( + bm25Results.filter((hit) => hit.nodeIds?.length), + LBUG_QUERY_BATCH_SIZE, + )) { + const hitsByNodeId = new Map>(); + for (const hit of hits) { + for (const nodeId of hit.nodeIds!) { + if (!hitsByNodeId.has(nodeId)) hitsByNodeId.set(nodeId, new Set()); + hitsByNodeId.get(nodeId)!.add(hit); + } + } + try { + const symbols = await executeParameterized( + repo.lbugPath, + ` + MATCH (n) + WHERE n.id IN $nodeIds + RETURN n.id AS id, n.name AS name, labels(n)[0] AS type, n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine + ORDER BY startLine, id + `, + { nodeIds: [...hitsByNodeId.keys()] }, + ); + for (const sym of symbols) assertSymbolRowIdentity(sym); + for (const hit of hits) hydratedSymbols.set(hit, []); + for (const sym of symbols) { + for (const hit of hitsByNodeId.get(sym.id ?? sym[0]) ?? []) { + hydratedSymbols.get(hit)!.push(sym); + } + } + } catch (err) { + rethrowSymbolIdentityError(err); + // A batch failure must not downgrade otherwise-readable files. Retry + // those files individually through the existing fallback below. + } + } + for (const bm25Result of bm25Results) { const fullPath = bm25Result.filePath; assertIdentityFields(fullPath); @@ -4135,24 +4180,26 @@ export class LocalBackend { // Without this, LIMIT 3 on filePath returns arbitrary symbols rather than // the nodes that actually scored highest in the BM25 index. const nodeIds = bm25Result.nodeIds?.length ? bm25Result.nodeIds : null; - const symbols = nodeIds - ? await executeParameterized( - repo.lbugPath, - ` + const symbols = + hydratedSymbols.get(bm25Result) ?? + (nodeIds + ? await executeParameterized( + repo.lbugPath, + ` MATCH (n) WHERE n.id IN $nodeIds RETURN n.id AS id, n.name AS name, labels(n)[0] AS type, n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine ORDER BY startLine, id `, - { nodeIds }, - ) - : await executeParameterized( - repo.lbugPath, - // Same BasicBlock exclusion as detect_changes (#2082 U7): on a - // --pdg index a function-heavy file has far more BasicBlock rows - // than symbols, so an unfiltered LIMIT 3 would surface nameless - // substrate rows and displace the real symbols. - ` + { nodeIds }, + ) + : await executeParameterized( + repo.lbugPath, + // Same BasicBlock exclusion as detect_changes (#2082 U7): on a + // --pdg index a function-heavy file has far more BasicBlock rows + // than symbols, so an unfiltered LIMIT 3 would surface nameless + // substrate rows and displace the real symbols. + ` MATCH (n) WHERE n.filePath = $filePath AND NOT n.id STARTS WITH 'BasicBlock:' @@ -4160,8 +4207,8 @@ export class LocalBackend { ORDER BY startLine, id LIMIT 3 `, - { filePath: fullPath }, - ); + { filePath: fullPath }, + )); if (symbols.length > 0) { for (const sym of symbols) { diff --git a/gitnexus/src/mcp/response-filter.ts b/gitnexus/src/mcp/response-filter.ts new file mode 100644 index 000000000..c90940d91 --- /dev/null +++ b/gitnexus/src/mcp/response-filter.ts @@ -0,0 +1,143 @@ +/** Opt-in views of MCP results. Never changes graph queries or default output. */ +export const QUERY_RESPONSE_SECTIONS = ['processes', 'process_symbols', 'definitions'] as const; +export const CONTEXT_RESPONSE_SECTIONS = [ + 'incoming', + 'outgoing', + 'processes', + 'chain', + 'routes', + 'typed_properties', +] as const; +export const CONTEXT_RELATION_TYPES = [ + 'CALLS', + 'IMPORTS', + 'EXTENDS', + 'IMPLEMENTS', + 'MEMBER_OF', + 'USES', + 'DECLARES', + 'HAS_METHOD', + 'HAS_PROPERTY', + 'METHOD_OVERRIDES', + 'OVERRIDES', + 'METHOD_IMPLEMENTS', + 'ACCESSES', + 'ADVISED_BY', +] as const; + +export interface McpResponseFilter { + tool: 'query' | 'context'; + sections?: string[]; + relationTypes?: string[]; +} + +function selection(value: unknown, allowed: readonly string[], name: string): string[] | undefined { + if (value === undefined) return undefined; + if ( + !Array.isArray(value) || + value.length === 0 || + value.length > allowed.length || + value.some((item) => typeof item !== 'string' || !allowed.includes(item)) + ) { + throw new Error(`${name} must be a non-empty array containing only: ${allowed.join(', ')}.`); + } + return [...new Set(value)]; +} + +/** Validate before dispatch; unsupported group views must never silently ignore a filter. */ +export function resolveMcpResponseFilter( + tool: string, + args: Record | undefined, +): McpResponseFilter | undefined { + if (tool !== 'query' && tool !== 'context') return undefined; + const sections = selection( + args?.response_sections, + tool === 'query' ? QUERY_RESPONSE_SECTIONS : CONTEXT_RESPONSE_SECTIONS, + 'response_sections', + ); + const relationTypes = + tool === 'context' + ? selection(args?.relationTypes, CONTEXT_RELATION_TYPES, 'relationTypes') + : undefined; + if (!sections && !relationTypes) return undefined; + if (typeof args?.repo === 'string' && args.repo.trim().startsWith('@')) { + throw new Error( + 'Response filters require a single indexed repository name or path, not @group routing.', + ); + } + if ( + sections && + relationTypes && + !sections.some((section) => section === 'incoming' || section === 'outgoing') + ) { + throw new Error('relationTypes requires incoming or outgoing in response_sections.'); + } + return { tool, sections, relationTypes }; +} + +export function withoutMcpResponseFilterArgs( + args: Record | undefined, + filter: McpResponseFilter | undefined, +): Record | undefined { + if (!filter || !args) return args; + const backendArgs = { ...args }; + delete backendArgs.response_sections; + if (filter.tool === 'context') delete backendArgs.relationTypes; + return backendArgs; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** + * Remove only explicitly selectable data fields. Everything else survives, + * including future warnings, identity, ambiguity, risk and epistemic fields. + * Reference filtering is over the backend's existing bounded result, not a + * new exhaustive search. Report omissions without reinterpreting graph totals. + */ +export function applyMcpResponseFilter( + result: unknown, + filter: McpResponseFilter | undefined, +): unknown { + if ( + !filter || + !isRecord(result) || + 'error' in result || + (filter.tool === 'context' && result.status !== 'found') + ) + return result; + + const output = { ...result }; + const omittedSections: string[] = []; + const omittedRelations: Record = {}; + if (filter.sections) { + const available = filter.tool === 'query' ? QUERY_RESPONSE_SECTIONS : CONTEXT_RESPONSE_SECTIONS; + for (const section of available) { + if (!filter.sections.includes(section) && Object.hasOwn(output, section)) { + delete output[section]; + omittedSections.push(section); + } + } + } + if (filter.relationTypes) { + const categories = new Set(filter.relationTypes.map((type) => type.toLowerCase())); + for (const direction of ['incoming', 'outgoing']) { + const refs = output[direction]; + if (!isRecord(refs)) continue; + output[direction] = Object.fromEntries( + Object.entries(refs).filter(([type]) => categories.has(type)), + ); + const omitted = Object.keys(refs).filter((type) => !categories.has(type)); + if (omitted.length) omittedRelations[direction] = omitted; + } + } + output.response_filter = { + ...(filter.sections ? { sections: filter.sections } : {}), + ...(filter.relationTypes ? { relationTypes: filter.relationTypes } : {}), + omitted_sections: omittedSections, + ...(Object.keys(omittedRelations).length ? { omitted_relations: omittedRelations } : {}), + note: 'Filtered view of the existing result, not an exhaustive search. Omitted data is not evidence of absence. Remove filters to expand; existing limits and completeness signals still apply.', + }; + return output; +} diff --git a/gitnexus/src/mcp/server.ts b/gitnexus/src/mcp/server.ts index c621cd86d..3cfe8c402 100644 --- a/gitnexus/src/mcp/server.ts +++ b/gitnexus/src/mcp/server.ts @@ -43,6 +43,11 @@ import { } from './repository-policy.js'; import { applyMcpMaxTokens, resolveMcpMaxTokens, withoutMcpBudgetArg } from './output-budget.js'; import { assertKnownMcpToolArguments, schemaSourceToolName } from './tool-arguments.js'; +import { + applyMcpResponseFilter, + resolveMcpResponseFilter, + withoutMcpResponseFilterArgs, +} from './response-filter.js'; /** * Next-step hints appended to tool responses. @@ -269,8 +274,16 @@ export function createMCPServer( const listed = toolForReadOnlyMcp(repositoryPolicy.toolForMcp(advertisedTool), readOnly); assertKnownMcpToolArguments(name, typedArgs, listed.inputSchema.properties); } + const responseFilter = resolveMcpResponseFilter(schemaSource, typedArgs); maxTokens = resolveMcpMaxTokens(name, typedArgs); - const result = await scopedBackend.callTool(name, withoutMcpBudgetArg(typedArgs)); + const backendArgs = withoutMcpResponseFilterArgs( + withoutMcpBudgetArg(typedArgs), + responseFilter, + ); + const result = applyMcpResponseFilter( + await scopedBackend.callTool(name, backendArgs), + responseFilter, + ); const resultText = typeof result === 'string' ? result : JSON.stringify(result, null, 2); const hint = getNextStepHint(name, args as Record | undefined, readOnly, result); diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index 6d94097cd..90a9a2e0b 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -7,6 +7,11 @@ import type { ToolAnnotations } from '@modelcontextprotocol/sdk/types.js'; import { REL_TYPES } from 'gitnexus-shared'; +import { + CONTEXT_RELATION_TYPES, + CONTEXT_RESPONSE_SECTIONS, + QUERY_RESPONSE_SECTIONS, +} from './response-filter.js'; export interface ToolDefinition { name: string; @@ -20,7 +25,9 @@ export interface ToolDefinition { type: string; description?: string; default?: unknown; - items?: { type: string }; + items?: { type: string; enum?: string[] }; + minItems?: number; + maxItems?: number; enum?: string[]; const?: boolean; minimum?: number; @@ -204,6 +211,14 @@ ${HOT_READ_STALENESS_NOTE}`, minimum: 1, maximum: QUERY_MAX_MAX_SYMBOLS, }, + response_sections: { + type: 'array', + items: { type: 'string', enum: [...QUERY_RESPONSE_SECTIONS] }, + minItems: 1, + maxItems: QUERY_RESPONSE_SECTIONS.length, + description: + 'Optional single-repo output view: select processes, process_symbols, and/or definitions. For symbol discovery use ["process_symbols", "definitions"] with include_content:false and chain_depth:0. Warnings and staleness always remain; response_filter identifies omitted sections, not absent evidence. Does not change ranking or existing search limits. Omit for existing full output. Not supported for @group routing.', + }, include_content: { type: 'boolean', description: @@ -354,6 +369,22 @@ ${HOT_READ_STALENESS_NOTE}`, type: 'string', description: 'Direct symbol UID from prior tool results (zero-ambiguity lookup)', }, + response_sections: { + type: 'array', + items: { type: 'string', enum: [...CONTEXT_RESPONSE_SECTIONS] }, + minItems: 1, + maxItems: CONTEXT_RESPONSE_SECTIONS.length, + description: + 'Optional single-repo output view: select incoming, outgoing, processes, chain, routes, and/or typed_properties. The symbol and all warnings/completeness/staleness fields always remain. For callers only use ["incoming"] plus relationTypes:["CALLS"]. Omitted data is not evidence of absence; remove filters to expand. Does not change traversal or existing reference limits. Omit for existing full output. Not supported for @group routing.', + }, + relationTypes: { + type: 'array', + items: { type: 'string', enum: [...CONTEXT_RELATION_TYPES] }, + minItems: 1, + maxItems: CONTEXT_RELATION_TYPES.length, + description: + 'Optional uppercase relationship categories to retain in the existing incoming/outgoing view, e.g. ["CALLS"] or ["CALLS", "USES"]. This filters the returned bounded reference window; it does not search for additional matches or filter chain. If response_sections is set, it must include incoming or outgoing. Omit for all existing categories. Single-repo MCP only, not @group routing.', + }, file_path: { type: 'string', description: 'File path to disambiguate common names' }, file: { type: 'string', diff --git a/gitnexus/test/integration/bm25-symbol-hydration.test.ts b/gitnexus/test/integration/bm25-symbol-hydration.test.ts new file mode 100644 index 000000000..aac3e0fb6 --- /dev/null +++ b/gitnexus/test/integration/bm25-symbol-hydration.test.ts @@ -0,0 +1,65 @@ +import { expect, it, vi } from 'vitest'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { searchFTSFromLbug } from '../../src/core/search/bm25-index.js'; +import { executeParameterized } from '../../src/core/lbug/pool-adapter.js'; +import { withTestLbugDB } from '../helpers/test-indexed-db.js'; +import { LOCAL_BACKEND_SEED_DATA } from '../fixtures/local-backend-seed.js'; + +// Supply ranked FTS hits; exercise hydration with the real native graph and pool. +vi.mock('../../src/core/search/bm25-index.js', () => ({ searchFTSFromLbug: vi.fn() })); + +withTestLbugDB( + 'bm25-symbol-hydration', + (handle) => { + it('matches individual native reads across files and node tables', async () => { + const hits = [ + { + filePath: 'src/auth.ts', + score: 8, + rank: 1, + nodeIds: ['method:AuthService.authenticate', 'func:validate', 'func:login'], + }, + { filePath: 'src/utils.ts', score: 4, rank: 2, nodeIds: ['func:hash'] }, + { filePath: 'src/deleted.ts', score: 2, rank: 3, nodeIds: ['func:deleted'] }, + ]; + vi.mocked(searchFTSFromLbug).mockResolvedValue({ results: hits, ftsAvailable: true }); + const expected = []; + for (const hit of hits) { + const symbols = await executeParameterized( + handle.repoId, + `MATCH (n) WHERE n.id IN $nodeIds + RETURN n.id AS id, n.name AS name, labels(n)[0] AS type, + n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine + ORDER BY startLine, id`, + { nodeIds: hit.nodeIds }, + ); + if (symbols.length) { + expected.push( + ...symbols.map(({ id, ...symbol }) => ({ + nodeId: id, + ...symbol, + type: symbol.type || undefined, + bm25Score: hit.score, + })), + ); + } else { + expected.push({ + name: 'deleted.ts', + type: 'File', + filePath: hit.filePath, + bm25Score: hit.score, + }); + } + } + + const backend = new LocalBackend() as unknown as { + bm25Search: (repo: { lbugPath: string }, query: string, limit: number) => Promise; + }; + expect(await backend.bm25Search({ lbugPath: handle.repoId }, 'authentication', 20)).toEqual({ + results: expected, + ftsUsed: true, + }); + }); + }, + { seed: LOCAL_BACKEND_SEED_DATA, poolAdapter: true }, +); diff --git a/gitnexus/test/unit/local-backend-bm25-hydration.test.ts b/gitnexus/test/unit/local-backend-bm25-hydration.test.ts new file mode 100644 index 000000000..c06373c06 --- /dev/null +++ b/gitnexus/test/unit/local-backend-bm25-hydration.test.ts @@ -0,0 +1,177 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { searchFTS, executeParameterized } = vi.hoisted(() => ({ + searchFTS: vi.fn(), + executeParameterized: vi.fn(), +})); + +vi.mock('../../src/core/search/bm25-index.js', () => ({ searchFTSFromLbug: searchFTS })); +vi.mock('../../src/core/lbug/pool-adapter.js', async (importOriginal) => ({ + ...(await importOriginal()), + executeParameterized, +})); + +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; + +const repo = { lbugPath: '/tmp/bm25-hydration/lbug' }; +type TestRepo = typeof repo; +const row = (index: number) => ({ + id: `Function:src/file-${index}.ts:handler`, + name: 'handler', + type: 'Function', + filePath: `src/file-${index}.ts`, + startLine: index, + endLine: index + 1, +}); + +const search = () => + ( + new LocalBackend() as unknown as { + bm25Search: ( + repo: TestRepo, + query: string, + limit: number, + ) => Promise<{ results: unknown[]; ftsUsed: boolean }>; + } + ).bm25Search(repo, 'handler', 100); + +describe('LocalBackend BM25 symbol hydration', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('hydrates 100 matching files with one graph read and preserves their ranking', async () => { + const symbols = Array.from({ length: 100 }, (_, index) => row(index)); + searchFTS.mockResolvedValue({ + ftsAvailable: true, + results: symbols.map((symbol, index) => ({ + filePath: symbol.filePath, + score: 100 - index, + nodeIds: [symbol.id], + })), + }); + executeParameterized.mockImplementation(async (_repo, _cypher, params) => + symbols.filter((symbol) => params.nodeIds.includes(symbol.id)), + ); + + const result = await search(); + + expect(result.ftsUsed).toBe(true); + expect(result.results).toEqual( + symbols.map(({ id, ...symbol }, index) => ({ + nodeId: id, + ...symbol, + bm25Score: 100 - index, + })), + ); + expect(executeParameterized).toHaveBeenCalledTimes(1); + }); + + it('bounds batches without changing the FTS file ranking', async () => { + const symbols = Array.from({ length: 201 }, (_, index) => row(index)); + searchFTS.mockResolvedValue({ + ftsAvailable: true, + results: symbols.map((symbol) => ({ + filePath: symbol.filePath, + score: 1, + nodeIds: [symbol.id], + })), + }); + executeParameterized.mockImplementation(async (_repo, _cypher, params) => + symbols.filter((symbol) => params.nodeIds.includes(symbol.id)).reverse(), + ); + const result = await search(); + expect(result.results.map((hit: { nodeId: string }) => hit.nodeId)).toEqual( + symbols.map((symbol) => symbol.id), + ); + expect(executeParameterized).toHaveBeenCalledTimes(3); + for (const [, cypher, params] of executeParameterized.mock.calls) { + expect(params.nodeIds.length).toBeLessThanOrEqual(100); + expect(cypher).toContain('WHERE n.id IN $nodeIds'); + expect(cypher).toContain('ORDER BY startLine, id'); + } + }); + + it('keeps database ordering, scores, positional rows, and duplicate-ID semantics', async () => { + const first = row(0); + const second = { ...row(1), filePath: first.filePath }; + searchFTS.mockResolvedValue({ + ftsAvailable: true, + results: [{ filePath: second.filePath, score: 8, nodeIds: [second.id, first.id, second.id] }], + }); + executeParameterized.mockResolvedValue([Object.values(first), Object.values(second)]); + expect((await search()).results).toEqual( + [first, second].map(({ id, ...symbol }) => ({ nodeId: id, ...symbol, bm25Score: 8 })), + ); + expect(executeParameterized.mock.calls[0][2].nodeIds).toEqual([second.id, first.id]); + }); + + it('retains file-only and legacy path fallbacks without adding an empty batch', async () => { + searchFTS.mockResolvedValue({ + ftsAvailable: false, + missingIndexes: ['Function.symbol_fts'], + nonBenignErrors: ['diagnostic'], + results: [ + { filePath: 'src/missing.ts', score: 2, nodeIds: ['Function:missing'] }, + { filePath: 'src/legacy.ts', score: 1 }, + ], + }); + executeParameterized.mockResolvedValue([]); + expect(await search()).toEqual({ + ftsUsed: false, + missingIndexes: ['Function.symbol_fts'], + nonBenignErrors: ['diagnostic'], + results: [ + { name: 'missing.ts', type: 'File', filePath: 'src/missing.ts', bm25Score: 2 }, + { name: 'legacy.ts', type: 'File', filePath: 'src/legacy.ts', bm25Score: 1 }, + ], + }); + expect(executeParameterized).toHaveBeenCalledTimes(2); + expect(executeParameterized.mock.calls[1][1]).toContain("NOT n.id STARTS WITH 'BasicBlock:'"); + expect(executeParameterized.mock.calls[1][1]).toContain('LIMIT 3'); + + executeParameterized.mockClear(); + searchFTS.mockResolvedValue({ ftsAvailable: true, results: [] }); + expect(await search()).toEqual({ ftsUsed: true, results: [] }); + expect(executeParameterized).not.toHaveBeenCalled(); + }); + + it('retries failed batches per file so one failure does not hide another file', async () => { + const symbols = [row(0), row(1)]; + searchFTS.mockResolvedValue({ + ftsAvailable: true, + results: symbols.map((symbol) => ({ + filePath: symbol.filePath, + score: 1, + nodeIds: [symbol.id], + })), + }); + executeParameterized.mockImplementation(async (_repo, _cypher, params) => { + if (params.nodeIds.includes(symbols[0].id)) throw new Error('query failed'); + return [symbols[1]]; + }); + const { id, ...symbol } = symbols[1]; + expect((await search()).results).toEqual([ + { name: 'file-0.ts', type: 'File', filePath: 'src/file-0.ts', bm25Score: 1 }, + { nodeId: id, ...symbol, bm25Score: 1 }, + ]); + expect(executeParameterized).toHaveBeenCalledTimes(3); + }); + + it('rejects corrupt input and hydrated identities rather than returning file fallbacks', async () => { + searchFTS.mockResolvedValue({ + ftsAvailable: true, + results: [{ filePath: 'src/file-0.ts', score: 1, nodeIds: [''] }], + }); + await expect(search()).rejects.toThrow('invalid symbol identity'); + expect(executeParameterized).not.toHaveBeenCalled(); + + searchFTS.mockResolvedValue({ + ftsAvailable: true, + results: [{ filePath: 'src/file-0.ts', score: 1, nodeIds: [row(0).id] }], + }); + executeParameterized.mockResolvedValue([{ ...row(0), id: '' }]); + await expect(search()).rejects.toThrow('invalid symbol identity'); + expect(executeParameterized).toHaveBeenCalledTimes(1); + }); +}); diff --git a/gitnexus/test/unit/mcp-response-filter.test.ts b/gitnexus/test/unit/mcp-response-filter.test.ts new file mode 100644 index 000000000..75d1ce641 --- /dev/null +++ b/gitnexus/test/unit/mcp-response-filter.test.ts @@ -0,0 +1,205 @@ +import { describe, expect, it } from 'vitest'; +import { + applyMcpResponseFilter, + resolveMcpResponseFilter, + withoutMcpResponseFilterArgs, +} from '../../src/mcp/response-filter.js'; + +const context = { + status: 'found', + symbol: { uid: 'Function:src/pay.ts:pay', name: 'pay', content: 'function pay() {}' }, + incoming: { + calls: [{ uid: 'Function:src/buy.ts:buy' }], + uses: [{ uid: 'Variable:src/ui.ts:onBuy' }], + }, + outgoing: { calls: [{ uid: 'Function:src/db.ts:save' }], imports: [{ uid: 'File:src/db.ts' }] }, + processes: [{ id: 'process:checkout', name: 'Checkout' }], + routes: [{ url: '/pay', method: 'POST' }], + chain: [{ depth: 1, upstream: [{ uid: 'Function:src/buy.ts:buy' }] }], + typed_properties: [{ uid: 'Property:src/pay.ts:currency' }], + epistemic: 'lower-bound', + boundaries: ['Dynamic dispatch is unresolved'], + causes: { callableValueReferences: 1 }, + warning: 'Source extraction incomplete', + partial: true, + truncated: true, + risk: 'UNKNOWN', + riskNote: 'Not a safety verdict', + staleness: { status: 'behind', lastCommit: 'abc' }, + futureCompletenessSignal: { incomplete: true }, +}; + +describe('opt-in MCP response filters', () => { + it('leaves omitted options and default output identical, including object identity', () => { + const args = { uid: context.symbol.uid }; + const filter = resolveMcpResponseFilter('context', args); + expect(filter).toBeUndefined(); + expect(withoutMcpResponseFilterArgs(args, filter)).toBe(args); + expect(applyMcpResponseFilter(context, filter)).toBe(context); + }); + + it('returns exact selected evidence with identity, source and all safety metadata intact', () => { + const before = JSON.stringify(context); + const filter = resolveMcpResponseFilter('context', { + response_sections: ['incoming'], + relationTypes: ['CALLS'], + }); + const result = applyMcpResponseFilter(context, filter) as typeof context & { + response_filter: unknown; + }; + expect(result.incoming).toEqual({ calls: context.incoming.calls }); + expect(result.symbol).toEqual(context.symbol); + for (const key of [ + 'epistemic', + 'boundaries', + 'causes', + 'warning', + 'partial', + 'truncated', + 'risk', + 'riskNote', + 'staleness', + 'futureCompletenessSignal', + ] as const) { + expect(result[key]).toEqual(context[key]); + } + for (const key of ['outgoing', 'processes', 'routes', 'chain', 'typed_properties']) { + expect(result).not.toHaveProperty(key); + } + expect(result.response_filter).toMatchObject({ + sections: ['incoming'], + relationTypes: ['CALLS'], + omitted_relations: { incoming: ['uses'] }, + }); + expect(JSON.stringify(context)).toBe(before); + expect(JSON.parse(JSON.stringify(result))).toEqual(result); + }); + + it('distinguishes an empty selected category from an unfiltered absence', () => { + const result = applyMcpResponseFilter( + context, + resolveMcpResponseFilter('context', { + response_sections: ['incoming'], + relationTypes: ['EXTENDS'], + }), + ); + expect(result).toMatchObject({ + incoming: {}, + epistemic: 'lower-bound', + response_filter: { omitted_relations: { incoming: ['calls', 'uses'] } }, + }); + }); + + it('filters only reference categories, leaving an explicitly requested chain unchanged', () => { + const result = applyMcpResponseFilter( + context, + resolveMcpResponseFilter('context', { + relationTypes: ['USES'], + }), + ) as typeof context; + expect(result.incoming).toEqual({ uses: context.incoming.uses }); + expect(result.chain).toBe(context.chain); + expect(result.processes).toBe(context.processes); + }); + + it('preserves selected process attachments, duplicate memberships and definitions exactly', () => { + const query = { + processes: [ + { id: 'proc:one', symbol_count: 1 }, + { id: 'proc:two', symbol_count: 1 }, + ], + process_symbols: [ + { id: 'Function:src/pay.ts:pay', process_id: 'proc:one', content: 'source' }, + { id: 'Function:src/pay.ts:pay', process_id: 'proc:two' }, + ], + definitions: [{ id: 'Interface:src/pay.ts:Payment' }], + warning: 'keyword-only', + }; + const result = applyMcpResponseFilter( + query, + resolveMcpResponseFilter('query', { + response_sections: ['process_symbols', 'definitions'], + }), + ); + expect(result).toMatchObject({ + process_symbols: query.process_symbols, + definitions: query.definitions, + warning: query.warning, + response_filter: { omitted_sections: ['processes'] }, + }); + expect(result).not.toHaveProperty('processes'); + expect(query.processes).toHaveLength(2); + }); + + it.each([ + { error: 'not found' }, + { status: 'ambiguous', candidates: [{ uid: 'a' }], totalCandidates: 1 }, + { status: 'not_found' }, + 'backend error', + null, + ])('does not hide errors or ambiguity: %j', (value) => { + expect( + applyMcpResponseFilter( + value, + resolveMcpResponseFilter('context', { + response_sections: ['incoming'], + }), + ), + ).toBe(value); + }); + + it.each([ + { label: 'null', value: null }, + { label: 'empty array', value: [] }, + { label: 'scalar string', value: 'incoming' }, + { label: 'unsupported section', value: ['risk'] }, + { label: 'non-string member', value: ['incoming', null] }, + { label: 'prototype key', value: ['__proto__'] }, + { label: 'too many sections', value: Array(100).fill('incoming') }, + ])('rejects malformed section filters: $label', ({ value }) => { + expect(() => resolveMcpResponseFilter('context', { response_sections: value })).toThrow( + /response_sections/, + ); + }); + it.each([ + { label: 'null', value: null }, + { label: 'empty array', value: [] }, + { label: 'lowercase relation', value: ['calls'] }, + { label: 'unsupported relation', value: ['MISSING'] }, + { label: 'non-string member', value: ['CALLS', 1] }, + ])('rejects invalid relations: $label', ({ value }) => { + expect(() => resolveMcpResponseFilter('context', { relationTypes: value })).toThrow( + /relationTypes/, + ); + }); + it('rejects conflicting filters and unsupported group routing', () => { + expect(() => + resolveMcpResponseFilter('context', { + response_sections: ['processes'], + relationTypes: ['CALLS'], + }), + ).toThrow(/requires incoming or outgoing/); + for (const repo of ['@group', '@group/member']) { + expect(() => + resolveMcpResponseFilter('query', { repo, response_sections: ['definitions'] }), + ).toThrow(/single indexed repository/); + } + }); + it('strips only the filter arguments and leaves existing impact relationTypes alone', () => { + const args = { + uid: 'a', + response_sections: ['incoming'], + relationTypes: ['CALLS'], + maxTokens: 1000, + }; + expect(withoutMcpResponseFilterArgs(args, resolveMcpResponseFilter('context', args))).toEqual({ + uid: 'a', + maxTokens: 1000, + }); + const impact = { target: 'a', relationTypes: ['CALLS'] }; + expect(withoutMcpResponseFilterArgs(impact, resolveMcpResponseFilter('impact', impact))).toBe( + impact, + ); + expect(args.relationTypes).toEqual(['CALLS']); + }); +}); diff --git a/gitnexus/test/unit/server.test.ts b/gitnexus/test/unit/server.test.ts index 0681369b6..034c2b689 100644 --- a/gitnexus/test/unit/server.test.ts +++ b/gitnexus/test/unit/server.test.ts @@ -739,3 +739,97 @@ describe('installSignalShutdown (#1132)', () => { } }); }); + +// Response filters are optional transport views. Backends and unfiltered bytes +// retain their established contract, including hints and safety metadata. +describe('MCP response filters', () => { + const payload = { + status: 'found', + symbol: { uid: 'Function:src/pay.ts:pay', name: 'pay' }, + incoming: { + calls: [{ uid: 'Function:src/buy.ts:buy' }], + uses: [{ uid: 'Variable:src/ui.ts:handler' }], + }, + outgoing: { calls: [{ uid: 'Function:src/db.ts:save' }] }, + processes: [{ id: 'proc:checkout' }], + epistemic: 'lower-bound', + causes: { callableValueReferences: 1 }, + warning: 'Dynamic dispatch unresolved', + staleness: { status: 'current', lastCommit: 'abc' }, + }; + it('preserves the exact default formatted response', async () => { + const backend = createMockBackend({ callTool: vi.fn().mockResolvedValue(payload) }); + const args = { uid: payload.symbol.uid }; + const { text, isError } = await callToolThroughServer(backend, 'context', args); + expect(isError).toBe(false); + expect(text).toBe( + JSON.stringify(payload, null, 2) + getNextStepHint('context', args, false, payload), + ); + expect(backend.callTool).toHaveBeenCalledWith('context', args); + }); + it.each(['context', 'explore'])( + 'applies opt-in views for %s without sending filter options to the backend', + async (name) => { + const backend = createMockBackend({ callTool: vi.fn().mockResolvedValue(payload) }); + const { text, isError } = await callToolThroughServer(backend, name, { + uid: payload.symbol.uid, + response_sections: ['incoming'], + relationTypes: ['CALLS'], + }); + expect(isError).toBe(false); + const result = JSON.parse(text.split('\n\n---')[0]); + expect(result.incoming).toEqual({ calls: payload.incoming.calls }); + expect(result.symbol).toEqual(payload.symbol); + expect(result.epistemic).toBe('lower-bound'); + expect(result.causes).toEqual(payload.causes); + expect(result.warning).toBe(payload.warning); + expect(result.staleness).toEqual(payload.staleness); + expect(result).not.toHaveProperty('outgoing'); + expect(result).not.toHaveProperty('processes'); + expect(result.response_filter.omitted_relations).toEqual({ incoming: ['uses'] }); + expect(backend.callTool).toHaveBeenCalledWith(name, { uid: payload.symbol.uid }); + }, + ); + it.each(['query', 'search'])('selects query result sections through %s', async (name) => { + const data = { + processes: [{ id: 'proc:checkout' }], + process_symbols: [{ id: payload.symbol.uid, process_id: 'proc:checkout' }], + definitions: [], + }; + const backend = createMockBackend({ callTool: vi.fn().mockResolvedValue(data) }); + const { text, isError } = await callToolThroughServer(backend, name, { + search_query: 'pay', + response_sections: ['process_symbols', 'definitions'], + }); + expect(isError).toBe(false); + const result = JSON.parse(text.split('\n\n---')[0]); + expect(result.process_symbols).toEqual(data.process_symbols); + expect(result.definitions).toEqual([]); + expect(result).not.toHaveProperty('processes'); + expect(backend.callTool).toHaveBeenCalledWith(name, { search_query: 'pay' }); + }); + it.each([ + { response_sections: ['risk'] }, + { response_sections: [] }, + { relationTypes: ['CALSL'] }, + { response_sections: ['processes'], relationTypes: ['CALLS'] }, + { response_sections: ['incoming'], repo: '@group' }, + ])('rejects invalid filters before backend execution: %j', async (args) => { + const backend = createMockBackend(); + const { isError } = await callToolThroughServer(backend, 'context', { name: 'pay', ...args }); + expect(isError).toBe(true); + expect(backend.callTool).not.toHaveBeenCalled(); + }); + it('applies the existing byte budget after filtering, without changing the cutoff contract', async () => { + const backend = createMockBackend({ callTool: vi.fn().mockResolvedValue(payload) }); + const { text, isError } = await callToolThroughServer(backend, 'context', { + uid: payload.symbol.uid, + response_sections: ['incoming'], + maxTokens: 8, + }); + expect(isError).toBe(false); + expect(Buffer.byteLength(text)).toBeLessThanOrEqual(32); + expect(text.endsWith('\n…')).toBe(true); + expect(backend.callTool).toHaveBeenCalledWith('context', { uid: payload.symbol.uid }); + }); +}); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index a6325b510..1f8179356 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -92,6 +92,7 @@ export default defineConfig({ 'test/integration/lbug-pool.test.ts', 'test/integration/lbug-pool-stability.test.ts', 'test/integration/local-backend.test.ts', + 'test/integration/bm25-symbol-hydration.test.ts', 'test/integration/local-backend-calltool.test.ts', 'test/integration/spring-aop-mcp.test.ts', // Native @ladybugdb/core: loads a real graph through loadGraphToLbug @@ -199,6 +200,7 @@ export default defineConfig({ 'test/integration/lbug-pool.test.ts', 'test/integration/lbug-pool-stability.test.ts', 'test/integration/local-backend.test.ts', + 'test/integration/bm25-symbol-hydration.test.ts', 'test/integration/local-backend-calltool.test.ts', 'test/integration/spring-aop-mcp.test.ts', 'test/integration/spring-destinations-lbug.test.ts',