scriptoriumadmin/GitNexus
1
0
Fork 0
mirror of https://github.com/abhigyanpatwari/GitNexus.git synced 2026-10-05 02:43:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

Merge branch 'main' into fix/dart-tree-sitter-napi-and-queries

Browse source
This commit is contained in:
Gergő Magyar 2026-05-08 12:03:42 +01:00 • committed by GitHub
commit 42c0d1c5a3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
135 changed files with 6507 additions and 747 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
.github/scripts/check-tree-sitter-upgrade-readiness.py vendored
View file

@ -32,6 +32,7 @@ import pathlib
import re
import sys
import urllib.error
import urllib.parse
import urllib.request
REPO_ROOT = pathlib.Path(__file__).resolve().parents[2]
@ -190,7 +191,17 @@ def fetch_text(url: str, timeout: int = 8) -> str | None:
set (raises the rate limit from 60 to 5 000 requests/hour).
"""
headers: dict[str, str] = {}
if _GITHUB_TOKEN and ("github.com" in url or "githubusercontent.com" in url):
# Parse the URL and check the hostname rather than substring-matching
# on the full URL string (CodeQL py/incomplete-url-substring-sanitization).
# `https://evil.com/?u=github.com` would have passed the substring check.
try:
parsed_host = urllib.parse.urlparse(url).hostname or ""
except ValueError:
parsed_host = ""
is_github_host = parsed_host == "github.com" or parsed_host.endswith(
(".github.com", ".githubusercontent.com")
) or parsed_host == "githubusercontent.com"
if _GITHUB_TOKEN and is_github_host:
headers["Authorization"] = f"Bearer {_GITHUB_TOKEN}"
try:
req = urllib.request.Request(url, headers=headers)

68
.github/workflows/ci-report.yml vendored
View file

@ -138,14 +138,15 @@ jobs:
const fs = require('fs');
const path = require('path');
// Find the latest successful CI run on main
// Find recent successful CI runs on main (check several in case
// the most recent artifact has expired).
const runs = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'ci.yml',
branch: 'main',
status: 'success',
per_page: 1,
per_page: 5,
});
if (runs.data.workflow_runs.length === 0) {
@ -154,32 +155,47 @@ jobs:
return;
}
const mainRunId = runs.data.workflow_runs[0].id;
const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: mainRunId,
});
// Try each run until we find a downloadable test-reports artifact
for (const run of runs.data.workflow_runs) {
const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: run.id,
});
const testReports = artifacts.data.artifacts.find(a => a.name === 'test-reports');
if (!testReports) {
core.setOutput('found', 'false');
core.info('No test-reports artifact on main branch');
return;
const testReports = artifacts.data.artifacts.find(a => a.name === 'test-reports');
if (!testReports) {
core.info(`Run ${run.id}: no test-reports artifact, trying next`);
continue;
}
try {
const zip = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: testReports.id,
archive_format: 'zip',
});
const dest = path.join(process.env.RUNNER_TEMP, 'base-coverage');
fs.mkdirSync(dest, { recursive: true });
fs.writeFileSync(path.join(dest, 'base.zip'), Buffer.from(zip.data));
core.setOutput('found', 'true');
core.setOutput('dir', dest);
return;
} catch (err) {
// 410 Gone means the artifact expired; try the next run
if (err.status === 410 || err.response?.status === 410) {
core.info(`Run ${run.id}: artifact expired, trying next`);
continue;
}
throw err;
}
}
const zip = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: testReports.id,
archive_format: 'zip',
});
const dest = path.join(process.env.RUNNER_TEMP, 'base-coverage');
fs.mkdirSync(dest, { recursive: true });
fs.writeFileSync(path.join(dest, 'base.zip'), Buffer.from(zip.data));
core.setOutput('found', 'true');
core.setOutput('dir', dest);
// All attempts exhausted — no usable base coverage
core.setOutput('found', 'false');
core.info('No downloadable test-reports artifact found on main (all expired or missing)');
- name: Extract base coverage
if: steps.meta.outputs.skip != 'true' && steps.base-coverage.outputs.found == 'true'
@ -234,7 +250,7 @@ jobs:
printf -v "${prefix}_BRANCH_COV" '%s' ""
printf -v "${prefix}_FUNCS_COV" '%s' ""
printf -v "${prefix}_LINES_COV" '%s' ""
return 1
return 0
fi
}

4
.github/workflows/codeql.yml vendored
View file

@ -45,7 +45,7 @@ jobs:
persist-credentials: false
- name: Initialize CodeQL
uses: github/codeql-action/init@0daab03d71ff584ef619d027a3fd9146679c5d84 # v3.35.3
uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
with:
languages: ${{ matrix.language }}
queries: security-and-quality
@ -66,6 +66,6 @@ jobs:
- '**/test/fixtures/**'
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@0daab03d71ff584ef619d027a3fd9146679c5d84 # v3.35.3
uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
with:
category: '/language:${{ matrix.language }}'

16
.github/workflows/release-candidate.yml vendored
View file

@ -123,7 +123,16 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: write # push rc tag + marker
# The default GITHUB_TOKEN cannot be granted `workflows: write`, so
# tag pushes that reach a commit which modified `.github/workflows/**`
# are rejected with: "refusing to allow a GitHub App to create or
# update workflow ... without `workflows` permission". We pass a
# fine-grained PAT (RELEASE_PUSH_TOKEN, scoped to this repo with
# Contents: write + Workflows: write) to `actions/checkout` so that
# the subsequent `git push --atomic` of the v-tag and rc marker
# carries the PAT's identity. Job-level GITHUB_TOKEN keeps its
# scoped permissions for everything else (npm provenance, etc.).
contents: write # push rc tag + marker (via PAT)
id-token: write # npm provenance
outputs:
vtag: ${{ steps.reltag.outputs.vtag }}
@ -132,6 +141,11 @@ jobs:
with:
fetch-depth: 0
fetch-tags: true
# Use the PAT so `origin` is preauthed for `git push`. Without
# this the default GITHUB_TOKEN is wired into the remote, and a
# workflows-touching tag push is rejected — see the permissions
# block above.
token: ${{ secrets.RELEASE_PUSH_TOKEN }}
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:

2
.github/workflows/scorecard.yml vendored
View file

@ -53,6 +53,6 @@ jobs:
retention-days: 5
- name: Upload to Security tab
uses: github/codeql-action/upload-sarif@0daab03d71ff584ef619d027a3fd9146679c5d84 # v3.35.3
uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
with:
sarif_file: results.sarif

4
.github/workflows/trivy.yml vendored
View file

@ -44,7 +44,7 @@ jobs:
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Build image (load locally for scan)
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
with:
context: .
file: ${{ matrix.image.dockerfile }}
@ -67,7 +67,7 @@ jobs:
exit-code: '0'
- name: Upload to Security tab
uses: github/codeql-action/upload-sarif@0daab03d71ff584ef619d027a3fd9146679c5d84 # v3.35.3
uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
with:
sarif_file: trivy-${{ matrix.image.name }}.sarif
category: trivy-${{ matrix.image.name }}

2
.github/workflows/workflow-lint.yml vendored
View file

@ -49,7 +49,7 @@ jobs:
continue-on-error: true
- name: Upload SARIF
uses: github/codeql-action/upload-sarif@0daab03d71ff584ef619d027a3fd9146679c5d84 # v3.35.3
uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
with:
sarif_file: zizmor.sarif
category: zizmor

4
README.md
View file

@ -109,6 +109,8 @@ That's it. This indexes the codebase, installs agent skills, registers Claude Co
To configure MCP for your editor, run `npx gitnexus setup` once — or set it up manually below.
> **Faster install (no C++ toolchain needed):** set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before `npm install -g gitnexus` to skip the native `tree-sitter-dart` and `tree-sitter-proto` builds. Dart/Proto files won't be parsed, but install completes in seconds without `python3`/`make`/`g++`. Strict `=1` only — any other value falls through to the rebuild.
### MCP Setup
`gitnexus setup` auto-detects your editors and writes the correct global MCP config. You only need to run it once.
@ -138,6 +140,8 @@ Built by the community — not officially maintained, but worth checking out.
If you prefer manual configuration:
> **Recommended for fastest startup:** install gitnexus globally (`npm i -g gitnexus`) and run `gitnexus setup` — this writes an absolute-path MCP config that bypasses `npx` entirely. The pinned-`npx` snippets below are a quickstart fallback; on a cold cache the `npx` install can exceed Claude Code's `MCP_TIMEOUT` default (~30s).
**Claude Code** (full support — MCP + skills + hooks):
```bash

79
eslint.config.mjs
View file

@ -4,6 +4,38 @@ import unusedImports from 'eslint-plugin-unused-imports';
import reactHooks from 'eslint-plugin-react-hooks';
import prettierConfig from 'eslint-config-prettier';
// Selectors that protect MCP-reachable code from corrupting the JSON-RPC
// stdio frame stream. The MCP-reachable block below uses these directly;
// the lbug-adapter file-specific block must spread them in too because
// ESLint flat config REPLACES (not merges) `no-restricted-syntax` when
// multiple matching configs target the same file. Extracting to a const
// makes the dependency mechanical instead of documentation-enforced.
const mcpStdoutWriteSelectors = [
{
selector:
"MemberExpression[object.type='MemberExpression'][object.object.name='process'][object.property.name='stdout'][property.name='write']",
message:
'Direct process.stdout.write is forbidden in MCP-reachable code. Route diagnostics through console.error or process.stderr.write — the MCP stdio transport owns stdout for JSON-RPC frames.',
},
{
selector:
"CallExpression[callee.type='MemberExpression'][callee.object.type='MemberExpression'][callee.object.object.name='process'][callee.object.property.name='stdout'][callee.property.name='write']",
message:
'Direct process.stdout.write is forbidden in MCP-reachable code. Route diagnostics through console.error or process.stderr.write — the MCP stdio transport owns stdout for JSON-RPC frames.',
},
{
// Catches the canonical destructuring shape:
// const { write } = process.stdout;
// (and any other ObjectPattern destructure rooted at process.stdout)
// which would otherwise capture a reference to the original write
// and bypass the sentinel.
selector:
"VariableDeclarator[init.type='MemberExpression'][init.object.name='process'][init.property.name='stdout'] > ObjectPattern",
message:
'Destructuring process.stdout is forbidden in MCP-reachable code — bypasses the sentinel. Use process.stderr.write for diagnostics.',
},
];
export default [
// Global ignores
{
@ -59,11 +91,47 @@ export default [
},
},
// CLI package — allow console.log (it's a CLI tool)
// CLI/server packages — `console.log` IS the contract (CLI tool data output
// on stdout, e.g. `gitnexus query | jq`; server pretty-printed banners).
// Diagnostic logging (`warn`/`error`/`debug`/`info`) goes through pino like
// the rest of the codebase.
{
files: ['gitnexus/src/cli/**/*.ts', 'gitnexus/src/server/**/*.ts'],
rules: {
'no-console': 'off',
'no-console': ['error', { allow: ['log'] }],
},
},
// Forcing function for the pino migration. Severity is `error` — the
// codebase-wide migration is complete; new `console.*` in core source
// must fail lint. CLI/server are exempt above (legitimate stdout output).
// Tests, bin scripts, and the logger module itself remain exempt.
{
files: ['gitnexus/src/**/*.ts'],
ignores: ['gitnexus/src/cli/**', 'gitnexus/src/server/**', 'gitnexus/src/core/logger.ts'],
rules: {
'no-console': 'error',
},
},
// MCP-reachable code: forbid stdout-corrupting writes. The MCP stdio
// transport writes JSON-RPC frames to stdout; per the spec, the server
// MUST NOT write anything to stdout that is not a valid MCP message.
// Diagnostics must go to stderr (console.error). Direct process.stdout.write
// bypasses the gate and is also forbidden in these dirs.
// cli/mcp.ts is included here even though it lives under cli/ — it is the
// MCP entrypoint and inherits stricter discipline than the rest of cli/.
{
files: [
'gitnexus/src/mcp/**/*.ts',
'gitnexus/src/core/lbug/**/*.ts',
'gitnexus/src/core/embeddings/**/*.ts',
'gitnexus/src/core/tree-sitter/**/*.ts',
'gitnexus/src/cli/mcp.ts',
],
rules: {
'no-console': ['error', { allow: ['error'] }],
'no-restricted-syntax': ['error', ...mcpStdoutWriteSelectors],
},
},
@ -83,11 +151,18 @@ export default [
// All close operations must go through safeClose() so the WAL is always
// flushed before the connection is released. The sole authorised call site
// inside safeClose itself uses an eslint-disable-next-line override.
//
// ESLint flat config REPLACES (not merges) `no-restricted-syntax` when
// multiple matching configs target the same file. lbug-adapter.ts is also
// covered by the MCP-reachable block above, so we spread the shared
// mcpStdoutWriteSelectors here alongside the safeClose selectors. Without
// this, lbug-adapter would silently lose its MCP stdout-write protection.
{
files: ['gitnexus/src/core/lbug/lbug-adapter.ts'],
rules: {
'no-restricted-syntax': [
'error',
...mcpStdoutWriteSelectors,
{
selector: "CallExpression[callee.object.name='conn'][callee.property.name='close']",
message: 'Use safeClose() instead of calling conn.close() directly (#1376).',

10
gitnexus-web/package-lock.json generated
View file

@ -8,7 +8,7 @@
"name": "gitnexus",
"version": "0.0.0",
"dependencies": {
"@langchain/anthropic": "^1.3.27",
"@langchain/anthropic": "^1.3.28",
"@langchain/core": "^1.1.44",
"@langchain/google-genai": "^2.1.28",
"@langchain/langgraph": "^1.2.9",
@ -1396,9 +1396,9 @@
}
},
"node_modules/@langchain/anthropic": {
"version": "1.3.27",
"resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.27.tgz",
"integrity": "sha512-A0pWKIMIhgF01z3ILA8uAbZ6ZR2H8UQP2Ww8Ofq5DtHp36uJkQgrNCdS+q9pUVJJ87eq5dEdFkpjrjmH4fVkfQ==",
"version": "1.3.28",
"resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.28.tgz",
"integrity": "sha512-gOF8oXJL8xDdYes2KXNI9vFm/9TldBBBHOjuCdt27kganVaQKzLvTw5kV6R4mjbnFagV5CWteNH7APLZYCpdwg==",
"license": "MIT",
"dependencies": {
"@anthropic-ai/sdk": "^0.90.0",
@ -1408,7 +1408,7 @@
"node": ">=20"
},
"peerDependencies": {
"@langchain/core": "^1.1.41"
"@langchain/core": "^1.1.42"
}
},
"node_modules/@langchain/core": {

2
gitnexus-web/package.json
View file

@ -19,7 +19,7 @@
},
"dependencies": {
"gitnexus-shared": "file:../gitnexus-shared",
"@langchain/anthropic": "^1.3.27",
"@langchain/anthropic": "^1.3.28",
"@langchain/core": "^1.1.44",
"@langchain/google-genai": "^2.1.28",
"@langchain/langgraph": "^1.2.9",

6
gitnexus-web/src/core/llm/agent.ts
View file

@ -277,8 +277,10 @@ const extractInstanceName = (endpoint: string): string => {
try {
const url = new URL(endpoint);
const hostname = url.hostname;
// Extract the first part before .openai.azure.com
const match = hostname.match(/^([^.]+)\.openai\.azure\.com/);
// Extract the first part before .openai.azure.com. The trailing `$`
// anchor is required (CodeQL js/regex/missing-regexp-anchor): without
// it `evil.openai.azure.com.attacker.tld` would match.
const match = hostname.match(/^([^.]+)\.openai\.azure\.com$/);
if (match) {
return match[1];
}

7
gitnexus-web/src/core/llm/tools.ts
View file

@ -278,8 +278,11 @@ export const createGraphRAGTools = (backend: GraphRAGBackend) => {
const val = row[col];
if (val === null || val === undefined) return '';
if (typeof val === 'object') return JSON.stringify(val);
// Truncate long values and escape pipe characters
const str = String(val).replace(/\|/g, '\\|');
// Truncate long values and escape pipe characters. Escape
// backslashes FIRST so the subsequent pipe escape isn't
// unescaped by a trailing backslash (CodeQL
// js/incomplete-sanitization).
const str = String(val).replace(/\\/g, '\\\\').replace(/\|/g, '\\|');
return str.length > 60 ? str.slice(0, 57) + '...' : str;
});
return `| ${values.join(' | ')} |`;

252
gitnexus/package-lock.json generated
View file

@ -30,6 +30,8 @@
"mnemonist": "^0.40.3",
"onnxruntime-node": "^1.24.0",
"pandemonium": "^2.4.0",
"pino": "^10.3.1",
"pino-pretty": "^13.1.3",
"tree-sitter": "^0.21.1",
"tree-sitter-c": "0.21.4",
"tree-sitter-c-sharp": "0.23.1",
@ -1579,6 +1581,12 @@
"url": "https://github.com/sponsors/Boshen"
}
},
"node_modules/@pinojs/redact": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
"integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==",
"license": "MIT"
},
"node_modules/@protobufjs/aspromise": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
@ -2057,9 +2065,9 @@
"license": "MIT"
},
"node_modules/@types/node": {
"version": "25.6.0",
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz",
"integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==",
"version": "25.6.1",
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.1.tgz",
"integrity": "sha512-coJCN8O1q4AGyyqCAUSP06P+SrMTu18BkEj3NVAK07q6QUneD2wzj3CLv9+yP+BMeZQlMvneXqqvDe3w+xcq7g==",
"license": "MIT",
"dependencies": {
"undici-types": "~7.19.0"
@ -2380,6 +2388,15 @@
"js-tokens": "^10.0.0"
}
},
"node_modules/atomic-sleep": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
"integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==",
"license": "MIT",
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/balanced-match": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
@ -2579,6 +2596,12 @@
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"license": "MIT"
},
"node_modules/colorette": {
"version": "2.0.20",
"resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz",
"integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==",
"license": "MIT"
},
"node_modules/commander": {
"version": "14.0.3",
"resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz",
@ -2683,6 +2706,15 @@
"node": ">= 8"
}
},
"node_modules/dateformat": {
"version": "4.6.3",
"resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",
"integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==",
"license": "MIT",
"engines": {
"node": "*"
}
},
"node_modules/debug": {
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
@ -2806,6 +2838,15 @@
"node": ">= 0.8"
}
},
"node_modules/end-of-stream": {
"version": "1.4.5",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
"integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",
"license": "MIT",
"dependencies": {
"once": "^1.4.0"
}
},
"node_modules/es-define-property": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
@ -3050,12 +3091,24 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/fast-copy": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.0.3.tgz",
"integrity": "sha512-58apWr0GUiDFM8+3afrO6eYwJBn9ZAhDOzG3L+/9llab/haCARS2UIfffmOurYLwbgDRs8n0rfr6qAAPEAuAQw==",
"license": "MIT"
},
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
"license": "MIT"
},
"node_modules/fast-safe-stringify": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
"integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==",
"license": "MIT"
},
"node_modules/fast-uri": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz",
@ -3416,6 +3469,12 @@
"node": ">= 0.4"
}
},
"node_modules/help-me": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz",
"integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==",
"license": "MIT"
},
"node_modules/hono": {
"version": "4.12.16",
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz",
@ -3575,6 +3634,15 @@
"url": "https://github.com/sponsors/panva"
}
},
"node_modules/joycon": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz",
"integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==",
"license": "MIT",
"engines": {
"node": ">=10"
}
},
"node_modules/js-tokens": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz",
@ -4183,6 +4251,15 @@
],
"license": "MIT"
},
"node_modules/on-exit-leak-free": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
"integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==",
"license": "MIT",
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/on-finished": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
@ -4332,6 +4409,79 @@
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/pino": {
"version": "10.3.1",
"resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz",
"integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==",
"license": "MIT",
"dependencies": {
"@pinojs/redact": "^0.4.0",
"atomic-sleep": "^1.0.0",
"on-exit-leak-free": "^2.1.0",
"pino-abstract-transport": "^3.0.0",
"pino-std-serializers": "^7.0.0",
"process-warning": "^5.0.0",
"quick-format-unescaped": "^4.0.3",
"real-require": "^0.2.0",
"safe-stable-stringify": "^2.3.1",
"sonic-boom": "^4.0.1",
"thread-stream": "^4.0.0"
},
"bin": {
"pino": "bin.js"
}
},
"node_modules/pino-abstract-transport": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz",
"integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==",
"license": "MIT",
"dependencies": {
"split2": "^4.0.0"
}
},
"node_modules/pino-pretty": {
"version": "13.1.3",
"resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-13.1.3.tgz",
"integrity": "sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg==",
"license": "MIT",
"dependencies": {
"colorette": "^2.0.7",
"dateformat": "^4.6.3",
"fast-copy": "^4.0.0",
"fast-safe-stringify": "^2.1.1",
"help-me": "^5.0.0",
"joycon": "^3.1.1",
"minimist": "^1.2.6",
"on-exit-leak-free": "^2.1.0",
"pino-abstract-transport": "^3.0.0",
"pump": "^3.0.0",
"secure-json-parse": "^4.0.0",
"sonic-boom": "^4.0.1",
"strip-json-comments": "^5.0.2"
},
"bin": {
"pino-pretty": "bin.js"
}
},
"node_modules/pino-pretty/node_modules/strip-json-comments": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.3.tgz",
"integrity": "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==",
"license": "MIT",
"engines": {
"node": ">=14.16"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/pino-std-serializers": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz",
"integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==",
"license": "MIT"
},
"node_modules/pkce-challenge": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz",
@ -4376,6 +4526,22 @@
"node": "^10 || ^12 || >=14"
}
},
"node_modules/process-warning": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz",
"integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fastify"
},
{
"type": "opencollective",
"url": "https://opencollective.com/fastify"
}
],
"license": "MIT"
},
"node_modules/protobufjs": {
"version": "7.5.5",
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz",
@ -4413,6 +4579,16 @@
"node": ">= 0.10"
}
},
"node_modules/pump": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz",
"integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==",
"license": "MIT",
"dependencies": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
}
},
"node_modules/qs": {
"version": "6.14.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz",
@ -4428,6 +4604,12 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/quick-format-unescaped": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz",
"integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==",
"license": "MIT"
},
"node_modules/range-parser": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
@ -4483,6 +4665,15 @@
"rc": "cli.js"
}
},
"node_modules/real-require": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz",
"integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==",
"license": "MIT",
"engines": {
"node": ">= 12.13.0"
}
},
"node_modules/require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@ -4591,12 +4782,37 @@
],
"license": "MIT"
},
"node_modules/safe-stable-stringify": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz",
"integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==",
"license": "MIT",
"engines": {
"node": ">=10"
}
},
"node_modules/safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"license": "MIT"
},
"node_modules/secure-json-parse": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz",
"integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fastify"
},
{
"type": "opencollective",
"url": "https://opencollective.com/fastify"
}
],
"license": "BSD-3-Clause"
},
"node_modules/semver": {
"version": "7.7.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz",
@ -4828,6 +5044,15 @@
"dev": true,
"license": "ISC"
},
"node_modules/sonic-boom": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz",
"integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==",
"license": "MIT",
"dependencies": {
"atomic-sleep": "^1.0.0"
}
},
"node_modules/source-map-js": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
@ -4838,6 +5063,15 @@
"node": ">=0.10.0"
}
},
"node_modules/split2": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
"license": "ISC",
"engines": {
"node": ">= 10.x"
}
},
"node_modules/stackback": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz",
@ -4925,6 +5159,18 @@
"node": ">=18"
}
},
"node_modules/thread-stream": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.0.0.tgz",
"integrity": "sha512-4iMVL6HAINXWf1ZKZjIPcz5wYaOdPhtO8ATvZ+Xqp3BTdaqtAwQkNmKORqcIo5YkQqGXq5cwfswDwMqqQNrpJA==",
"license": "MIT",
"dependencies": {
"real-require": "^0.2.0"
},
"engines": {
"node": ">=20"
}
},
"node_modules/tinybench": {
"version": "2.9.0",
"resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz",

3
gitnexus/package.json
View file

@ -44,6 +44,7 @@
"dev": "tsx watch src/cli/index.ts",
"test": "vitest run",
"test:unit": "vitest run test/unit",
"pretest:integration": "node scripts/build.js",
"test:integration": "vitest run test/integration",
"test:watch": "vitest",
"test:coverage": "vitest run --coverage",
@ -72,6 +73,8 @@
"mnemonist": "^0.40.3",
"onnxruntime-node": "^1.24.0",
"pandemonium": "^2.4.0",
"pino": "^10.3.1",
"pino-pretty": "^13.1.3",
"tree-sitter": "^0.21.1",
"tree-sitter-c": "0.21.4",
"tree-sitter-c-sharp": "0.23.1",

11
gitnexus/scripts/build-tree-sitter-dart.cjs
View file

@ -3,6 +3,17 @@ const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
// Opt-out: skip the native rebuild entirely. Dart parsing becomes
// unavailable but `npm install gitnexus` finishes much faster on machines
// without a C++ toolchain. Strict `=== '1'` only — '=true', '=yes', '=0'
// (read as a string), and any other value all fall through to the rebuild.
if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') {
console.warn(
'[tree-sitter-dart] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Dart parsing will be unavailable until reinstalled without the env var.',
);
process.exit(0);
}
const dartDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-dart');
const bindingGyp = path.join(dartDir, 'binding.gyp');
const bindingNode = path.join(dartDir, 'build', 'Release', 'tree_sitter_dart_binding.node');

11
gitnexus/scripts/build-tree-sitter-proto.cjs
View file

@ -34,6 +34,17 @@ const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
// Opt-out: skip the native rebuild entirely. Proto parsing becomes
// unavailable but `npm install gitnexus` finishes much faster on machines
// without a C++ toolchain. Strict `=== '1'` only — '=true', '=yes', '=0'
// (read as a string), and any other value all fall through to the rebuild.
if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') {
console.warn(
'[tree-sitter-proto] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Proto parsing will be unavailable until reinstalled without the env var.',
);
process.exit(0);
}
const protoDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-proto');
const bindingGyp = path.join(protoDir, 'binding.gyp');
const bindingNode = path.join(protoDir, 'build', 'Release', 'tree_sitter_proto_binding.node');

3
gitnexus/src/cli/ai-context.ts
View file

@ -10,6 +10,7 @@ import fs from 'fs/promises';
import path from 'path';
import { fileURLToPath } from 'url';
import { type GeneratedSkillInfo } from './skill-gen.js';
import { logger } from '../core/logger.js';
// ESM equivalent of __dirname
const __filename = fileURLToPath(import.meta.url);
@ -293,7 +294,7 @@ Use GitNexus tools to accomplish this task.
installedSkills.push(skill.name);
} catch (err) {
// Skip on error, don't fail the whole process
console.warn(`Warning: Could not install skill ${skill.name}:`, err);
logger.warn({ err }, `Warning: Could not install skill ${skill.name}:`);
}
}

138
gitnexus/src/cli/analyze.ts
View file

@ -23,7 +23,11 @@ import {
import { getGitRoot, hasGitDir } from '../storage/git.js';
import { runFullAnalysis } from '../core/run-analyze.js';
import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js';
import { warnMissingOptionalGrammars } from './optional-grammars.js';
import { glob } from 'glob';
import fs from 'fs/promises';
import { cliError } from './cli-message.js';
import { isHfDownloadFailure } from '../core/embeddings/hf-env.js';
// Capture stderr.write at module load BEFORE anything (LadybugDB native
// init, progress bar, console redirection) can monkey-patch it. The
@ -165,7 +169,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
if (options?.workerTimeout) {
const workerTimeoutSeconds = Number(options.workerTimeout);
if (!Number.isFinite(workerTimeoutSeconds) || workerTimeoutSeconds < 1) {
console.error(' --worker-timeout must be at least 1 second.\n');
cliError(' --worker-timeout must be at least 1 second.\n');
process.exitCode = 1;
return;
}
@ -182,7 +186,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
if (typeof options?.embeddings === 'string') {
const parsed = Number(options.embeddings);
if (!Number.isInteger(parsed) || parsed < 0) {
console.error(
cliError(
` --embeddings expects a non-negative integer (got "${options.embeddings}"). ` +
`Pass 0 to disable the safety cap, or omit the value to keep the default.\n`,
);
@ -201,7 +205,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
if (value === undefined) return true;
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed <= 0) {
console.error(` ${optionName} must be a positive integer.\n`);
cliError(` ${optionName} must be a positive integer.\n`);
process.exitCode = 1;
return false;
}
@ -232,7 +236,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
if (options?.embeddingDevice) {
const allowed = new Set(['auto', 'cpu', 'dml', 'cuda', 'wasm']);
if (!allowed.has(options.embeddingDevice)) {
console.error(' --embedding-device must be one of: auto, cpu, dml, cuda, wasm.\n');
cliError(' --embedding-device must be one of: auto, cpu, dml, cuda, wasm.\n');
process.exitCode = 1;
return;
}
@ -273,6 +277,30 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
);
}
// If the target repo contains files an optional grammar would parse but
// that grammar's native binding is absent, warn before analysis so users
// learn why those files end up unparsed instead of silently getting a
// degraded index.
try {
const matches = await glob(['**/*.dart', '**/*.proto'], {
cwd: repoPath,
ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'],
dot: false,
nodir: true,
absolute: false,
});
if (matches.length > 0) {
const present = new Set<string>();
for (const m of matches) {
const ext = path.extname(m).toLowerCase();
if (ext) present.add(ext);
}
warnMissingOptionalGrammars({ context: 'analyze', relevantExtensions: present });
}
} catch {
// Best-effort warning \u2014 never block analyze on the precheck.
}
// KuzuDB migration cleanup is handled by runFullAnalysis internally.
// Note: --skills is handled after runFullAnalysis using the returned pipelineResult.
@ -304,7 +332,9 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
bar.start(100, 0, { phase: 'Initializing...' });
// Graceful SIGINT handling
// Graceful SIGINT handling. Pino's default destination is `sync: false`
// (buffered) — flush before exit so in-flight records reach stderr.
// See `gitnexus/src/core/logger.ts:flushLoggerSync`.
let aborted = false;
const sigintHandler = () => {
if (aborted) process.exit(1);
@ -313,13 +343,23 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
console.log('\n Interrupted — cleaning up...');
closeLbug()
.catch(() => {})
.finally(() => process.exit(130));
.finally(async () => {
const { flushLoggerSync } = await import('../core/logger.js');
flushLoggerSync();
process.exit(130);
});
};
process.on('SIGINT', sigintHandler);
// Route console output through bar.log() to prevent progress bar corruption
// Route console output through bar.log() to prevent progress bar corruption.
// This is a deliberate UI pattern (not a logging concern): analyze runs a
// long-lived progress bar on stdout; any concurrent console.* write would
// overwrite the bar mid-render. We capture originals, swap to barLog for
// the lifetime of the run, and restore on completion/error/SIGINT.
const origLog = console.log.bind(console);
// eslint-disable-next-line no-console -- intentional console-routing for progress bar UX
const origWarn = console.warn.bind(console);
// eslint-disable-next-line no-console -- intentional console-routing for progress bar UX
const origError = console.error.bind(console);
let barCurrentValue = 0;
const barLog = (...args: any[]) => {
@ -328,7 +368,9 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
bar.update(barCurrentValue);
};
console.log = barLog;
// eslint-disable-next-line no-console -- intentional console-routing for progress bar UX
console.warn = barLog;
// eslint-disable-next-line no-console -- intentional console-routing for progress bar UX
console.error = barLog;
// Track elapsed time per phase
@ -394,7 +436,9 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
clearInterval(elapsedTimer);
process.removeListener('SIGINT', sigintHandler);
console.log = origLog;
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
console.warn = origWarn;
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
console.error = origError;
bar.stop();
console.log(' Already up to date\n');
@ -467,7 +511,9 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
process.removeListener('SIGINT', sigintHandler);
console.log = origLog;
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
console.warn = origWarn;
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
console.error = origError;
bar.update(100, { phase: 'Done' });
@ -492,7 +538,9 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
clearInterval(elapsedTimer);
process.removeListener('SIGINT', sigintHandler);
console.log = origLog;
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
console.warn = origWarn;
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
console.error = origError;
bar.stop();
@ -501,14 +549,14 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
// Registry name-collision from --name (#829) — surface as an
// actionable error rather than a generic stack-trace.
if (err instanceof RegistryNameCollisionError) {
console.error(`\n Registry name collision:\n`);
console.error(` "${err.registryName}" is already used by "${err.existingPath}".\n`);
console.error(` Options:`);
console.error(` • Pick a different alias: gitnexus analyze --name <alias>`);
console.error(
` • Allow the duplicate: gitnexus analyze --allow-duplicate-name (leaves "-r ${err.registryName}" ambiguous)`,
cliError(
`\n Registry name collision:\n` +
` "${err.registryName}" is already used by "${err.existingPath}".\n\n` +
` Options:\n` +
` • Pick a different alias: gitnexus analyze --name <alias>\n` +
` • Allow the duplicate: gitnexus analyze --allow-duplicate-name (leaves "-r ${err.registryName}" ambiguous)\n`,
{ registryName: err.registryName, existingPath: err.existingPath },
);
console.error('');
process.exitCode = 1;
return;
}
@ -529,6 +577,26 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
return;
}
// HF download failure — show clean guidance without the raw stack trace.
// Checked before writeFatalToStderr so the user sees one focused message
// rather than a stack-trace dump followed by a second remediation block.
if (isHfDownloadFailure(msg) || msg.includes('Failed to download embedding model')) {
cliError(
` The embedding model could not be downloaded.\n` +
` huggingface.co may be unreachable from your network\n` +
` (e.g. behind a corporate proxy or a regional firewall).\n` +
` Suggestions:\n` +
` 1. Set HF_ENDPOINT to a mirror and retry:\n` +
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)\n` +
` 2. Check your proxy / VPN settings.\n` +
` 3. Once downloaded the model is cached — future runs work offline.\n`,
{ recoveryHint: 'hf-endpoint-unreachable' },
);
process.exitCode = 1;
return;
}
// Bypass the redirected console.error and write the full stack to
// the real stderr captured at module load. The redirected
// console.error wraps every line with `\\x1b[2K\\r` (ANSI clear-line)
@ -548,34 +616,40 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
msg.includes('heap out of memory') ||
msg.includes('JavaScript heap')
) {
console.error(' This error typically occurs on very large repositories.');
console.error(' Suggestions:');
console.error(' 1. Add large vendored/generated directories to .gitnexusignore');
console.error(' 2. Increase Node.js heap: NODE_OPTIONS="--max-old-space-size=16384"');
console.error(' 3. Increase stack size: NODE_OPTIONS="--stack-size=4096"');
console.error('');
cliError(
` This error typically occurs on very large repositories.\n` +
` Suggestions:\n` +
` 1. Add large vendored/generated directories to .gitnexusignore\n` +
` 2. Increase Node.js heap: NODE_OPTIONS="--max-old-space-size=16384"\n` +
` 3. Increase stack size: NODE_OPTIONS="--stack-size=4096"\n`,
{ recoveryHint: 'large-repo' },
);
} else if (msg.includes('ERESOLVE') || msg.includes('Could not resolve dependency')) {
// Note: the original arborist "Cannot destructure property 'package' of
// 'node.target'" crash happens inside npm *before* gitnexus code runs,
// so it can't be caught here. This branch handles dependency-resolution
// errors that surface at runtime (e.g. dynamic require failures).
console.error(' This looks like an npm dependency resolution issue.');
console.error(' Suggestions:');
console.error(' 1. Clear the npm cache: npm cache clean --force');
console.error(' 2. Update npm: npm install -g npm@latest');
console.error(' 3. Reinstall gitnexus: npm install -g gitnexus@latest');
console.error(' 4. Or try npx directly: npx gitnexus@latest analyze');
console.error('');
cliError(
` This looks like an npm dependency resolution issue.\n` +
` Suggestions:\n` +
` 1. Clear the npm cache: npm cache clean --force\n` +
` 2. Update npm: npm install -g npm@latest\n` +
` 3. Reinstall gitnexus: npm install -g gitnexus@latest\n` +
` 4. Or try npx directly: npx gitnexus@latest analyze\n`,
{ recoveryHint: 'npm-resolution' },
);
} else if (
msg.includes('MODULE_NOT_FOUND') ||
msg.includes('Cannot find module') ||
msg.includes('ERR_MODULE_NOT_FOUND')
) {
console.error(' A required module could not be loaded. The installation may be corrupt.');
console.error(' Suggestions:');
console.error(' 1. Reinstall: npm install -g gitnexus@latest');
console.error(' 2. Clear cache: npm cache clean --force && npx gitnexus@latest analyze');
console.error('');
cliError(
` A required module could not be loaded. The installation may be corrupt.\n` +
` Suggestions:\n` +
` 1. Reinstall: npm install -g gitnexus@latest\n` +
` 2. Clear cache: npm cache clean --force && npx gitnexus@latest analyze\n`,
{ recoveryHint: 'module-not-found' },
);
}
process.exitCode = 1;

7
gitnexus/src/cli/clean.ts
View file

@ -6,6 +6,7 @@
*/
import fs from 'fs/promises';
import { logger } from '../core/logger.js';
import {
findRepo,
unregisterRepo,
@ -45,7 +46,7 @@ export const cleanCommand = async (options?: { force?: boolean; all?: boolean })
assertSafeStoragePath(entry);
} catch (err) {
if (err instanceof UnsafeStoragePathError) {
console.error(`Refusing to clean ${entry.name}: ${err.message}`);
logger.error(`Refusing to clean ${entry.name}: ${err.message}`);
continue;
}
throw err;
@ -56,7 +57,7 @@ export const cleanCommand = async (options?: { force?: boolean; all?: boolean })
await unregisterRepo(entry.path);
console.log(`Deleted: ${entry.name} (${entry.storagePath})`);
} catch (err) {
console.error(`Failed to delete ${entry.name}:`, err);
logger.error({ err }, `Failed to delete ${entry.name}:`);
}
}
return;
@ -85,6 +86,6 @@ export const cleanCommand = async (options?: { force?: boolean; all?: boolean })
await unregisterRepo(repo.repoPath);
console.log(`Deleted: ${repo.storagePath}`);
} catch (err) {
console.error('Failed to delete:', err);
logger.error({ err }, 'Failed to delete:');
}
};

65
gitnexus/src/cli/cli-message.ts Normal file
View file

@ -0,0 +1,65 @@
/**
* CLI message helpers — for user-facing banners, error guidance, and
* recovery hints emitted by `gitnexus` subcommands.
*
* These functions write **plain text** directly to `process.stderr` AND
* tee a structured pino record through the singleton `logger`. Plain text
* preserves the human-readable contract for users running `gitnexus`
* interactively, redirecting to a file, or piping to `cat`/`grep`. The
* structured tee keeps log aggregators happy.
*
* **Use these for:**
* - User-facing banners ("Server listening on http://...:N")
* - Validation errors ("--worker-timeout must be at least 1 second")
* - Recovery hints ("Suggestions: 1. Clear the npm cache, 2. ...")
* - One-line user notices ("No indexed repositories found.")
*
* **Do NOT use these for:**
* - Internal diagnostics (worker progress, retry counts, telemetry)
* — use `logger.info`/`warn`/`error` directly. Internal logs only
* need structured fields, not double-output to stderr.
* - High-volume hot paths — every `cliMessage` call writes twice (raw
* + structured). Acceptable for user-facing messages, wasteful for
* ingestion pipeline events.
*
* Design note: stderr is the right channel even for non-error messages
* because GitNexus CLI tools (`query`, `cypher`, `impact`) emit JSON
* data on stdout for piping (`gitnexus query | jq`). User banners on
* stdout would corrupt that pipeline.
*/
import { logger } from '../core/logger.js';
function writeStderr(msg: string): void {
// Direct write — bypassing `console.*` so it cannot be intercepted by
// progress-bar redirection (see `cli/analyze.ts:barLog`) or other
// routing. The structured tee below still goes through the logger so
// log aggregation works either way.
process.stderr.write(msg.endsWith('\n') ? msg : msg + '\n');
}
/**
* User-facing informational message. Use for banners, listening URLs,
* and any message the user expects to read in plain text.
*/
export function cliInfo(msg: string, fields?: Record<string, unknown>): void {
writeStderr(msg);
logger.info(fields ?? {}, msg);
}
/**
* User-facing warning. Operator-actionable but non-fatal — `cliWarn`
* indicates the command can still proceed in some form.
*/
export function cliWarn(msg: string, fields?: Record<string, unknown>): void {
writeStderr(msg);
logger.warn(fields ?? {}, msg);
}
/**
* User-facing error. Indicates the command cannot proceed; usually
* paired with a non-zero exit code at the call site.
*/
export function cliError(msg: string, fields?: Record<string, unknown>): void {
writeStderr(msg);
logger.error(fields ?? {}, msg);
}

52
gitnexus/src/cli/eval-server.ts
View file

@ -27,6 +27,8 @@
import http from 'http';
import { writeSync } from 'node:fs';
import { LocalBackend } from '../mcp/local/local-backend.js';
import { logger } from '../core/logger.js';
import { cliInfo, cliWarn } from './cli-message.js';
export interface EvalServerOptions {
port?: string;
@ -332,13 +334,19 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise<vo
const ok = await backend.init();
if (!ok) {
console.error('GitNexus eval-server: No indexed repositories found. Run: gitnexus analyze');
// Operator-actionable but the server cannot start; warn-level so log
// aggregators don't trip error alerts on a configuration miss. Use
// cliWarn so the diagnostic reaches stderr synchronously before
// process.exit() — direct logger.warn would be lost to the buffered
// pino destination on hard exit (skips beforeExit flush).
cliWarn('GitNexus eval-server: No indexed repositories found. Run: gitnexus analyze');
process.exit(1);
}
const repos = await backend.listRepos();
console.error(
`GitNexus eval-server: ${repos.length} repo(s) loaded: ${repos.map((r) => r.name).join(', ')}`,
logger.info(
{ repoCount: repos.length, repos: repos.map((r) => r.name) },
'GitNexus eval-server: repos loaded',
);
let idleTimer: ReturnType<typeof setTimeout> | null = null;
@ -347,7 +355,7 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise<vo
if (idleTimeoutSec <= 0) return;
if (idleTimer) clearTimeout(idleTimer);
idleTimer = setTimeout(async () => {
console.error('GitNexus eval-server: Idle timeout reached, shutting down');
logger.info({ idleTimeoutSec }, 'GitNexus eval-server: idle timeout reached, shutting down');
await backend.disconnect();
process.exit(0);
}, idleTimeoutSec * 1000);
@ -419,16 +427,34 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise<vo
});
server.listen(port, '127.0.0.1', () => {
console.error(`GitNexus eval-server: listening on http://127.0.0.1:${port}`);
console.error(` POST /tool/query — search execution flows`);
console.error(` POST /tool/context — 360-degree symbol view`);
console.error(` POST /tool/impact — blast radius analysis`);
console.error(` POST /tool/cypher — raw Cypher query`);
console.error(` GET /health — health check`);
console.error(` POST /shutdown — graceful shutdown`);
// Plain-text banner for the human watching stderr; structured record
// for log aggregation (split into two so the user sees a real banner
// not `{"level":30,"msg":"...","port":4747,"endpoints":[...]}`).
const bannerLines = [
`GitNexus eval-server: listening on http://127.0.0.1:${port}`,
` POST /tool/query — search execution flows`,
` POST /tool/context — 360-degree symbol view`,
` POST /tool/impact — blast radius analysis`,
` POST /tool/cypher — raw Cypher query`,
` GET /health — health check`,
` POST /shutdown — graceful shutdown`,
];
if (idleTimeoutSec > 0) {
console.error(` Auto-shutdown after ${idleTimeoutSec}s idle`);
bannerLines.push(` Auto-shutdown after ${idleTimeoutSec}s idle`);
}
cliInfo(bannerLines.join('\n'), {
port,
host: '127.0.0.1',
idleTimeoutSec: idleTimeoutSec > 0 ? idleTimeoutSec : undefined,
endpoints: [
'POST /tool/query',
'POST /tool/context',
'POST /tool/impact',
'POST /tool/cypher',
'GET /health',
'POST /shutdown',
],
});
try {
// Use fd 1 directly — LadybugDB captures process.stdout (#324)
writeSync(1, `GITNEXUS_EVAL_SERVER_READY:${port}\n`);
@ -440,7 +466,7 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise<vo
resetIdleTimer();
const shutdown = async () => {
console.error('GitNexus eval-server: shutting down...');
logger.info('GitNexus eval-server: shutting down...');
await backend.disconnect();
server.close();
process.exit(0);

7
gitnexus/src/cli/group.ts
View file

@ -1,6 +1,7 @@
// gitnexus/src/cli/group.ts
import { createRequire } from 'node:module';
import type { Command } from 'commander';
import { logger } from '../core/logger.js';
const _require = createRequire(import.meta.url);
const yaml = _require('js-yaml') as typeof import('js-yaml');
@ -51,7 +52,7 @@ export function registerGroupCommands(program: Command): void {
const groupDir = getGroupDir(getDefaultGitnexusDir(), groupName);
const config = await loadGroupConfig(groupDir);
if (!(repoPath in config.repos)) {
console.error(`Repo path "${repoPath}" not found in group "${groupName}"`);
logger.error(`Repo path "${repoPath}" not found in group "${groupName}"`);
process.exitCode = 1;
return;
}
@ -239,7 +240,7 @@ export function registerGroupCommands(program: Command): void {
const raw = await backend.getGroupService().groupImpact(payload);
if (raw && typeof raw === 'object' && 'error' in raw) {
console.error(String((raw as { error: string }).error));
logger.error(String((raw as { error: string }).error));
process.exitCode = 1;
return;
}
@ -333,7 +334,7 @@ export function registerGroupCommands(program: Command): void {
});
if (raw && typeof raw === 'object' && 'error' in raw) {
console.error(String((raw as { error: string }).error));
logger.error(String((raw as { error: string }).error));
process.exitCode = 1;
return;
}

73
gitnexus/src/cli/mcp.ts
View file

@ -4,23 +4,61 @@
* Starts the MCP server in standalone mode.
* Loads all indexed repos from the global registry.
* No longer depends on cwd — works from any directory.
*
* IMPORTANT: this module's static-import closure is intentionally tiny
* (one chain: `mcp/stdio-context.js` → `mcp/stdio-capture.js`, which is a
* leaf with zero non-`node:` imports). All heavy backend modules
* (`startMCPServer`, `LocalBackend`, `warnMissingOptionalGrammars`) load
* via `await import(...)` AFTER `installGlobalStdoutSentinel()` runs.
*
* This closes the ESM-evaluation-order window where native init banners
* from `@ladybugdb/core` (or any future heavy import) could reach raw
* stdout before the sentinel exists. Codex's adversarial review on
* PR #1383 found that even with the sentinel-install call as the first
* statement of `mcpCommand`, ESM evaluates static imports of THIS module
* before the function body runs — so any native side effects during
* those imports happen before the sentinel can intercept them.
*
* If you find yourself adding a static `import` to this file, ask
* whether the imported module (or anything it transitively imports)
* touches `process.stdout` or loads a native binding at module init. If
* either is true, switch it to a dynamic `await import(...)` inside
* `mcpCommand` after the sentinel install. The regression test at
* `gitnexus/test/integration/mcp/import-closure.test.ts` enforces this.
*/
import { startMCPServer } from '../mcp/server.js';
import { LocalBackend } from '../mcp/local/local-backend.js';
import { installGlobalStdoutSentinel } from '../mcp/stdio-context.js';
export const mcpCommand = async () => {
// Prevent unhandled errors from crashing the MCP server process.
// LadybugDB lock conflicts and transient errors should degrade gracefully.
process.on('uncaughtException', (err) => {
console.error(`GitNexus MCP: uncaught exception — ${err.message}`);
// Process is in an undefined state after uncaughtException — exit after flushing
setTimeout(() => process.exit(1), 100);
});
process.on('unhandledRejection', (reason) => {
const msg = reason instanceof Error ? reason.message : String(reason);
console.error(`GitNexus MCP: unhandled rejection — ${msg}`);
});
// Install the global stdout sentinel as the very first thing — before
// ANY other module loads. The static-import closure above is leaf-only
// (stdio-context → stdio-capture, zero non-`node:` deps), so this is
// also the first chance any code in this process has to write to stdout.
installGlobalStdoutSentinel();
// uncaughtException/unhandledRejection handlers are owned by
// startMCPServer (gitnexus/src/mcp/server.ts) so the server's shutdown
// path runs cleanly with full stack traces. Registering duplicates here
// would only produce noisy double-logging on the same exception.
// Dynamically import heavy backend modules AND the pino logger AFTER
// the sentinel installs. The logger is dynamic-imported (rather than
// static) to preserve the leaf-only static-import closure documented at
// the top of this file — `core/logger.js` itself doesn't write to
// stdout at module init, but transitive deps (pino, pino-pretty, the
// worker-thread transport) could in theory, and the import-closure
// regression test enforces the leaf invariant.
const [{ startMCPServer }, { LocalBackend }, { logger }] = await Promise.all([
import('../mcp/server.js'),
import('../mcp/local/local-backend.js'),
import('../core/logger.js'),
]);
// Missing-optional-grammar warnings are intentionally NOT emitted here.
// `gitnexus analyze` already warns at index time, filtered by the repo's
// actual extensions, and a repo can only be served by MCP after analyze
// has run. Repeating an unconditional warning at every MCP startup is
// pure noise for users whose indexed repos don't use Dart/Proto.
// Initialize multi-repo backend from registry.
// The server starts even with 0 repos — tools call refreshRepos() lazily,
@ -30,12 +68,15 @@ export const mcpCommand = async () => {
const repos = await backend.listRepos();
if (repos.length === 0) {
console.error(
// Operator-actionable but the server still starts and serves; warn-level,
// not error. Tools will discover newly-analyzed repos via lazy refresh.
logger.warn(
'GitNexus: No indexed repos yet. Run `gitnexus analyze` in a git repo — the server will pick it up automatically.',
);
} else {
console.error(
`GitNexus: MCP server starting with ${repos.length} repo(s): ${repos.map((r) => r.name).join(', ')}`,
logger.info(
{ repoCount: repos.length, repos: repos.map((r) => r.name) },
'GitNexus: MCP server starting',
);
}

114
gitnexus/src/cli/optional-grammars.ts Normal file
View file

@ -0,0 +1,114 @@
/**
* Optional grammar availability check.
*
* tree-sitter-dart and tree-sitter-proto are optionalDependencies that
* require a `node-gyp rebuild` at install time. The build can be skipped
* via GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (postinstall scripts), or it can
* silently soft-fail when the C++ toolchain is missing.
*
* Either path produces the same observable: the .node binding is absent
* at runtime. This helper detects that condition and surfaces a single
* stderr line per missing grammar so users learn why .dart/.proto support
* is unavailable instead of silently getting a degraded index.
*/
import { createRequire } from 'module';
import { cliWarn } from './cli-message.js';
const _require = createRequire(import.meta.url);
interface OptionalGrammar {
/** Display name in warnings */
name: string;
/** Module name to require.resolve */
pkg: string;
/** File extensions this grammar parses */
extensions: string[];
}
const OPTIONAL_GRAMMARS: OptionalGrammar[] = [
{ name: 'tree-sitter-dart', pkg: 'tree-sitter-dart', extensions: ['.dart'] },
{ name: 'tree-sitter-proto', pkg: 'tree-sitter-proto', extensions: ['.proto'] },
];
export interface MissingGrammar {
name: string;
extensions: string[];
}
/**
* Returns the list of optional grammars whose native binding cannot be
* loaded. Actually `require()`s the package — `require.resolve` would
* locate the entry path even when the `.node` binding is absent (the
* `file:` package directory is installed regardless of postinstall
* outcome), giving false negatives for the exact users we want to warn:
* those who installed with `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` or whose
* native rebuild soft-failed for missing toolchain.
*
* Node's module cache memoizes `require()` for us — calling this multiple
* times is cheap. The catch distinguishes "missing" (MODULE_NOT_FOUND or
* the typical node-gyp-build "could not find any binding" pattern) from
* "broken" (SyntaxError, EACCES, native crash). Broken bindings surface a
* separate stderr line so users get an actionable message instead of a
* misleading "reinstall" hint.
*/
export function detectMissingOptionalGrammars(): MissingGrammar[] {
const missing: MissingGrammar[] = [];
for (const g of OPTIONAL_GRAMMARS) {
try {
_require(g.pkg);
} catch (err) {
const code = (err as NodeJS.ErrnoException | undefined)?.code;
const msg = err instanceof Error ? err.message : String(err);
const looksMissing =
code === 'MODULE_NOT_FOUND' ||
code === 'ERR_MODULE_NOT_FOUND' ||
/could not find|no native build|prebuilds/i.test(msg);
if (!looksMissing) {
// Present but broken — surface so the user doesn't get a misleading
// "reinstall" recovery message that wouldn't actually help. cliWarn
// writes plain text to stderr AND tees a structured logger.warn
// record; the merged repo-wide ESLint pino-migration rule forbids
// direct `console.error` in CLI code (only `console.log` is allowed
// there for tool-data stdout output).
cliWarn(
`GitNexus: optional grammar "${g.name}" is installed but failed to load (${msg.slice(0, 200)}). ${g.extensions.join('/')} files will not be parsed.`,
{ grammar: g.name, extensions: g.extensions, error: msg },
);
}
missing.push({ name: g.name, extensions: g.extensions });
}
}
return missing;
}
/**
* Log a one-line stderr warning for each missing grammar. Safe to call
* unconditionally — silent if all grammars are present.
*
* `relevantExtensions`, if provided, filters the warning to grammars whose
* extensions appear in the set (e.g. an analyze run can pass the set of
* extensions actually present in the target repo so users without any
* .dart/.proto files don't see noise).
*/
export function warnMissingOptionalGrammars(opts?: {
context?: string;
relevantExtensions?: ReadonlySet<string>;
}): void {
const missing = detectMissingOptionalGrammars();
if (missing.length === 0) return;
const ctx = opts?.context ? ` [${opts.context}]` : '';
// Hoist the optional set into a local so the closure below can narrow
// its type; references to `opts?.relevantExtensions` inside `.some()`
// lose the outer null-check narrowing and require a non-null assertion.
const relevantExtensions = opts?.relevantExtensions;
for (const g of missing) {
if (relevantExtensions && !g.extensions.some((e) => relevantExtensions.has(e))) {
continue;
}
cliWarn(
`GitNexus${ctx}: optional grammar "${g.name}" is unavailable — ${g.extensions.join('/')} files will not be parsed. Reinstall without GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (and ensure python3, make, g++) to enable.`,
{ grammar: g.name, extensions: g.extensions, context: opts?.context },
);
}
}

11
gitnexus/src/cli/remove.ts
View file

@ -27,6 +27,8 @@
*/
import fs from 'fs/promises';
import { logger } from '../core/logger.js';
import { cliError } from './cli-message.js';
import {
readRegistry,
resolveRegistryEntry,
@ -51,14 +53,14 @@ export const removeCommand = async (target: string, options?: { force?: boolean
// Idempotent: missing target is a no-op warning, not an error.
// The `availableNames` hint comes from the error itself so users
// can see what they might have meant.
console.warn(`Nothing to remove: ${err.message}`);
logger.warn(`Nothing to remove: ${err.message}`);
return;
}
if (err instanceof RegistryAmbiguousTargetError) {
// Duplicate aliases are allowed via --allow-duplicate-name (#829);
// refuse to guess which one the user meant — surface the full list
// and exit non-zero so scripts don't silently pick the wrong repo.
console.error(`Error: ${err.message}`);
cliError(`Error: ${err.message}`);
process.exit(1);
}
throw err;
@ -86,7 +88,7 @@ export const removeCommand = async (target: string, options?: { force?: boolean
assertSafeStoragePath(entry);
} catch (err) {
if (err instanceof UnsafeStoragePathError) {
console.error(`Error: ${err.message}`);
cliError(`Error: ${err.message}`);
process.exit(1);
}
throw err;
@ -104,7 +106,8 @@ export const removeCommand = async (target: string, options?: { force?: boolean
console.log(` Path: ${entry.path}`);
console.log(` Storage: ${entry.storagePath}`);
} catch (err) {
console.error(`Failed to remove ${entry.name}:`, err);
const msg = err instanceof Error ? err.message : String(err);
cliError(`Failed to remove ${entry.name}: ${msg}`, { err });
process.exit(1);
}
};

46
gitnexus/src/cli/serve.ts
View file

@ -1,14 +1,26 @@
import { createServer } from '../server/api.js';
import { logger, flushLoggerSync } from '../core/logger.js';
import { cliError } from './cli-message.js';
// Catch anything that would cause a silent exit
// Catch anything that would cause a silent exit. Pino v10's default
// destination is `sync: false` (SonicBoom buffered) — call
// `flushLoggerSync()` between the log and `process.exit(1)` so the crash
// record is not lost to the unflushed buffer. Worker-thread transports
// (pino-pretty under TTY) handle their own flush on process exit in v10,
// so no separate `pino.final` integration is needed (the API was removed
// in v10 because the transport architecture made it unnecessary).
//
// We pass the Error itself in `{ err }` so pino's built-in err serializer
// captures `type`, `message`, and `stack` as structured fields.
process.on('uncaughtException', (err) => {
console.error('\n[gitnexus serve] Uncaught exception:', err.message);
if (process.env.DEBUG) console.error(err.stack);
logger.error({ err }, '[gitnexus serve] Uncaught exception');
flushLoggerSync();
process.exit(1);
});
process.on('unhandledRejection', (reason: any) => {
console.error('\n[gitnexus serve] Unhandled rejection:', reason?.message || reason);
if (process.env.DEBUG) console.error(reason?.stack);
process.on('unhandledRejection', (reason) => {
const err = reason instanceof Error ? reason : new Error(String(reason));
logger.error({ err }, '[gitnexus serve] Unhandled rejection');
flushLoggerSync();
process.exit(1);
});
@ -22,16 +34,26 @@ export const serveCommand = async (options?: { port?: string; host?: string }) =
try {
await createServer(port, host);
} catch (err: any) {
console.error(`\nFailed to start GitNexus server:\n`);
console.error(` ${err.message || err}\n`);
if (err.code === 'EADDRINUSE') {
console.error(` Port ${port} is already in use. Either:`);
console.error(` 1. Stop the other process using port ${port}`);
console.error(` 2. Use a different port: gitnexus serve --port 4748\n`);
cliError(
`\nFailed to start GitNexus server:\n` +
` ${err.message || err}\n\n` +
` Port ${port} is already in use. Either:\n` +
` 1. Stop the other process using port ${port}\n` +
` 2. Use a different port: gitnexus serve --port 4748\n`,
{ code: err.code, port, host },
);
} else {
cliError(`\nFailed to start GitNexus server:\n ${err.message || err}\n`, {
code: err.code,
port,
host,
});
}
if (err.stack && process.env.DEBUG) {
console.error(err.stack);
logger.debug({ stack: err.stack }, 'serve start error stack');
}
flushLoggerSync();
process.exit(1);
}
};

39
gitnexus/src/cli/setup.ts
View file

@ -10,6 +10,7 @@ import fs from 'fs/promises';
import path from 'path';
import os from 'os';
import { execFile, execFileSync } from 'child_process';
import { createRequire } from 'module';
import { promisify } from 'util';
import { fileURLToPath } from 'url';
import { glob } from 'glob';
@ -20,6 +21,21 @@ const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const execFileAsync = promisify(execFile);
// Pin the npx fallback to the installed version. Reason: setup.ts writes
// a config that persists in the user's editor and is invoked on every MCP
// connect. Pinning to the installed version means subsequent invocations
// skip the npm-registry metadata roundtrip (and stay reproducible until
// the user upgrades). Static configs and READMEs intentionally use
// `gitnexus@latest` since they're quickstart docs, not persisted state.
const _require = createRequire(import.meta.url);
const _pkg = _require('../../package.json') as { version?: unknown };
if (typeof _pkg.version !== 'string' || !_pkg.version) {
throw new Error(
'gitnexus/package.json#version is missing or not a string — cannot generate MCP fallback config.',
);
}
const NPX_REF = `gitnexus@${_pkg.version}`;
interface SetupResult {
configured: string[];
skipped: string[];
@ -62,8 +78,10 @@ function resolveGitnexusBin(): string | null {
* The MCP server entry for all editors.
*
* Prefers the globally-installed `gitnexus` binary (starts in ~1 s) over
* `npx -y gitnexus@latest` (cold-cache install of native deps can take
* >60 s, exceeding Claude Code's 30 s MCP connection timeout).
* `npx -y gitnexus@<version>` (cold-cache install of native deps can take
* >60 s, exceeding Claude Code's 30 s MCP connection timeout). The fallback
* version is read from gitnexus/package.json#version at module load so the
* persisted user config matches the installed package.
*
* Falls back to npx when the binary isn't on PATH — e.g. first-time
* users who ran `npx gitnexus analyze` but haven't done `npm i -g`.
@ -79,12 +97,12 @@ function getMcpEntry() {
if (process.platform === 'win32') {
return {
command: 'cmd',
args: ['/c', 'npx', '-y', 'gitnexus@latest', 'mcp'],
args: ['/c', 'npx', '-y', NPX_REF, 'mcp'],
};
}
return {
command: 'npx',
args: ['-y', 'gitnexus@latest', 'mcp'],
args: ['-y', NPX_REF, 'mcp'],
};
}
@ -100,9 +118,9 @@ function getOpenCodeMcpEntry() {
}
if (process.platform === 'win32') {
return { type: 'local', command: ['cmd', '/c', 'npx', '-y', 'gitnexus@latest', 'mcp'] };
return { type: 'local', command: ['cmd', '/c', 'npx', '-y', NPX_REF, 'mcp'] };
}
return { type: 'local', command: ['npx', '-y', 'gitnexus@latest', 'mcp'] };
return { type: 'local', command: ['npx', '-y', NPX_REF, 'mcp'] };
}
/**
@ -347,7 +365,12 @@ async function installClaudeCodeHooks(result: SetupResult): Promise<void> {
}
const hookPath = path.join(destHooksDir, 'gitnexus-hook.cjs').replace(/\\/g, '/');
const hookCmd = `node "${hookPath.replace(/"/g, '\\"')}"`;
// Escape backslashes FIRST, then quotes (CodeQL js/incomplete-sanitization).
// The previous shape `replace(/"/g, '\\"')` alone would let `path\with"quote`
// become `path\with\"quote`, where the trailing `\` before `"` could
// unescape the quote inside the surrounding double-quoted shell context.
const escapedHookPath = hookPath.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
const hookCmd = `node "${escapedHookPath}"`;
// Check which hook events need entries (idempotent: skip if already registered)
const parsed = await (async () => {
@ -604,7 +627,7 @@ async function installOpenCodeSkills(result: SetupResult): Promise<void> {
const installed = await installSkillsTo(skillsDir);
if (installed.length > 0) {
result.configured.push(
`OpenCode skills (${installed.length} skills → ~/.config/opencode/skill/)`,
`OpenCode skills (${installed.length} skills → ~/.config/opencode/skills/)`,
);
}
} catch (err: any) {

11
gitnexus/src/cli/tool.ts
View file

@ -17,6 +17,7 @@
import { writeSync } from 'node:fs';
import { LocalBackend } from '../mcp/local/local-backend.js';
import { cliError } from './cli-message.js';
let _backend: LocalBackend | null = null;
@ -25,7 +26,7 @@ async function getBackend(): Promise<LocalBackend> {
_backend = new LocalBackend();
const ok = await _backend.init();
if (!ok) {
console.error('GitNexus: No indexed repositories found. Run: gitnexus analyze');
cliError('GitNexus: No indexed repositories found. Run: gitnexus analyze');
process.exit(1);
}
return _backend;
@ -67,7 +68,7 @@ export async function queryCommand(
},
): Promise<void> {
if (!queryText?.trim()) {
console.error('Usage: gitnexus query <search_query>');
cliError('Usage: gitnexus query <search_query>');
process.exit(1);
}
@ -93,7 +94,7 @@ export async function contextCommand(
},
): Promise<void> {
if (!name?.trim() && !options?.uid) {
console.error('Usage: gitnexus context <symbol_name> [--uid <uid>] [--file <path>]');
cliError('Usage: gitnexus context <symbol_name> [--uid <uid>] [--file <path>]');
process.exit(1);
}
@ -118,7 +119,7 @@ export async function impactCommand(
},
): Promise<void> {
if (!target?.trim()) {
console.error('Usage: gitnexus impact <symbol_name> [--direction upstream|downstream]');
cliError('Usage: gitnexus impact <symbol_name> [--direction upstream|downstream]');
process.exit(1);
}
@ -153,7 +154,7 @@ export async function cypherCommand(
},
): Promise<void> {
if (!query?.trim()) {
console.error('Usage: gitnexus cypher <cypher_query>');
cliError('Usage: gitnexus cypher <cypher_query>');
process.exit(1);
}

48
gitnexus/src/cli/wiki.ts
View file

@ -19,6 +19,7 @@ import {
import { WikiGenerator, type WikiOptions } from '../core/wiki/generator.js';
import { resolveLLMConfig, type LLMProvider } from '../core/wiki/llm-client.js';
import { detectCursorCLI } from '../core/wiki/cursor-client.js';
import { logger } from '../core/logger.js';
export interface WikiCommandOptions {
force?: boolean;
@ -583,7 +584,7 @@ export const wikiCommand = async (inputPath?: string, options?: WikiCommandOptio
} else {
console.log(`\n Error: ${err.message}\n`);
if (process.env.GITNEXUS_VERBOSE) {
console.error(err);
logger.error({ err }, 'wiki command failed');
}
}
process.exitCode = 1;
@ -601,6 +602,38 @@ function hasGhCLI(): boolean {
}
}
/**
* Strict Gist URL predicate. Rejects:
* - any URL that does not parse (URL constructor throws)
* - schemes other than https (drops `http:`, `file:`, `gist:`-style spoofs)
* - hostnames that are not exactly `gist.github.com` (drops substring spoofs
* like `https://evil.com/?u=gist.github.com` and userinfo-prefixed shapes
* like `https://[email protected]/...` — note that URL.hostname
* strips userinfo, so the equality check rejects the userinfo-prefixed
* spoof if the actual host differs from gist.github.com)
* - any URL containing userinfo (`username[:password]@`), which the URL
* parser exposes via `.username` / `.password`. Defense-in-depth: even
* when hostname matches, a credential-bearing URL is suspect and not
* produced by `gh gist create`.
*
* Closes the substring-bypass class CodeQL `js/incomplete-url-substring-
* sanitization` flags.
*/
function isGistUrl(line: string): boolean {
const trimmed = line.trim();
try {
const u = new URL(trimmed);
return (
u.protocol === 'https:' &&
u.hostname === 'gist.github.com' &&
u.username === '' &&
u.password === ''
);
} catch {
return false;
}
}
function publishGist(htmlPath: string): { url: string; rawUrl: string } | null {
try {
const output = execFileSync(
@ -609,13 +642,14 @@ function publishGist(htmlPath: string): { url: string; rawUrl: string } | null {
{ encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] },
).trim();
// gh gist create prints the gist URL as the last line
const lines = output.split('\n');
const gistUrl = lines.find((l) => l.includes('gist.github.com')) || lines[lines.length - 1];
// `gh gist create` prints the gist URL as a line in the output. Find the
// first parseable Gist URL — if no line is a valid Gist URL, fail closed
// (do NOT fall back to lines[last]: a non-Gist last line would propagate
// through the regex below and produce a malformed `rawUrl`).
const gistUrl = output.split('\n').find(isGistUrl);
if (!gistUrl) return null;
if (!gistUrl || !gistUrl.includes('gist.github.com')) return null;
// Build a raw viewer URL via gist.githack.com
// Build a raw viewer URL via gist.githack.com.
// gist URL format: https://gist.github.com/{user}/{id}
const match = gistUrl.match(/gist\.github\.com\/([^/]+)\/([a-f0-9]+)/);
let rawUrl = gistUrl;

3
gitnexus/src/config/ignore-service.ts
View file

@ -2,6 +2,7 @@ import ignore, { type Ignore } from 'ignore';
import fs from 'fs/promises';
import nodePath from 'path';
import type { Path } from 'path-scurry';
import { logger } from '../core/logger.js';
const DEFAULT_IGNORE_LIST = new Set([
// Version Control
@ -365,7 +366,7 @@ export const loadIgnoreRules = async (
} catch (err: unknown) {
const code = (err as NodeJS.ErrnoException).code;
if (code !== 'ENOENT') {
console.warn(` Warning: could not read ${filename}: ${(err as Error).message}`);
logger.warn(` Warning: could not read ${filename}: ${(err as Error).message}`);
}
}
}

89
gitnexus/src/core/embeddings/embedder.ts
View file

@ -14,7 +14,12 @@ if (!process.env.ORT_LOG_LEVEL) {
process.env.ORT_LOG_LEVEL = '3';
}
import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers';
import {
pipeline,
env,
type FeatureExtractionPipeline,
type ProgressInfo,
} from '@huggingface/transformers';
import { existsSync } from 'fs';
import { execFileSync } from 'child_process';
import { join, dirname } from 'path';
@ -22,7 +27,8 @@ import { createRequire } from 'module';
import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types.js';
import { isHttpMode, getHttpDimensions, httpEmbed } from './http-client.js';
import { resolveEmbeddingConfig } from './config.js';
import { applyHfEnvOverrides } from './hf-env.js';
import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js';
import { logger } from '../logger.js';
/**
* Check whether the onnxruntime-node package that @huggingface/transformers
@ -166,17 +172,22 @@ export const initEmbedder = async (
const isDev = process.env.NODE_ENV === 'development';
if (isDev) {
console.log(`🧠 Loading embedding model: ${finalConfig.modelId}`);
logger.info(`🧠 Loading embedding model: ${finalConfig.modelId}`);
}
const progressCallback = onProgress
? (data: any) => {
? (data: ProgressInfo) => {
const progress: ModelProgress = {
status: data.status || 'progress',
file: data.file,
progress: data.progress,
loaded: data.loaded,
total: data.total,
// Map the `progress_total` aggregate event (not in ModelProgress.status)
// back to 'progress' so callers don't need to handle it separately.
status:
data.status === 'progress_total'
? 'progress'
: ((data.status as ModelProgress['status']) ?? 'progress'),
file: 'file' in data ? data.file : undefined,
progress: 'progress' in data ? data.progress : undefined,
loaded: 'loaded' in data ? data.loaded : undefined,
total: 'total' in data ? data.total : undefined,
};
onProgress(progress);
}
@ -192,26 +203,38 @@ export const initEmbedder = async (
for (const device of devicesToTry) {
try {
if (isDev && device === 'dml') {
console.log('🔧 Trying DirectML (DirectX12) GPU backend...');
logger.info('🔧 Trying DirectML (DirectX12) GPU backend...');
} else if (isDev && device === 'cuda') {
console.log('🔧 Trying CUDA GPU backend...');
logger.info('🔧 Trying CUDA GPU backend...');
} else if (isDev && device === 'cpu') {
console.log('🔧 Using CPU backend...');
logger.info('🔧 Using CPU backend...');
} else if (isDev && device === 'wasm') {
console.log('🔧 Using WASM backend (slower)...');
logger.info('🔧 Using WASM backend (slower)...');
}
embedderInstance = await (pipeline as any)('feature-extraction', finalConfig.modelId, {
device: device,
dtype: 'fp32',
progress_callback: progressCallback,
session_options: {
logSeverityLevel: 3,
intraOpNumThreads: finalConfig.threads,
interOpNumThreads: 1,
executionMode: 'sequential',
embedderInstance = await withHfDownloadRetry(
() =>
pipeline('feature-extraction', finalConfig.modelId, {
device: device,
dtype: 'fp32',
progress_callback: progressCallback,
session_options: {
logSeverityLevel: 3,
intraOpNumThreads: finalConfig.threads,
interOpNumThreads: 1,
executionMode: 'sequential',
},
}),
{
onRetry: isDev
? (attempt, max, err) =>
logger.warn(
{ attempt, max, err: err.message },
`⚠️ Model download network error (attempt ${attempt}/${max}), retrying…`,
)
: undefined,
},
});
);
currentDevice = device;
if (isDev) {
@ -221,15 +244,29 @@ export const initEmbedder = async (
: device === 'cuda'
? 'GPU (CUDA)'
: device.toUpperCase();
console.log(`✅ Using ${label} backend`);
console.log('✅ Embedding model loaded successfully');
logger.info(`✅ Using ${label} backend`);
logger.info('✅ Embedding model loaded successfully');
}
return embedderInstance!;
} catch (deviceError) {
// Network errors and circuit-open errors are not device-specific —
// they will fail the same way on every device. Rethrow immediately
// with actionable HF_ENDPOINT guidance rather than silently falling
// back to the next device.
const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError);
if (isHfDownloadFailure(errMsg)) {
const endpointHint = process.env.HF_ENDPOINT
? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.`
: `huggingface.co may be unreachable from your network.\n` +
` Set HF_ENDPOINT to a mirror and retry:\n` +
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`;
throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`);
}
if (isDev && (device === 'cuda' || device === 'dml')) {
const gpuType = device === 'dml' ? 'DirectML' : 'CUDA';
console.log(`⚠️ ${gpuType} not available, falling back to CPU...`);
logger.info(`⚠️ ${gpuType} not available, falling back to CPU...`);
}
// Continue to next device in list
if (device === devicesToTry[devicesToTry.length - 1]) {

31
gitnexus/src/core/embeddings/embedding-pipeline.ts
View file

@ -44,6 +44,7 @@ import {
} from '../lbug/schema.js';
import { loadVectorExtension } from '../lbug/lbug-adapter.js';
import { getExactScanLimit } from '../platform/capabilities.js';
import { logger } from '../logger.js';
const isDev = process.env.NODE_ENV === 'development';
@ -157,7 +158,7 @@ const queryEmbeddableNodes = async (
}
} catch (error) {
if (isDev) {
console.warn(`Query for ${label} nodes failed:`, error);
logger.warn({ error }, `Query for ${label} nodes failed:`);
}
}
}
@ -212,7 +213,7 @@ const createVectorIndex = async (
return true;
} catch (error) {
if (isDev) {
console.warn('Vector index creation warning:', error);
logger.warn({ error }, 'Vector index creation warning:');
}
return false;
}
@ -256,7 +257,9 @@ export const runEmbeddingPipeline = async (
try {
const vectorAvailable = await ensureVectorExtensionAvailable();
if (!vectorAvailable && isDev) console.warn(vectorUnavailableMessage);
if (!vectorAvailable && isDev) {
logger.warn(vectorUnavailableMessage);
}
// Phase 1: Load embedding model
onProgress({
@ -283,7 +286,7 @@ export const runEmbeddingPipeline = async (
});
if (isDev) {
console.log('🔍 Querying embeddable nodes...');
logger.info('🔍 Querying embeddable nodes...');
}
// Phase 2: Query embeddable nodes
@ -325,7 +328,7 @@ export const runEmbeddingPipeline = async (
// (Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the sanctioned pattern)
if (staleNodeIds.length > 0) {
if (isDev) {
console.log(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`);
logger.info(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`);
}
try {
await executeWithReusedStatement(
@ -346,7 +349,7 @@ export const runEmbeddingPipeline = async (
}
if (isDev) {
console.log(
logger.info(
`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.length} stale, ${nodes.length} to embed`,
);
}
@ -355,7 +358,7 @@ export const runEmbeddingPipeline = async (
const totalNodes = nodes.length;
if (isDev) {
console.log(`📊 Found ${totalNodes} embeddable nodes`);
logger.info(`📊 Found ${totalNodes} embeddable nodes`);
}
if (totalNodes === 0) {
@ -442,9 +445,9 @@ export const runEmbeddingPipeline = async (
);
} catch (chunkErr) {
if (isDev) {
console.warn(
logger.warn(
{ chunkErr },
`⚠️ AST chunking failed for ${node.label} "${node.name}" (${node.filePath}), falling back to character-based chunking:`,
chunkErr,
);
}
chunks = characterChunk(node.content, startLine, endLine, chunkSize, overlap);
@ -482,9 +485,9 @@ export const runEmbeddingPipeline = async (
try {
embeddings = await embedBatch(subTexts);
} catch (embedErr) {
console.error(
logger.error(
{ embedErr },
`❌ embedBatch failed for ${subTexts.length} texts (first: "${subTexts[0]?.substring(0, 80)}..."):`,
embedErr,
);
throw embedErr;
}
@ -520,7 +523,7 @@ export const runEmbeddingPipeline = async (
});
if (isDev) {
console.log('📇 Creating vector index...');
logger.info('📇 Creating vector index...');
}
const vectorIndexReady = await createVectorIndex(executeQuery);
@ -533,7 +536,7 @@ export const runEmbeddingPipeline = async (
});
if (isDev) {
console.log(
logger.info(
`✅ Embedding pipeline complete! (${totalChunks} chunks from ${totalNodes} nodes)`,
);
}
@ -547,7 +550,7 @@ export const runEmbeddingPipeline = async (
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
if (isDev) {
console.error('❌ Embedding pipeline error:', error);
logger.error({ error }, '❌ Embedding pipeline error:');
}
onProgress({

281
gitnexus/src/core/embeddings/hf-env.ts
View file

@ -1,6 +1,25 @@
import os from 'node:os';
import { join } from 'node:path';
// ---------------------------------------------------------------------------
// Download resilience defaults
// ---------------------------------------------------------------------------
/** Per-attempt timeout for the full model download (5 minutes). */
export const HF_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1_000;
/** Maximum total download attempts (1 initial + N-1 retries). */
export const HF_MAX_ATTEMPTS = 3;
/** Initial delay between retry attempts; doubles on each subsequent retry. */
export const HF_BASE_DELAY_MS = 2_000;
/** Number of consecutive failures required to open the circuit. */
export const CB_FAILURE_THRESHOLD = 3;
/** How long the circuit stays open before transitioning to half-open. */
export const CB_RESET_TIMEOUT_MS = 60_000;
/** Upper bound clamped on the env-override per-attempt timeout (30 minutes). */
export const HF_MAX_TIMEOUT_MS = 30 * 60 * 1_000;
/** Upper bound clamped on the env-override attempt count. */
export const HF_MAX_ATTEMPTS_CAP = 10;
/**
* @internal Exported only for unit tests and the two embedder entry points
* (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
@ -60,3 +79,265 @@ export function applyHfEnvOverrides(env: HfEnvSubset): void {
env.remoteHost = endpoint.endsWith('/') ? endpoint : endpoint + '/';
}
}
/**
* @internal Exported for unit tests and the two embedder entry points.
*
* Returns true when an error message indicates a network-level fetch failure
* during HuggingFace model download (e.g. `TypeError: fetch failed`,
* `ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, `ECONNRESET`).
*
* These errors are not device-specific and cannot be fixed by falling back to
* a different ONNX device — the caller should rethrow immediately with
* guidance about `HF_ENDPOINT`.
*/
export function isNetworkFetchError(message: string): boolean {
return (
message.includes('fetch failed') ||
message.includes('ECONNREFUSED') ||
message.includes('ENOTFOUND') ||
message.includes('ETIMEDOUT') ||
message.includes('ECONNRESET')
);
}
// ---------------------------------------------------------------------------
// Circuit breaker
// ---------------------------------------------------------------------------
/** @internal Used by `withHfDownloadRetry` to mark a circuit-open rejection. */
export const CIRCUIT_OPEN_TAG = 'hf-circuit-open';
/** Circuit-breaker states. */
type CircuitState = 'closed' | 'open' | 'half-open';
/**
* Circuit breaker for HuggingFace model downloads.
*
* After `failureThreshold` consecutive network failures the circuit opens and
* all subsequent calls to `withHfDownloadRetry` fail immediately without
* issuing any network requests. After `resetTimeoutMs` the circuit enters the
* half-open state and the next call is attempted — if it succeeds the circuit
* closes again; if it fails the circuit re-opens.
*
* Exported for unit-testing; production code should use the module-level
* `hfDownloadCircuit` singleton.
*/
export class HfDownloadCircuitBreaker {
private _state: CircuitState = 'closed';
private _failures = 0;
/** Timestamp of the last recorded failure (ms since epoch). */
lastFailureAt = 0;
constructor(
readonly failureThreshold: number = CB_FAILURE_THRESHOLD,
readonly resetTimeoutMs: number = CB_RESET_TIMEOUT_MS,
) {}
/** Effective state, factoring in the reset-timeout transition. */
get state(): CircuitState {
if (this._state === 'open' && Date.now() - this.lastFailureAt > this.resetTimeoutMs) {
this._state = 'half-open';
}
return this._state;
}
/** Returns true when the circuit is open and calls should be rejected. */
isOpen(): boolean {
return this.state === 'open';
}
/** Record a successful call — resets the failure counter and closes the circuit. */
recordSuccess(): void {
this._failures = 0;
this._state = 'closed';
}
/** Record a failed call — increments the counter and opens the circuit when the threshold is reached. */
recordFailure(): void {
this._failures++;
this.lastFailureAt = Date.now();
if (this._failures >= this.failureThreshold) {
this._state = 'open';
}
}
/** @internal Reset to initial state (used in tests). */
reset(): void {
this._failures = 0;
this._state = 'closed';
this.lastFailureAt = 0;
}
}
/** Module-level singleton shared by both embedder entry points. */
export const hfDownloadCircuit = new HfDownloadCircuitBreaker();
// ---------------------------------------------------------------------------
// Retry + timeout wrapper
// ---------------------------------------------------------------------------
/** @internal Returns true for errors that should abort without retry (circuit-open). */
export function isHfCircuitOpenError(message: string): boolean {
return message.includes(CIRCUIT_OPEN_TAG);
}
/**
* Returns true for any HuggingFace download failure that warrants showing the
* `HF_ENDPOINT` remediation hint: either a raw network error or a
* circuit-open rejection (which itself was caused by repeated network errors).
*/
export function isHfDownloadFailure(message: string): boolean {
return isNetworkFetchError(message) || isHfCircuitOpenError(message);
}
/** @internal Wraps `fn` in a hard time-limit. The timeout error contains
* `ETIMEDOUT` so that `isNetworkFetchError` classifies it correctly.
*/
export function withDownloadTimeout<T>(fn: () => Promise<T>, timeoutMs: number): Promise<T> {
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(
() =>
reject(
new Error(
`ETIMEDOUT: model download timed out after ${Math.round(timeoutMs / 1000)}s — ` +
`check your network speed or set HF_ENDPOINT to a faster mirror`,
),
),
timeoutMs,
);
fn().then(
(v) => {
clearTimeout(timer);
resolve(v);
},
(e) => {
clearTimeout(timer);
reject(e);
},
);
});
}
/** @internal Async sleep (exposed for testing). */
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export interface HfRetryOptions {
/** Maximum total attempts including the initial one (default: `HF_MAX_ATTEMPTS`). */
maxAttempts?: number;
/** Delay before the first retry; doubles on each subsequent attempt (default: `HF_BASE_DELAY_MS`). */
baseDelayMs?: number;
/** Per-attempt wall-clock timeout in ms (default: `HF_DOWNLOAD_TIMEOUT_MS`). */
timeoutMs?: number;
/**
* Circuit-breaker instance to use. Defaults to the module-level
* `hfDownloadCircuit` singleton. Pass a fresh instance in tests.
*/
circuit?: HfDownloadCircuitBreaker;
/**
* Optional callback invoked before each retry (not the initial attempt).
* @param attempt - 1-based retry number
* @param max - total allowed attempts
* @param error - the error that triggered the retry
*/
onRetry?: (attempt: number, max: number, error: Error) => void;
}
/**
* Retry wrapper for HuggingFace model downloads with per-attempt timeout and
* circuit-breaker protection.
*
* Behaviour:
* - If the circuit is **open**, fails immediately with a `CIRCUIT_OPEN_TAG`
* message (so `isHfDownloadFailure` still returns true and the caller can
* show `HF_ENDPOINT` guidance).
* - Each attempt is wrapped in `withDownloadTimeout`.
* - On a network-level error (`isNetworkFetchError`) the attempt is retried
* with exponential back-off; non-network errors (e.g. ONNX device failure)
* are rethrown immediately without retry.
* - Every network failure is recorded on the circuit breaker; a success resets
* it.
* - After all attempts are exhausted, the last network error is rethrown
* so the existing `isNetworkFetchError` / `isHfDownloadFailure` guards in
* the calling code still fire.
*/
export async function withHfDownloadRetry<T>(
fn: () => Promise<T>,
options: HfRetryOptions = {},
): Promise<T> {
// Resolve effective values — explicit options take precedence over env vars,
// which take precedence over built-in defaults. This lets users lower the
// per-attempt timeout without rebuilding (e.g.
// HF_DOWNLOAD_TIMEOUT_MS=60000 npx gitnexus analyze --embeddings
// reduces the worst-case wait from 15 minutes to ~3 minutes).
//
// Upper bounds are clamped to prevent accidental runaway configuration:
// - timeoutMs is capped at HF_MAX_TIMEOUT_MS (30 min)
// - maxAttempts is floored (fractional values → integer) and capped at
// HF_MAX_ATTEMPTS_CAP (10). Values ≤ 0, NaN, or Infinity fall back to
// the built-in defaults.
const envTimeout = Number(process.env.HF_DOWNLOAD_TIMEOUT_MS);
const envMaxAttempts = Number(process.env.HF_MAX_ATTEMPTS);
const resolvedTimeout =
Number.isFinite(envTimeout) && envTimeout > 0
? Math.min(envTimeout, HF_MAX_TIMEOUT_MS)
: HF_DOWNLOAD_TIMEOUT_MS;
const resolvedMaxAttempts =
Number.isFinite(envMaxAttempts) && envMaxAttempts > 0
? Math.min(Math.floor(envMaxAttempts), HF_MAX_ATTEMPTS_CAP)
: HF_MAX_ATTEMPTS;
const {
maxAttempts = resolvedMaxAttempts,
baseDelayMs = HF_BASE_DELAY_MS,
timeoutMs = resolvedTimeout,
circuit = hfDownloadCircuit,
onRetry,
} = options;
if (circuit.isOpen()) {
const secsUntilReset = Math.ceil(
(circuit.resetTimeoutMs - (Date.now() - circuit.lastFailureAt)) / 1000,
);
throw new Error(
`${CIRCUIT_OPEN_TAG}: HuggingFace download circuit is open after repeated network failures` +
(secsUntilReset > 0 ? ` — will reset in ~${secsUntilReset}s` : ''),
);
}
let lastError: Error = new Error('unknown error');
for (let attempt = 0; attempt < maxAttempts; attempt++) {
try {
const result = await withDownloadTimeout(fn, timeoutMs);
circuit.recordSuccess();
return result;
} catch (err) {
lastError = err instanceof Error ? err : new Error(String(err));
if (!isNetworkFetchError(lastError.message)) {
// Non-network error (e.g. CUDA unavailable) — propagate without retry
throw lastError;
}
circuit.recordFailure();
if (circuit.isOpen()) {
// Circuit just tripped — fail fast, no more retries
throw new Error(
`${CIRCUIT_OPEN_TAG}: HuggingFace download circuit opened after ${circuit.failureThreshold} consecutive failures`,
);
}
if (attempt < maxAttempts - 1) {
const delay = baseDelayMs * Math.pow(2, attempt);
onRetry?.(attempt + 1, maxAttempts, lastError);
await sleep(delay);
}
}
}
// All retries exhausted — throw the last network error so isNetworkFetchError
// patterns in the calling code still match and surface HF_ENDPOINT guidance.
throw lastError;
}

38
gitnexus/src/core/git-staleness.ts
View file

@ -3,11 +3,14 @@
* Lives in core/ so application code does not depend on the MCP package layer.
*/
import { execFileSync } from 'node:child_process';
import { execFile, execFileSync } from 'node:child_process';
import { promisify } from 'node:util';
import path from 'path';
import { readRegistry, type RegistryEntry, type CwdMatch } from '../storage/repo-manager.js';
import { findGitRootByDotGit, getCurrentCommit, getRemoteUrl } from '../storage/git.js';
const execFileAsync = promisify(execFile);
export interface StalenessInfo {
isStale: boolean;
commitsBehind: number;
@ -41,6 +44,39 @@ export function checkStaleness(repoPath: string, lastCommit: string): StalenessI
}
}
/**
* Async variant of {@link checkStaleness} — spawns git as a child process
* instead of blocking the event loop. Used by `listRepos()` to check many
* repos in parallel (issue #1363: 200 repos × sync spawn ≈ 50 s).
*/
export async function checkStalenessAsync(
repoPath: string,
lastCommit: string,
): Promise<StalenessInfo> {
try {
// Note: promisified execFile captures stdout/stderr by default (no stdio option needed,
// unlike the sync variant which requires explicit stdio: ['pipe','pipe','pipe']).
const { stdout } = await execFileAsync('git', ['rev-list', '--count', `${lastCommit}..HEAD`], {
cwd: repoPath,
encoding: 'utf-8',
});
const commitsBehind = parseInt(stdout.trim(), 10) || 0;
if (commitsBehind > 0) {
return {
isStale: true,
commitsBehind,
hint: `⚠️ Index is ${commitsBehind} commit${commitsBehind > 1 ? 's' : ''} behind HEAD. Run analyze tool to update.`,
};
}
return { isStale: false, commitsBehind: 0 };
} catch {
return { isStale: false, commitsBehind: 0 };
}
}
/**
* Compare a sibling-clone HEAD against an indexed `lastCommit`. Returns
* `undefined` when the indexed commit is not reachable from the sibling

418
gitnexus/src/core/group/bridge-db.ts
View file

@ -1,6 +1,6 @@
import fsp from 'node:fs/promises';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { createHash, randomBytes } from 'node:crypto';
import lbug from '@ladybugdb/core';
import type { LbugValue } from '@ladybugdb/core';
import type { BridgeHandle, BridgeMeta, StoredContract, CrossLink, RepoSnapshot } from './types.js';
@ -11,6 +11,9 @@ import {
type LbugConnectionHandle,
} from '../lbug/lbug-config.js';
import { dedupeContracts, dedupeCrossLinks } from './normalization.js';
import { createLogger } from '../logger.js';
const bridgeLogger = createLogger('bridge-db', { debugEnvVar: 'GITNEXUS_DEBUG_BRIDGE' });
/**
* Sidecar files that LadybugDB creates next to a `bridge.lbug` file.
@ -24,7 +27,7 @@ import { dedupeContracts, dedupeCrossLinks } from './normalization.js';
* - `.shadow` — non-blocking concurrent checkpoint sidecar (added in
* LadybugDB 0.15.4); same pairing constraint as `.wal`.
*
* `bridge-db` writes to a `bridge.lbug.tmp` file and then atomically renames
* `bridge-db` writes to a `bridge.lbug.tmp.<random>` file and then atomically renames
* it into place. The rename only moves the main file; sidecars must be
* cleaned up explicitly or the next writer trips the database-id check.
*/
@ -276,8 +279,24 @@ export async function retryRename(src: string, dst: string, attempts = 3): Promi
export async function writeBridgeMeta(groupDir: string, meta: BridgeMeta): Promise<void> {
const target = path.join(groupDir, 'meta.json');
const tmp = `${target}.tmp.${Date.now()}`;
await fsp.writeFile(tmp, JSON.stringify(meta, null, 2), 'utf-8');
// Unpredictable suffix + O_EXCL via `'wx'` flag closes the symlink/
// pre-create attack window. The third argument `0o600` is the
// user-only mode mask — CodeQL's `js/insecure-temporary-file` query
// sources its verdict from the `mode` argument, NOT from `flags`:
// its `isSecureMode(mode)` predicate requires the low 6 bits to be
// zero (no group/world bits). Without an explicit mode the file is
// created with the process umask (typically 0o644 = group/world
// readable), which the query treats as the actual vulnerability.
// Both `'wx'` (runtime O_EXCL) AND `0o600` (CodeQL-credited mode)
// are needed: one closes the symlink race, the other closes the
// permissions exposure.
const tmp = `${target}.tmp.${randomBytes(8).toString('hex')}`;
const handle = await fsp.open(tmp, 'wx', 0o600);
try {
await handle.writeFile(JSON.stringify(meta, null, 2), 'utf-8');
} finally {
await handle.close();
}
// Use retryRename for consistency with writeBridge's atomic swap — on
// Windows a concurrent reader can cause EBUSY/EPERM even on a tiny
// meta.json, and we don't want meta write to be less robust than the
@ -346,7 +365,19 @@ export async function writeBridge(
const crossLinks = dedupeCrossLinks(input.crossLinks);
const finalPath = path.join(groupDir, 'bridge.lbug');
const tmpPath = path.join(groupDir, 'bridge.lbug.tmp');
// Stage the temp database inside a unique mkdtemp directory rather than
// a fixed `bridge.lbug.tmp` name. The previous shape was flagged by
// CodeQL js/insecure-temporary-file as a predictable path: a co-located
// attacker (or a parallel writeBridge call into the same group) could
// pre-create or symlink that path before this writer opens it. mkdtemp
// returns a directory whose suffix is filled with cryptographically
// random bytes, so the staging path is unguessable AND collision-free
// across parallel callers. We anchor the staging directory inside
// `groupDir` so the subsequent rename of `bridge.lbug` (and its
// `.wal` / `.shadow` sidecars) into place stays on the same filesystem
// and remains atomic — moving across `os.tmpdir()` could trip EXDEV.
const stagingDir = await fsp.mkdtemp(path.join(groupDir, 'bridge-tmp-'));
const tmpPath = path.join(stagingDir, 'bridge.lbug');
const bakPath = path.join(groupDir, 'bridge.lbug.bak');
const report: WriteBridgeReport = {
@ -366,42 +397,42 @@ export async function writeBridge(
}
};
// Clean up any leftover tmp main file AND its `.wal` / `.shadow` sidecars.
// LadybugDB 0.16.0 rejects opening a database whose sidecars belong to a
// different database instance (database-id check), so any stale sidecar
// from a crashed previous run will fail the next writeBridge.
await removeLbugFile(tmpPath);
// The mkdtemp staging directory above is freshly created with a unique
// random suffix, so there are no leftover `bridge.lbug.tmp` / `.wal` /
// `.shadow` sidecars from a previous crashed run to clean up here — the
// directory is empty by construction.
// 1. Create temp DB, insert all data.
//
// Everything after `openBridgeDb` must run inside a try/finally so that
// if ANY step before the explicit `closeBridgeDb` throws — schema
// creation, a contract insert loop that rethrows, a snapshot write, the
// cross-link loop, or anything else — the handle is still released. A
// leaked handle holds the native LadybugDB file lock on tmpPath, which
// (a) leaks a FD and (b) prevents the next writeBridge call from
// reusing the same tmp slot.
const handle = await openBridgeDb(tmpPath);
let handleClosed = false;
try {
await ensureBridgeSchema(handle);
// 1. Create temp DB, insert all data.
//
// Everything after `openBridgeDb` must run inside a try/finally so that
// if ANY step before the explicit `closeBridgeDb` throws — schema
// creation, a contract insert loop that rethrows, a snapshot write, the
// cross-link loop, or anything else — the handle is still released. A
// leaked handle holds the native LadybugDB file lock on tmpPath, which
// (a) leaks a FD and (b) prevents the next writeBridge call from
// reusing the same tmp slot.
const handle = await openBridgeDb(tmpPath);
let handleClosed = false;
try {
await ensureBridgeSchema(handle);
// Build the lookup index incrementally as contracts are inserted, so
// failed inserts are never in the index (and therefore never resolved
// by the cross-link loop below). This replaces a previous N+1 query
// pattern where each link made up to 6 DB round-trips to find its
// endpoints — see ContractLookupIndex.
const lookupIndex = createContractLookupIndex();
// Build the lookup index incrementally as contracts are inserted, so
// failed inserts are never in the index (and therefore never resolved
// by the cross-link loop below). This replaces a previous N+1 query
// pattern where each link made up to 6 DB round-trips to find its
// endpoints — see ContractLookupIndex.
const lookupIndex = createContractLookupIndex();
// Insert contracts — tolerate individual failures (e.g., a corrupt meta
// that can't be serialized). The whole sync must not fail because one
// contract is broken.
for (const c of contracts) {
const id = contractNodeId(c.repo, c.contractId, c.role, c.symbolRef.filePath);
try {
await queryBridge(
handle,
`CREATE (n:Contract {
// Insert contracts — tolerate individual failures (e.g., a corrupt meta
// that can't be serialized). The whole sync must not fail because one
// contract is broken.
for (const c of contracts) {
const id = contractNodeId(c.repo, c.contractId, c.role, c.symbolRef.filePath);
try {
await queryBridge(
handle,
`CREATE (n:Contract {
id: $id,
contractId: $contractId,
type: $type,
@ -414,91 +445,91 @@ export async function writeBridge(
confidence: $confidence,
meta: $meta
})`,
{
id,
contractId: c.contractId,
type: c.type,
role: c.role,
repo: c.repo,
service: c.service ?? '',
symbolUid: c.symbolUid,
filePath: c.symbolRef.filePath,
symbolName: c.symbolName,
confidence: c.confidence,
meta: JSON.stringify(c.meta),
},
);
report.contractsInserted++;
// Only index on successful insert — the cross-link loop must never
// resolve to a row that isn't actually in the DB.
indexContract(lookupIndex, c, id);
} catch (err) {
report.contractsFailed++;
recordError('contract', id, err);
{
id,
contractId: c.contractId,
type: c.type,
role: c.role,
repo: c.repo,
service: c.service ?? '',
symbolUid: c.symbolUid,
filePath: c.symbolRef.filePath,
symbolName: c.symbolName,
confidence: c.confidence,
meta: JSON.stringify(c.meta),
},
);
report.contractsInserted++;
// Only index on successful insert — the cross-link loop must never
// resolve to a row that isn't actually in the DB.
indexContract(lookupIndex, c, id);
} catch (err) {
report.contractsFailed++;
recordError('contract', id, err);
}
}
}
// Insert repo snapshots
for (const [repoId, snap] of Object.entries(input.repoSnapshots)) {
try {
await queryBridge(
handle,
`CREATE (s:RepoSnapshot {
// Insert repo snapshots
for (const [repoId, snap] of Object.entries(input.repoSnapshots)) {
try {
await queryBridge(
handle,
`CREATE (s:RepoSnapshot {
id: $id,
indexedAt: $indexedAt,
lastCommit: $lastCommit
})`,
{
id: repoId,
indexedAt: snap.indexedAt,
lastCommit: snap.lastCommit,
},
);
report.snapshotsInserted++;
} catch (err) {
report.snapshotsFailed++;
recordError('snapshot', repoId, err);
}
}
// Insert cross-links (tolerating missing nodes).
//
// `findContractNode` consults the in-memory lookup index built above,
// not the DB — that's an O(1) pure-function lookup per endpoint instead
// of the previous 2-3 DB queries. For M cross-links, the previous code
// issued up to 6M round-trips; this version issues zero.
//
// `link.contractId` may differ between the consumer and provider sides
// (e.g. wildcard consumer `grpc::Service/*` → method-level provider
// `grpc::Service/Method`) — that's why we resolve each endpoint
// independently via its own `(repo, role, symbolUid, filePath, symbolName)`
// tuple rather than matching on contractId.
for (const link of crossLinks) {
const linkId = `${link.from.repo}::${link.contractId}->${link.to.repo}::${link.contractId}`;
try {
const fromId = findContractNode(
lookupIndex,
link.from.repo,
'consumer',
link.from.symbolUid,
link.from.symbolRef.filePath,
link.from.symbolRef.name,
);
const toId = findContractNode(
lookupIndex,
link.to.repo,
'provider',
link.to.symbolUid,
link.to.symbolRef.filePath,
link.to.symbolRef.name,
);
if (!fromId || !toId) {
report.linksDroppedMissingNode++;
continue;
{
id: repoId,
indexedAt: snap.indexedAt,
lastCommit: snap.lastCommit,
},
);
report.snapshotsInserted++;
} catch (err) {
report.snapshotsFailed++;
recordError('snapshot', repoId, err);
}
await queryBridge(
handle,
`
}
// Insert cross-links (tolerating missing nodes).
//
// `findContractNode` consults the in-memory lookup index built above,
// not the DB — that's an O(1) pure-function lookup per endpoint instead
// of the previous 2-3 DB queries. For M cross-links, the previous code
// issued up to 6M round-trips; this version issues zero.
//
// `link.contractId` may differ between the consumer and provider sides
// (e.g. wildcard consumer `grpc::Service/*` → method-level provider
// `grpc::Service/Method`) — that's why we resolve each endpoint
// independently via its own `(repo, role, symbolUid, filePath, symbolName)`
// tuple rather than matching on contractId.
for (const link of crossLinks) {
const linkId = `${link.from.repo}::${link.contractId}->${link.to.repo}::${link.contractId}`;
try {
const fromId = findContractNode(
lookupIndex,
link.from.repo,
'consumer',
link.from.symbolUid,
link.from.symbolRef.filePath,
link.from.symbolRef.name,
);
const toId = findContractNode(
lookupIndex,
link.to.repo,
'provider',
link.to.symbolUid,
link.to.symbolRef.filePath,
link.to.symbolRef.name,
);
if (!fromId || !toId) {
report.linksDroppedMissingNode++;
continue;
}
await queryBridge(
handle,
`
MATCH (a:Contract), (b:Contract)
WHERE a.id = $fromId AND b.id = $toId
CREATE (a)-[:ContractLink {
@ -509,83 +540,93 @@ export async function writeBridge(
toRepo: $toRepo
}]->(b)
`,
{
fromId,
toId,
matchType: link.matchType,
confidence: link.confidence,
contractId: link.contractId,
fromRepo: link.from.repo,
toRepo: link.to.repo,
},
);
report.linksInserted++;
} catch (err) {
report.linksFailed++;
recordError('link', linkId, err);
{
fromId,
toId,
matchType: link.matchType,
confidence: link.confidence,
contractId: link.contractId,
fromRepo: link.from.repo,
toRepo: link.to.repo,
},
);
report.linksInserted++;
} catch (err) {
report.linksFailed++;
recordError('link', linkId, err);
}
}
// 2. Close temp DB (happy path). The finally block also calls
// closeBridgeDb if we threw above; `handleClosed` prevents a
// double-close on the native handle.
await closeBridgeDb(handle);
handleClosed = true;
} finally {
if (!handleClosed) {
await closeBridgeDb(handle).catch(() => {
/* ignore: cleanup path, best effort */
});
}
}
// 2. Close temp DB (happy path). The finally block also calls
// closeBridgeDb if we threw above; `handleClosed` prevents a
// double-close on the native handle.
await closeBridgeDb(handle);
handleClosed = true;
} finally {
if (!handleClosed) {
await closeBridgeDb(handle).catch(() => {
/* ignore: cleanup path, best effort */
});
// 3. Atomic swap: old→.bak, tmp→final, rm .bak
//
// The current database file (with its `.wal` / `.shadow` sidecars) is
// moved aside, then the freshly built tmp database takes its place.
// We move the sidecars together with the main file so the open below
// and any external readers see a consistent set; orphan sidecars from
// the tmp namespace are then removed because LadybugDB looks for them
// under the renamed-to base name and would reject mismatching IDs.
try {
await fsp.access(finalPath);
await retryRename(finalPath, bakPath);
for (const suffix of LBUG_SIDECAR_SUFFIXES) {
try {
await fsp.access(`${finalPath}${suffix}`);
await retryRename(`${finalPath}${suffix}`, `${bakPath}${suffix}`);
} catch {
/* sidecar absent — nothing to move */
}
}
} catch {
/* no existing db */
}
}
// 3. Atomic swap: old→.bak, tmp→final, rm .bak
//
// The current database file (with its `.wal` / `.shadow` sidecars) is
// moved aside, then the freshly built tmp database takes its place.
// We move the sidecars together with the main file so the open below
// and any external readers see a consistent set; orphan sidecars from
// the tmp namespace are then removed because LadybugDB looks for them
// under the renamed-to base name and would reject mismatching IDs.
try {
await fsp.access(finalPath);
await retryRename(finalPath, bakPath);
await retryRename(tmpPath, finalPath);
for (const suffix of LBUG_SIDECAR_SUFFIXES) {
// Rename — not delete — so the WAL (which may carry uncommitted-at-
// close-time pages on a graceful close, depending on
// `autoCheckpoint` / `checkpointThreshold`) and the `.shadow`
// checkpoint snapshot stay paired with the database file under its
// final name. LadybugDB 0.16.0's database-id check rejects an open
// when the sidecars belong to a different base name.
try {
await fsp.access(`${finalPath}${suffix}`);
await retryRename(`${finalPath}${suffix}`, `${bakPath}${suffix}`);
await fsp.access(`${tmpPath}${suffix}`);
await retryRename(`${tmpPath}${suffix}`, `${finalPath}${suffix}`);
} catch {
/* sidecar absent — nothing to move */
}
}
} catch {
/* no existing db */
}
await retryRename(tmpPath, finalPath);
for (const suffix of LBUG_SIDECAR_SUFFIXES) {
// Rename — not delete — so the WAL (which may carry uncommitted-at-
// close-time pages on a graceful close, depending on
// `autoCheckpoint` / `checkpointThreshold`) and the `.shadow`
// checkpoint snapshot stay paired with the database file under its
// final name. LadybugDB 0.16.0's database-id check rejects an open
// when the sidecars belong to a different base name.
try {
await fsp.access(`${tmpPath}${suffix}`);
await retryRename(`${tmpPath}${suffix}`, `${finalPath}${suffix}`);
} catch {
/* sidecar absent — nothing to move */
}
}
await removeLbugFile(bakPath);
await removeLbugFile(bakPath);
// 4. Write meta.json
await writeBridgeMeta(groupDir, {
version: BRIDGE_SCHEMA_VERSION,
generatedAt: new Date().toISOString(),
missingRepos: input.missingRepos,
});
// 4. Write meta.json
await writeBridgeMeta(groupDir, {
version: BRIDGE_SCHEMA_VERSION,
generatedAt: new Date().toISOString(),
missingRepos: input.missingRepos,
});
return report;
return report;
} finally {
// Always remove the mkdtemp staging directory. On the happy path the
// main file and sidecars have been renamed out of it, so it's empty;
// on any error path it may still contain a partial database — either
// way `recursive: true, force: true` removes it without surfacing
// "directory not empty" or ENOENT.
await fsp.rm(stagingDir, { recursive: true, force: true }).catch(() => {
/* best-effort cleanup */
});
}
}
/* ------------------------------------------------------------------ */
@ -681,14 +722,15 @@ export async function openBridgeDbReadOnly(groupDir: string): Promise<BridgeHand
await new Promise((r) => setTimeout(r, delay));
}
}
if (process.env.GITNEXUS_DEBUG_BRIDGE) {
console.warn(
`[bridge-db] openBridgeDbReadOnly(${groupDir}) gave up after ` +
`${LBUG_OPEN_RETRY_ATTEMPTS} attempts: ${
lastErr instanceof Error ? lastErr.message : String(lastErr)
}`,
);
}
// Pino's NDJSON serialization is structurally injection-resistant
// (CodeQL js/log-injection): groupDir and err.message are JSON-escaped
// by the serializer, so no manual CRLF / U+2028 / ANSI sanitization is
// needed. Demoted to debug — only fires when the bridge truly gave up
// after retries, and operators only need it at debug verbosity.
bridgeLogger.debug(
{ groupDir, err: lastErr, attempts: LBUG_OPEN_RETRY_ATTEMPTS },
'openBridgeDbReadOnly gave up',
);
return null;
}

120
gitnexus/src/core/group/cross-impact.ts
View file

@ -91,6 +91,25 @@ function clampCrossDepth(raw: unknown): { depth: number; warning?: string } {
return { depth: d };
}
/**
* Clamp the impact timeout to a sane bounded range. Callers can feed this
* via tool params, so an unclamped value lets a single request hold a
* timer slot for an arbitrarily long duration (CodeQL js/resource-
* exhaustion). 100ms lower bound preserves test-suite scenarios that
* exercise tight timeouts; 5min upper bound is well above any legitimate
* single-impact compute. Applied at the validate boundary so the
* downstream `deadline` (Date.now() + timeoutMs) and the local-leg
* `setTimeout` see the same clamped value — earlier shapes had a 1hr
* outer cap and a 5min inner clamp that disagreed.
*/
export const IMPACT_TIMEOUT_MIN_MS = 100;
export const IMPACT_TIMEOUT_MAX_MS = 5 * 60 * 1_000;
export function clampTimeout(timeoutMs: number): number {
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return IMPACT_TIMEOUT_MIN_MS;
return Math.min(IMPACT_TIMEOUT_MAX_MS, Math.max(IMPACT_TIMEOUT_MIN_MS, Math.trunc(timeoutMs)));
}
export function validateGroupImpactParams(params: Record<string, unknown>):
| {
ok: true;
@ -143,13 +162,19 @@ export function validateGroupImpactParams(params: Record<string, unknown>):
const service = normalizeServicePrefix(params.service);
const subgroup = typeof params.subgroup === 'string' ? params.subgroup : undefined;
let timeoutMs =
// Clamp at the validate boundary so the downstream `deadline` (line
// ~366) and `safeLocalImpact`'s `setTimeout` both see a single
// bounded value. Without this, the outer deadline budgeted Phase-2
// cross-repo fanout up to 1hr while only the inner setTimeout was
// capped to 5min — the two halves of CodeQL #184's mitigation
// disagreed.
const rawTimeoutMs =
typeof params.timeoutMs === 'number' && params.timeoutMs > 0
? params.timeoutMs
: typeof params.timeout === 'number' && params.timeout > 0
? params.timeout
: DEFAULT_LOCAL_IMPACT_TIMEOUT_MS;
if (timeoutMs > 3_600_000) timeoutMs = 3_600_000;
const timeoutMs = clampTimeout(rawTimeoutMs);
return {
ok: true,
@ -191,12 +216,13 @@ async function safeLocalImpact(
impactParams: Parameters<GroupToolPort['impact']>[1],
timeoutMs: number,
): Promise<{ value: unknown; timedOut: boolean }> {
const safeTimeoutMs = clampTimeout(timeoutMs);
let timer: ReturnType<typeof setTimeout> | undefined;
const impactP = port.impact(repo, impactParams).catch((err) => ({
error: err instanceof Error ? err.message : String(err),
}));
const timeoutP = new Promise<'timeout'>((resolve) => {
timer = setTimeout(() => resolve('timeout'), timeoutMs);
timer = setTimeout(() => resolve('timeout'), safeTimeoutMs);
});
const won = await Promise.race([
impactP.then((v) => ({ tag: 'impact' as const, v })),
@ -212,6 +238,65 @@ async function safeLocalImpact(
return { value: won.v, timedOut: false };
}
/**
* Race a single Phase-2 `impactByUid` call against a remaining-budget
* timer. The Codex adversarial review on PR #1331 surfaced that the
* fanout loop only checked `Date.now() > deadline` *between* neighbor
* calls — once `await port.impactByUid(...)` was reached, a hung
* neighbor could pin the request indefinitely, and slow neighbors
* could compound past the 5-min `IMPACT_TIMEOUT_MAX_MS` cap.
*
* This helper wraps each call: a `setTimeout(remainingMs)` aborts an
* `AbortController` whose signal is forwarded to `impactByUid`, and a
* `Promise.race` resolves to `{ timedOut: true }` when the timer
* fires before the call completes. Implementors that ignore the
* signal (current local backend) still see their await resolved by
* the race; full cooperative cancellation inside the BFS is a future
* follow-up. On rejection, the value is `null` (matching the
* fanout's existing `if (fan == null)` truncation contract).
*
* Exported for direct unit testing — the helper IS the load-bearing
* mitigation surface, so the U3 regression test pins it directly
* rather than driving the full `runGroupImpact` path.
*/
export async function safeNeighborImpact(
port: GroupToolPort,
repoId: string,
uid: string,
direction: string,
opts: {
maxDepth: number;
relationTypes: string[];
minConfidence: number;
includeTests: boolean;
},
remainingMs: number,
): Promise<{ value: unknown; timedOut: boolean }> {
const controller = new AbortController();
let timer: ReturnType<typeof setTimeout> | undefined;
const callP = port
.impactByUid(repoId, uid, direction, { ...opts, signal: controller.signal })
.catch(() => null);
const timeoutP = new Promise<'timeout'>((resolve) => {
timer = setTimeout(
() => {
controller.abort();
resolve('timeout');
},
Math.max(0, remainingMs),
);
});
const won = await Promise.race([
callP.then((v) => ({ tag: 'impact' as const, v })),
timeoutP.then(() => ({ tag: 'timeout' as const })),
]);
if (timer !== undefined) clearTimeout(timer);
if (won.tag === 'timeout') {
return { value: null, timedOut: true };
}
return { value: won.v, timedOut: false };
}
export function collectImpactSymbolUids(
local: unknown,
servicePrefix: string | undefined,
@ -476,7 +561,8 @@ export async function runGroupImpact(
if (seen.has(key)) continue;
seen.add(key);
if (Date.now() > deadline) {
const remainingMs = deadline - Date.now();
if (remainingMs <= 0) {
truncatedRepos.push(n.neighborRepo);
continue;
}
@ -492,13 +578,25 @@ export async function runGroupImpact(
continue;
}
const fan = await deps.port.impactByUid(neighborHandle.id, n.neighborUid, direction, {
maxDepth,
relationTypes: relationTypes ?? [],
minConfidence,
includeTests,
});
if (fan == null) {
// Phase-2 hardening: race each impactByUid against a per-call
// timeout derived from the remaining budget. Without this wrap a
// single hung neighbor would pin the request past the clamped
// timeout, which Codex's adversarial review on PR #1331 flagged
// as the still-open half of CodeQL #184 / js/resource-exhaustion.
const { value: fan, timedOut: neighborTimedOut } = await safeNeighborImpact(
deps.port,
neighborHandle.id,
n.neighborUid,
direction,
{
maxDepth,
relationTypes: relationTypes ?? [],
minConfidence,
includeTests,
},
remainingMs,
);
if (neighborTimedOut || fan == null) {
truncatedRepos.push(n.neighborRepo);
continue;
}

3
gitnexus/src/core/group/extractors/elixir-workspace-extractor.ts
View file

@ -4,6 +4,7 @@ import type { CypherExecutor } from '../contract-extractor.js';
import type { GroupManifestLink, ContractRole } from '../types.js';
import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
import { logger } from '../../logger.js';
interface ElixirAppMeta {
appName: string;
modulePrefix: string;
@ -202,7 +203,7 @@ export async function extractElixirWorkspaceLinks(
};
const existing = appsByName.get(manifest.appName);
if (existing) {
console.warn(
logger.warn(
`[elixir-workspace-extractor] duplicate app "${manifest.appName}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
);
continue;

3
gitnexus/src/core/group/extractors/go-workspace-extractor.ts
View file

@ -4,6 +4,7 @@ import type { CypherExecutor } from '../contract-extractor.js';
import type { GroupManifestLink, ContractRole } from '../types.js';
import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
import { logger } from '../../logger.js';
interface GoModuleMeta {
modulePath: string;
groupPath: string;
@ -211,7 +212,7 @@ export async function extractGoWorkspaceLinks(
};
const existing = modulesByPath.get(manifest.modulePath);
if (existing) {
console.warn(
logger.warn(
`[go-workspace-extractor] duplicate module "${manifest.modulePath}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
);
continue;

3
gitnexus/src/core/group/extractors/grpc-extractor.ts
View file

@ -5,6 +5,7 @@ import { createIgnoreFilter } from '../../../config/ignore-service.js';
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
import type { ExtractedContract, RepoHandle } from '../types.js';
import { readSafe } from './fs-utils.js';
import { logger } from '../../logger.js';
import {
GRPC_SCAN_GLOB,
getPluginForFile,
@ -344,7 +345,7 @@ export function resolveProtoConflict(
// services under a fabricated package-qualified contract id.
if (winners.length !== 1) {
const paths = candidates.map((c) => c.protoPath).join(', ');
console.warn(
logger.warn(
`[grpc-extractor] Ambiguous proto resolution for service "${serviceName}" from ${sourceFilePath}: ${winners.length} candidates tied at score ${maxScore} among [${paths}] — skipping canonical contract`,
);
return null;

3
gitnexus/src/core/group/extractors/java-workspace-extractor.ts
View file

@ -4,6 +4,7 @@ import type { CypherExecutor } from '../contract-extractor.js';
import type { GroupManifestLink, ContractRole } from '../types.js';
import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
import { logger } from '../../logger.js';
interface JavaProjectMeta {
groupId: string;
artifactId: string;
@ -213,7 +214,7 @@ export async function extractJavaWorkspaceLinks(
};
const existing = projectsByKey.get(key);
if (existing) {
console.warn(
logger.warn(
`[java-workspace-extractor] duplicate artifact "${key}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
);
continue;

3
gitnexus/src/core/group/extractors/manifest-extractor.ts
View file

@ -1,6 +1,7 @@
import type { ContractType, CrossLink, GroupManifestLink, StoredContract } from '../types.js';
import type { CypherExecutor } from '../contract-extractor.js';
import { logger } from '../../logger.js';
export interface ManifestExtractResult {
contracts: StoredContract[];
crossLinks: CrossLink[];
@ -303,7 +304,7 @@ export class ManifestExtractor {
// fail the whole manifest extraction. Unresolved contracts still
// get a synthetic symbolUid below, so cross-impact can proceed.
const message = err instanceof Error ? err.message : String(err);
console.warn(
logger.warn(
`[manifest-extractor] resolveSymbol failed for ${link.type}:${link.contract} ` +
`in ${repoPathKey}: ${message}`,
);

3
gitnexus/src/core/group/extractors/node-workspace-extractor.ts
View file

@ -4,6 +4,7 @@ import type { CypherExecutor } from '../contract-extractor.js';
import type { GroupManifestLink, ContractRole } from '../types.js';
import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
import { logger } from '../../logger.js';
interface PackageMeta {
name: string;
groupPath: string;
@ -205,7 +206,7 @@ export async function extractNodeWorkspaceLinks(
};
const existing = packagesByName.get(manifest.name);
if (existing) {
console.warn(
logger.warn(
`[node-workspace-extractor] duplicate package name "${manifest.name}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
);
continue;

3
gitnexus/src/core/group/extractors/python-workspace-extractor.ts
View file

@ -4,6 +4,7 @@ import type { CypherExecutor } from '../contract-extractor.js';
import type { GroupManifestLink, ContractRole } from '../types.js';
import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
import { logger } from '../../logger.js';
interface PythonPackageMeta {
name: string;
importName: string;
@ -204,7 +205,7 @@ export async function extractPythonWorkspaceLinks(
};
const existing = packagesByImportName.get(manifest.importName);
if (existing) {
console.warn(
logger.warn(
`[python-workspace-extractor] duplicate package "${manifest.name}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
);
continue;

34
gitnexus/src/core/group/extractors/rust-workspace-extractor.ts
View file

@ -5,6 +5,7 @@ import type { GroupManifestLink, ContractRole } from '../types.js';
import { shouldIgnorePath } from '../../../config/ignore-service.js';
import { loadIgnoreRules } from '../../../config/ignore-service.js';
import { logger } from '../../logger.js';
/**
* Discover cross-crate contracts in a Rust workspace by reading each
* member's `Cargo.toml` dependencies and scanning source files for
@ -30,6 +31,32 @@ interface ImportedSymbol {
filePath: string;
}
/**
* Linear-time `[package].name = "..."` lookup. The previous regex
* `^\[package\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"` had a nested
* lazy quantifier on `\n` that CodeQL js/redos flagged as exponential
* on inputs like `[package]\n` + many bare `\n`. We walk lines
* explicitly: scan from the first `[package]` header until we hit the
* next `[...]` section header, looking for the `name = "..."` line.
* O(n) with the line count.
*
* Exported so the U8 ReDoS regression test can drive the production
* line-walk directly with adversarial fixtures (multi-line strings,
* trailing sections, etc.) instead of duplicating it inline.
*/
export function parseCargoPackageName(content: string): string | null {
const lines = content.split('\n');
const packageStart = lines.findIndex((l) => l.trim() === '[package]');
if (packageStart < 0) return null;
for (let i = packageStart + 1; i < lines.length; i++) {
const line = lines[i].trimStart();
if (line.startsWith('[')) break; // hit the next section header
const m = /^name\s*=\s*"([^"]+)"/.exec(line);
if (m) return m[1];
}
return null;
}
/**
* Parse a Cargo.toml to extract the crate name and workspace dependency
* names. Uses simple line-based parsing — no TOML library needed for
@ -46,12 +73,9 @@ async function parseCrateManifest(
return null;
}
let name = '';
const name = parseCargoPackageName(content) ?? '';
const workspaceDeps: string[] = [];
const nameMatch = content.match(/^\[package\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"/m);
if (nameMatch) name = nameMatch[1];
// Match dependencies that use workspace = true, which indicates they
// are workspace-internal deps:
// dep_name = { workspace = true }
@ -224,7 +248,7 @@ export async function extractRustWorkspaceLinks(
};
const existing = cratesByName.get(manifest.name);
if (existing) {
console.warn(
logger.warn(
`[rust-workspace-extractor] duplicate crate name "${manifest.name}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
);
continue;

18
gitnexus/src/core/group/service.ts
View file

@ -14,6 +14,7 @@ import {
} from './group-path-utils.js';
import { getDefaultGitnexusDir, getGroupDir, listGroups, readContractRegistry } from './storage.js';
import { syncGroup } from './sync.js';
import { logger } from '../logger.js';
import type {
ContractRegistry,
CrossLink,
@ -64,6 +65,15 @@ export interface GroupToolPort {
relationTypes: string[];
minConfidence: number;
includeTests: boolean;
// Optional cancellation signal. Callers (notably the cross-impact
// Phase-2 fanout) wrap this call in a Promise.race against a
// setTimeout-driven AbortController so a single hung neighbor
// cannot exceed the request's clamped timeout budget. Implementors
// may honor the signal cooperatively or simply let the caller's
// race resolve the await — the latter is sufficient for the
// resource-exhaustion mitigation. When the signal is absent or
// already aborted at call time, behavior is unchanged.
signal?: AbortSignal;
},
): Promise<unknown | null>;
context(
@ -170,11 +180,11 @@ async function loadContractRegistryResilient(
contracts.push(row);
} else {
skippedCorrupt++;
console.warn('[group] skipping corrupt contract row in contracts.json');
logger.warn('[group] skipping corrupt contract row in contracts.json');
}
} catch {
skippedCorrupt++;
console.warn('[group] skipping corrupt contract row in contracts.json');
logger.warn('[group] skipping corrupt contract row in contracts.json');
}
}
}
@ -187,11 +197,11 @@ async function loadContractRegistryResilient(
crossLinks.push(row);
} else {
skippedCorrupt++;
console.warn('[group] skipping corrupt crossLinks row in contracts.json');
logger.warn('[group] skipping corrupt crossLinks row in contracts.json');
}
} catch {
skippedCorrupt++;
console.warn('[group] skipping corrupt crossLinks row in contracts.json');
logger.warn('[group] skipping corrupt crossLinks row in contracts.json');
}
}
}

60
gitnexus/src/core/group/storage.ts
View file

@ -2,8 +2,18 @@ import * as fs from 'node:fs';
import * as fsp from 'node:fs/promises';
import * as path from 'node:path';
import * as os from 'node:os';
import { randomBytes } from 'node:crypto';
import type { ContractRegistry } from './types.js';
/**
* Build an unpredictable suffix for atomic-write tmp files. Replaces the
* previous `Date.now()` pattern which CodeQL flagged as
* js/insecure-temporary-file: a guessable suffix in a writable directory
* lets a co-located attacker pre-create or symlink the tmp path before the
* write lands.
*/
const tmpSuffix = (): string => randomBytes(8).toString('hex');
const CONTRACTS_FILE = 'contracts.json';
export function getDefaultGitnexusDir(): string {
@ -34,9 +44,21 @@ export async function writeContractRegistry(
registry: ContractRegistry,
): Promise<void> {
const targetPath = path.join(groupDir, CONTRACTS_FILE);
const tmpPath = `${targetPath}.tmp.${Date.now()}`;
const tmpPath = `${targetPath}.tmp.${tmpSuffix()}`;
await fsp.writeFile(tmpPath, JSON.stringify(registry, null, 2), 'utf-8');
// O_EXCL via `'wx'` flag + explicit `0o600` mode — closes both halves
// of the CodeQL js/insecure-temporary-file finding: `'wx'` rejects a
// pre-planted symlink at the path, and `0o600` (user-only) prevents
// the file from being created group/world readable while it briefly
// contains contract data en route to the rename. The query's
// `isSecureMode` predicate inspects ONLY the mode argument, not the
// flags, so the explicit mode is what credits the fix.
const handle = await fsp.open(tmpPath, 'wx', 0o600);
try {
await handle.writeFile(JSON.stringify(registry, null, 2), 'utf-8');
} finally {
await handle.close();
}
await fsp.rename(tmpPath, targetPath);
}
@ -106,6 +128,38 @@ matching:
# exclude_links_paths: [/ping, /health, /healthcheck]
# exclude_links_param_only_paths: false
`;
await fsp.writeFile(path.join(groupDir, 'group.yaml'), template, 'utf-8');
// Always write group.yaml with O_EXCL via `fsp.open(..., 'wx')` —
// refuses to follow a pre-planted symlink at the target path, closing
// the TOCTOU window between the existence check (line ~98) and the
// write that CodeQL js/insecure-temporary-file flags. Under
// `force=true` we unlink the existing file first (best-effort, no-op
// when absent) so the subsequent O_EXCL open succeeds AND the same
// symlink-rejection guarantee holds — this is strictly safer than
// the previous `flag: force ? 'w' : 'wx'` shape, which silently
// followed symlinks under force. CodeQL's rule does not recognize
// the `writeFile(path, content, { flag: 'wx' })` shape as O_EXCL;
// the explicit open() handle below is what credits the mitigation.
const yamlPath = path.join(groupDir, 'group.yaml');
if (force) {
try {
await fsp.unlink(yamlPath);
} catch (err) {
// ENOENT (file absent) is expected on first run; rethrow anything
// else so we don't silently mask permission/EBUSY failures.
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err;
}
}
// `'wx'` rejects a pre-planted symlink at the path; `0o600` is
// user-only (no group/world bits) — gitnexus storage is per-user
// (`~/.gitnexus/...`), so any "other user wants to read this" case is
// a misconfiguration, not a feature. Keeping the file user-only also
// satisfies CodeQL's `isSecureMode` predicate (low 6 bits == 0) and
// closes the js/insecure-temporary-file alert at this site.
const handle = await fsp.open(yamlPath, 'wx', 0o600);
try {
await handle.writeFile(template, 'utf-8');
} finally {
await handle.close();
}
return groupDir;
}

7
gitnexus/src/core/group/sync.ts
View file

@ -16,6 +16,7 @@ import type { CypherExecutor } from './contract-extractor.js';
import { writeContractRegistry } from './storage.js';
import type { ContractRegistry } from './types.js';
import { logger } from '../logger.js';
export interface SyncOptions {
extractorOverride?:
| ((repo: RepoHandle) => Promise<StoredContract[]>)
@ -211,7 +212,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
allLinks = [...allLinks, ...wsResult.links];
if (opts?.verbose) {
for (const s of wsResult.stats) {
console.log(
logger.info(
` workspace-deps: discovered ${s.linkCount} cross-${s.ecosystem.toLowerCase()} links from ${s.projectCount} ${s.ecosystem} projects`,
);
}
@ -230,7 +231,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
for (const link of allLinks) {
const dangling = [link.from, link.to].filter((r) => !knownRepos.has(r));
if (dangling.length > 0) {
console.warn(
logger.warn(
`[group/sync] manifest link ${link.type}:${link.contract} references repos not in config.repos: ${dangling.join(', ')} — cross-links will use synthetic UIDs`,
);
}
@ -241,7 +242,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
autoContracts.push(...manifestResult.contracts);
manifestCrossLinks = manifestResult.crossLinks;
if (opts?.verbose) {
console.log(
logger.info(
` manifest: ${manifestCrossLinks.length} cross-links from ${allLinks.length} links (${config.links.length} declared + ${allLinks.length - config.links.length} discovered)`,
);
}

3
gitnexus/src/core/ingestion/ast-cache.ts
View file

@ -1,6 +1,7 @@
import { LRUCache } from 'lru-cache';
import Parser from 'tree-sitter';
import { logger } from '../logger.js';
/**
* Minimal structural shape consumers need when reading Trees back
* through a phase-dependency boundary. Declared here so phases that
@ -49,7 +50,7 @@ export const createASTCache = (maxSize: number = 50): ASTCache => {
// will hand freed memory to scope-resolution.
(tree as unknown as { delete?: () => void }).delete?.();
} catch (e) {
console.warn('Failed to delete tree from WASM memory', e);
logger.warn({ e }, 'Failed to delete tree from WASM memory');
}
},
});

5
gitnexus/src/core/ingestion/call-processor.ts
View file

@ -75,6 +75,7 @@ import { extractReturnTypeName, stripNullable } from './type-extractors/shared.j
import type { LiteralTypeInferrer } from './type-extractors/types.js';
import type { SyntaxNode } from './utils/ast-helpers.js';
import { logger } from '../logger.js';
/** Per-file resolved type bindings for exported symbols.
* Populated during call processing, consumed by Phase 14 re-resolution pass. */
export type ExportedTypeMap = Map<string, Map<string, string>>;
@ -784,7 +785,7 @@ export const processCalls = async (
const query = new Parser.Query(lang, queryStr);
matches = query.matches(tree.rootNode);
} catch (queryError) {
console.warn(`Query error for ${file.path}:`, queryError);
logger.warn({ queryError }, `Query error for ${file.path}:`);
continue;
}
@ -1391,7 +1392,7 @@ export const processCalls = async (
if (skippedByLang && skippedByLang.size > 0) {
for (const [lang, count] of skippedByLang.entries()) {
console.warn(
logger.warn(
`[ingestion] Skipped ${count} ${lang} file(s) in call processing — ${lang} parser not available.`,
);
}

5
gitnexus/src/core/ingestion/cluster-enricher.ts
View file

@ -7,6 +7,7 @@
import { CommunityNode } from './community-processor.js';
import { logger } from '../logger.js';
// ============================================================================
// TYPES
// ============================================================================
@ -128,7 +129,7 @@ export const enrichClusters = async (
enrichments.set(community.id, enrichment);
} catch (error) {
// On error, fallback to heuristic
console.warn(`Failed to enrich cluster ${community.id}:`, error);
logger.warn({ error }, `Failed to enrich cluster ${community.id}:`);
enrichments.set(community.id, {
name: community.heuristicLabel,
keywords: [],
@ -210,7 +211,7 @@ Output JSON array:
}
}
} catch (error) {
console.warn('Batch enrichment failed, falling back to heuristics:', error);
logger.warn({ error }, 'Batch enrichment failed, falling back to heuristics:');
// Fallback for this batch
for (const community of batch) {
enrichments.set(community.id, {

7
gitnexus/src/core/ingestion/cobol/cobol-copy-expander.ts
View file

@ -1,3 +1,4 @@
import { logger } from '../../logger.js';
/**
* COBOL COPY statement expansion engine.
*
@ -454,7 +455,7 @@ export function expandCopies(
if (visited.has(resolvedPath)) {
if (!warnedCircular.has(resolvedPath)) {
warnedCircular.add(resolvedPath);
console.warn(
logger.warn(
`[cobol-copy-expander] Circular COPY detected: ${cs.target} (${resolvedPath}) ` +
`includes itself. Skipping expansion.`,
);
@ -464,7 +465,7 @@ export function expandCopies(
// Max depth exceeded — keep unexpanded
if (depth >= maxDepth) {
console.warn(
logger.warn(
`[cobol-copy-expander] Max expansion depth (${maxDepth}) reached for ` +
`COPY ${cs.target} in ${srcPath}. Skipping expansion.`,
);
@ -475,7 +476,7 @@ export function expandCopies(
if (++totalExpansions > MAX_TOTAL_EXPANSIONS) {
if (!warnedCircular.has('__max_total__')) {
warnedCircular.add('__max_total__');
console.warn(
logger.warn(
`[cobol-copy-expander] Max total expansions (${MAX_TOTAL_EXPANSIONS}) reached ` +
`in ${srcPath}. Skipping further expansions.`,
);

17
gitnexus/src/core/ingestion/cobol/cobol-preprocessor.ts
View file

@ -369,9 +369,20 @@ const RE_USE_AFTER =
/\bUSE\s+(?:AFTER\s+)?(?:STANDARD\s+)?(?:EXCEPTION|ERROR)\s+ON\s+([A-Z][A-Z0-9-]+|INPUT|OUTPUT|I-O|EXTEND)\b/i;
// SET statement (condition, index)
const RE_SET_TO_TRUE = /\bSET\s+((?:[A-Z][A-Z0-9-]+(?:\s+OF\s+[A-Z][A-Z0-9-]+)?\s+)+)TO\s+TRUE\b/i;
const RE_SET_INDEX =
/\bSET\s+((?:[A-Z][A-Z0-9-]+\s+)+)(TO|UP\s+BY|DOWN\s+BY)\s+(\d+|[A-Z][A-Z0-9-]+)/i;
//
// Catastrophic-backtracking note (CodeQL js/redos): the previous shape
// `((?:[A-Z][A-Z0-9-]+(?:\s+OF\s+[A-Z][A-Z0-9-]+)?\s+)+)TO\s+TRUE`
// nested `\s+` quantifiers across alternations and was exponential on
// inputs like "SET a OF a OF a ... TO TRUE". Replaced with a lazy
// dot-match bounded by the explicit `\s+TO\s+TRUE` suffix — `.+?` is
// O(n) with the trailing anchor, and the captured group is parsed
// downstream the same way as before.
// Exported so the U8 ReDoS regression test can pin the exact production
// pattern. Direct import is the only way to ensure the test's
// pathological-input timing assertion exercises the production regex
// instead of an inline copy that drifts.
export const RE_SET_TO_TRUE = /\bSET\s+(.+?)\s+TO\s+TRUE\b/i;
export const RE_SET_INDEX = /\bSET\s+(.+?)\s+(TO|UP\s+BY|DOWN\s+BY)\s+(\d+|[A-Z][A-Z0-9-]+)/i;
// INITIALIZE statement — data reset (captures targets before REPLACING/WITH clause)
const RE_INITIALIZE = /\bINITIALIZE\s+([\s\S]*?)(?=\bREPLACING\b|\bWITH\b|\.\s*$|$)/i;

5
gitnexus/src/core/ingestion/filesystem-walker.ts
View file

@ -5,6 +5,7 @@ import path from 'path';
import { glob } from 'glob';
import { createIgnoreFilter } from '../../config/ignore-service.js';
import { logger } from '../logger.js';
export interface FileEntry {
path: string;
content: string;
@ -74,10 +75,10 @@ export const walkRepositoryPaths = async (
if (skippedLarge > 0) {
const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES;
const suffix = isDefault ? ', likely generated/vendored' : '';
console.warn(` Skipped ${skippedLarge} large files (>${maxFileSizeBytes / 1024}KB${suffix})`);
logger.warn(` Skipped ${skippedLarge} large files (>${maxFileSizeBytes / 1024}KB${suffix})`);
if (isVerboseIngestionEnabled()) {
for (const p of skippedLargePaths) {
console.warn(` - ${p}`);
logger.warn(` - ${p}`);
}
}
}

5
gitnexus/src/core/ingestion/heritage-processor.ts
View file

@ -34,6 +34,7 @@ import type { ResolutionContext } from './model/resolution-context.js';
import { TIER_CONFIDENCE } from './model/resolution-context.js';
import type { HeritageInfo } from './heritage-types.js';
import { logger } from '../logger.js';
/**
* Derive the heritage-resolution strategy for a language from its
* `LanguageProvider`. This is the production wiring that `buildHeritageMap`
@ -237,7 +238,7 @@ export const processHeritage = async (
query = new Parser.Query(treeSitterLang, queryStr);
matches = query.matches(tree.rootNode);
} catch (queryError) {
console.warn(`Heritage query error for ${file.path}:`, queryError);
logger.warn({ queryError }, `Heritage query error for ${file.path}:`);
continue;
}
@ -267,7 +268,7 @@ export const processHeritage = async (
if (skippedByLang && skippedByLang.size > 0) {
for (const [lang, count] of skippedByLang.entries()) {
console.warn(
logger.warn(
`[ingestion] Skipped ${count} ${lang} file(s) in heritage processing — ${lang} parser not available.`,
);
}

29
gitnexus/src/core/ingestion/import-processor.ts
View file

@ -27,6 +27,7 @@ import type { SyntaxNode } from './utils/ast-helpers.js';
import { isDev } from './utils/env.js';
import { isRegistryPrimary } from './registry-primary-flag.js';
import { logger } from '../logger.js';
// Type: Map<FilePath, Set<ResolvedFilePath>>
// Stores all files that a given file imports from
export type ImportMap = Map<string, Set<string>>;
@ -324,14 +325,18 @@ export const processImports = async (
matches = query.matches(tree.rootNode);
} catch (queryError: any) {
if (isDev) {
console.group(`🔴 Query Error: ${file.path}`);
console.log('Language:', language);
console.log('Query (first 200 chars):', queryStr.substring(0, 200) + '...');
console.log('Error:', queryError?.message || queryError);
console.log('File content (first 300 chars):', file.content.substring(0, 300));
console.log('AST root type:', tree.rootNode?.type);
console.log('AST has errors:', tree.rootNode?.hasError);
console.groupEnd();
logger.error(
{
file: file.path,
language,
err: queryError?.message || queryError,
queryPreview: queryStr.substring(0, 200) + '...',
contentPreview: file.content.substring(0, 300),
astRootType: tree.rootNode?.type,
astHasError: tree.rootNode?.hasError,
},
'tree-sitter query error',
);
}
if (wasReparsed) (tree as unknown as { delete?: () => void }).delete?.();
@ -346,7 +351,7 @@ export const processImports = async (
const sourceNode = captureMap['import.source'];
if (!sourceNode) {
if (isDev) {
console.log(`⚠️ Import captured but no source node in ${file.path}`);
logger.info(`⚠️ Import captured but no source node in ${file.path}`);
}
return;
}
@ -399,14 +404,14 @@ export const processImports = async (
if (skippedByLang && skippedByLang.size > 0) {
for (const [lang, count] of skippedByLang.entries()) {
console.warn(
logger.warn(
`[ingestion] Skipped ${count} ${lang} file(s) in import processing — ${lang} parser not available.`,
);
}
}
if (isDev) {
console.log(
logger.info(
`📊 Import processing complete: ${getResolvedCount()}/${totalImportsFound} imports resolved to graph edges`,
);
}
@ -498,7 +503,7 @@ export const processImportsFromExtracted = async (
);
if (isDev) {
console.log(
logger.info(
`📊 Import processing (fast path): ${getResolvedCount()}/${totalImportsFound} imports resolved to graph edges`,
);
}

11
gitnexus/src/core/ingestion/language-config.ts
View file

@ -4,6 +4,7 @@ import type { ImportConfigs } from './import-resolvers/types.js';
import { isDev } from './utils/env.js';
import { logger } from '../logger.js';
// ============================================================================
// LANGUAGE-SPECIFIC CONFIG TYPES
// ============================================================================
@ -82,7 +83,7 @@ export async function loadTsconfigPaths(repoRoot: string): Promise<TsconfigPaths
if (aliases.size > 0) {
if (isDev) {
console.log(`📦 Loaded ${aliases.size} path aliases from ${filename}`);
logger.info(`📦 Loaded ${aliases.size} path aliases from ${filename}`);
}
return { aliases, baseUrl };
}
@ -104,7 +105,7 @@ export async function loadGoModulePath(repoRoot: string): Promise<GoModuleConfig
const match = content.match(/^module\s+(\S+)/m);
if (match) {
if (isDev) {
console.log(`📦 Loaded Go module path: ${match[1]}`);
logger.info(`📦 Loaded Go module path: ${match[1]}`);
}
return { modulePath: match[1] };
}
@ -132,7 +133,7 @@ export async function loadComposerConfig(repoRoot: string): Promise<ComposerConf
}
if (isDev) {
console.log(`📦 Loaded ${psr4.size} PSR-4 mappings from composer.json`);
logger.info(`📦 Loaded ${psr4.size} PSR-4 mappings from composer.json`);
}
return { psr4 };
} catch {
@ -178,7 +179,7 @@ export async function loadCSharpProjectConfig(repoRoot: string): Promise<CSharpP
const projectDir = path.relative(repoRoot, dir).replace(/\\/g, '/');
configs.push({ rootNamespace, projectDir });
if (isDev) {
console.log(
logger.info(
`📦 Loaded C# project: ${entry.name} (namespace: ${rootNamespace}, dir: ${projectDir})`,
);
}
@ -217,7 +218,7 @@ export async function loadSwiftPackageConfig(repoRoot: string): Promise<SwiftPac
if (targets.size > 0) {
if (isDev) {
console.log(`📦 Loaded ${targets.size} Swift package targets`);
logger.info(`📦 Loaded ${targets.size} Swift package targets`);
}
return { targets };
}

3
gitnexus/src/core/ingestion/method-extractors/generic.ts
View file

@ -8,6 +8,7 @@
*/
import type { SyntaxNode } from '../utils/ast-helpers.js';
import { logger } from '../../logger.js';
import type {
MethodExtractor,
MethodExtractorContext,
@ -158,7 +159,7 @@ function findBodies(node: SyntaxNode, bodyNodeSet: Set<string>): SyntaxNode[] {
// Fallback: body field exists but its type is not in bodyNodeTypes.
// This may indicate a config typo — log for debugging if NODE_ENV is development.
if (process.env.NODE_ENV === 'development') {
console.warn(
logger.warn(
`[MethodExtractor] body field type '${bodyField.type}' not in bodyNodeTypes for node '${node.type}'`,
);
}

13
gitnexus/src/core/ingestion/parsing-processor.ts
View file

@ -34,6 +34,7 @@ import {
import type { LanguageProvider } from './language-provider.js';
import type { ParsedFile } from 'gitnexus-shared';
import { WorkerPool } from './workers/worker-pool.js';
import { logger } from '../logger.js';
import type {
ParseWorkerResult,
ParseWorkerInput,
@ -191,7 +192,7 @@ const processParsingWithWorkers = async (
const summary = Array.from(skippedLanguages.entries())
.map(([lang, count]) => `${lang}: ${count}`)
.join(', ');
console.warn(` Skipped unsupported languages: ${summary}`);
logger.warn(` Skipped unsupported languages: ${summary}`);
}
// Final progress
@ -382,7 +383,7 @@ const processParsingSequential = async (
bufferSize: getTreeSitterBufferSize(parseContent),
});
} catch (parseError) {
console.warn(`Skipping unparseable file: ${file.path}`);
logger.warn(`Skipping unparseable file: ${file.path}`);
continue;
}
@ -408,7 +409,7 @@ const processParsingSequential = async (
query = new Parser.Query(language, queryString);
matches = query.matches(tree.rootNode);
} catch (queryError) {
console.warn(`Query error for ${file.path}:`, queryError);
logger.warn({ queryError }, `Query error for ${file.path}:`);
continue;
}
@ -701,7 +702,7 @@ const processParsingSequential = async (
if (skippedByLang && skippedByLang.size > 0) {
for (const [lang, count] of skippedByLang.entries()) {
console.warn(
logger.warn(
`[ingestion] Skipped ${count} ${lang} file(s) in parsing processing — ${lang} parser not available.`,
);
}
@ -742,7 +743,7 @@ export const processParsing = async (
// in scope-resolution with an empty cache and get re-parsed.
// Surfacing this in PROF mode prevents silent perf cliffs when
// a repo crosses the worker-pool threshold.
console.warn(
logger.warn(
`[scope-resolution prof] worker pool engaged for ${files.length} files — cross-phase tree cache will be empty; scope-resolution re-parses.`,
);
}
@ -757,7 +758,7 @@ export const processParsing = async (
);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
console.warn('Worker pool parsing stopped; continuing with sequential parser:', message);
logger.warn({ message }, 'Worker pool parsing stopped; continuing with sequential parser:');
reportProgress?.(
lastProgress,
files.length,

7
gitnexus/src/core/ingestion/pipeline-phases/cobol.ts
View file

@ -15,6 +15,7 @@ import { readFileContents } from '../filesystem-walker.js';
import type { StructureOutput } from './structure.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
export interface CobolOutput {
programs: number;
paragraphs: number;
@ -47,7 +48,7 @@ export const cobolPhase: PipelinePhase<CobolOutput> = {
const cobolResult = processCobol(ctx.graph, cobolFiles, allPathSet);
if (isDev) {
console.log(
logger.info(
` COBOL: ${cobolResult.programs} programs, ${cobolResult.paragraphs} paragraphs, ${cobolResult.sections} sections from ${cobolFiles.length} files`,
);
if (
@ -55,12 +56,12 @@ export const cobolPhase: PipelinePhase<CobolOutput> = {
cobolResult.execCicsBlocks > 0 ||
cobolResult.entryPoints > 0
) {
console.log(
logger.info(
` COBOL enriched: ${cobolResult.execSqlBlocks} SQL blocks, ${cobolResult.execCicsBlocks} CICS blocks, ${cobolResult.entryPoints} entry points, ${cobolResult.moves} moves, ${cobolResult.fileDeclarations} file declarations`,
);
}
if (cobolResult.jclJobs > 0) {
console.log(` JCL: ${cobolResult.jclJobs} jobs, ${cobolResult.jclSteps} steps`);
logger.info(` JCL: ${cobolResult.jclJobs} jobs, ${cobolResult.jclSteps} steps`);
}
}

3
gitnexus/src/core/ingestion/pipeline-phases/communities.ts
View file

@ -15,6 +15,7 @@ import type { StructureOutput } from './structure.js';
import { processCommunities, type CommunityDetectionResult } from '../community-processor.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
export interface CommunitiesOutput {
communityResult: CommunityDetectionResult;
}
@ -47,7 +48,7 @@ export const communitiesPhase: PipelinePhase<CommunitiesOutput> = {
});
if (isDev) {
console.log(
logger.info(
`🏘️ Community detection: ${communityResult.stats.totalCommunities} communities found (modularity: ${communityResult.stats.modularity.toFixed(3)})`,
);
}

9
gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts
View file

@ -23,6 +23,7 @@ import { topologicalLevelSort } from '../utils/graph-sort.js';
import type { KnowledgeGraph } from '../../graph/types.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
/** Max AST trees to keep in LRU cache for cross-file binding propagation. */
const AST_CACHE_CAP = 50;
@ -60,7 +61,7 @@ export async function runCrossFileBindingPropagation(
const { levels, cycleCount } = topologicalLevelSort(ctx.importMap);
if (isDev && cycleCount > 0) {
console.log(`🔄 ${cycleCount} files in import cycles (processed last in undefined order)`);
logger.info(`🔄 ${cycleCount} files in import cycles (processed last in undefined order)`);
}
let filesWithGaps = 0;
@ -88,7 +89,7 @@ export async function runCrossFileBindingPropagation(
const gapRatio = totalFiles > 0 ? filesWithGaps / totalFiles : 0;
if (gapRatio < CROSS_FILE_SKIP_THRESHOLD && filesWithGaps < gapThreshold) {
if (isDev) {
console.log(
logger.info(
`⏭️ Cross-file re-resolution skipped (${filesWithGaps}/${totalFiles} files, ${(gapRatio * 100).toFixed(1)}% < ${CROSS_FILE_SKIP_THRESHOLD * 100}% threshold)`,
);
}
@ -193,7 +194,7 @@ export async function runCrossFileBindingPropagation(
if (crossFileResolved >= MAX_CROSS_FILE_REPROCESS) {
if (isDev)
console.log(`⚠️ Cross-file re-resolution capped at ${MAX_CROSS_FILE_REPROCESS} files`);
logger.info(`⚠️ Cross-file re-resolution capped at ${MAX_CROSS_FILE_REPROCESS} files`);
break;
}
}
@ -204,7 +205,7 @@ export async function runCrossFileBindingPropagation(
const elapsed = Date.now() - crossFileStart;
const totalElapsed = Date.now() - pipelineStart;
const reResolutionPct = totalElapsed > 0 ? ((elapsed / totalElapsed) * 100).toFixed(1) : '0';
console.log(
logger.info(
`🔗 Cross-file re-resolution: ${crossFileResolved} candidates re-processed` +
` in ${elapsed}ms (${reResolutionPct}% of total ingestion time so far)`,
);

5
gitnexus/src/core/ingestion/pipeline-phases/cross-file.ts
View file

@ -36,6 +36,7 @@ import type { ParseOutput } from './parse.js';
import { runCrossFileBindingPropagation } from './cross-file-impl.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
export interface CrossFileOutput {
/** Number of files re-processed during cross-file propagation. */
filesReprocessed: number;
@ -59,11 +60,11 @@ export const crossFilePhase: PipelinePhase<CrossFileOutput> = {
if (isDev) {
if (bindingAccumulator.totalBindings > 0) {
const memKB = Math.round(bindingAccumulator.estimateMemoryBytes() / 1024);
console.log(
logger.info(
`📦 BindingAccumulator: ${bindingAccumulator.totalBindings} bindings across ${bindingAccumulator.fileCount} files (~${memKB} KB)`,
);
} else if (totalFiles > 0) {
console.log(
logger.info(
`📦 BindingAccumulator: EMPTY — 0 bindings across 0 files despite ${totalFiles} parsed files. If the codebase has typed bindings, this indicates an upstream regression.`,
);
}

3
gitnexus/src/core/ingestion/pipeline-phases/markdown.ts
View file

@ -15,6 +15,7 @@ import { readFileContents } from '../filesystem-walker.js';
import type { StructureOutput } from './structure.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
export interface MarkdownOutput {
/** Number of markdown sections extracted. */
sections: number;
@ -48,7 +49,7 @@ export const markdownPhase: PipelinePhase<MarkdownOutput> = {
const mdResult = processMarkdown(ctx.graph, mdFiles, allPathSet);
if (isDev) {
console.log(
logger.info(
` Markdown: ${mdResult.sections} sections, ${mdResult.links} cross-links from ${mdFiles.length} files`,
);
}

3
gitnexus/src/core/ingestion/pipeline-phases/mro.ts
View file

@ -15,6 +15,7 @@ import type { StructureOutput } from './structure.js';
import { computeMRO } from '../mro-processor.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
export interface MROOutput {
entries: number;
ambiguityCount: number;
@ -42,7 +43,7 @@ export const mroPhase: PipelinePhase<MROOutput> = {
const mroResult = computeMRO(ctx.graph);
if (isDev && mroResult.entries.length > 0) {
console.log(
logger.info(
`🔀 MRO: ${mroResult.entries.length} classes analyzed, ${mroResult.ambiguityCount} ambiguities, ${mroResult.overrideEdges} METHOD_OVERRIDES, ${mroResult.methodImplementsEdges} METHOD_IMPLEMENTS`,
);
}

3
gitnexus/src/core/ingestion/pipeline-phases/orm.ts
View file

@ -16,6 +16,7 @@ import type { ExtractedORMQuery } from '../workers/parse-worker.js';
import type { KnowledgeGraph } from '../../graph/types.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
export interface ORMOutput {
edgesCreated: number;
modelCount: number;
@ -91,7 +92,7 @@ function processORMQueries(
}
if (isDev) {
console.log(
logger.info(
`ORM dataflow: ${edgesCreated} QUERIES edges, ${modelNodes.size} models (${queries.length} total calls)`,
);
}

21
gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts
View file

@ -69,6 +69,7 @@ import { isDev } from '../utils/env.js';
import { synthesizeWildcardImportBindings, needsSynthesis } from './wildcard-synthesis.js';
import { extractORMQueriesInline } from './orm-extraction.js';
import { logger } from '../../logger.js';
// ── Constants ──────────────────────────────────────────────────────────────
/** Max bytes of source content to load per parse chunk. */
@ -136,7 +137,7 @@ export async function runChunkedParseAndResolve(
}
}
for (const [lang, count] of skippedByLang) {
console.warn(
logger.warn(
`Skipping ${count} ${lang} file(s) — ${lang} parser not available (native binding may not have built). Try: npm rebuild tree-sitter-${lang}`,
);
}
@ -171,7 +172,7 @@ export async function runChunkedParseAndResolve(
if (isDev) {
const totalMB = parseableScanned.reduce((s, f) => s + f.size, 0) / (1024 * 1024);
console.log(
logger.info(
`📂 Scan: ${totalFiles} paths, ${totalParseable} parseable (${totalMB.toFixed(0)}MB), ${numChunks} chunks @ ${CHUNK_BYTE_BUDGET / (1024 * 1024)}MB budget`,
);
}
@ -220,9 +221,9 @@ export async function runChunkedParseAndResolve(
}
workerPool = createWorkerPool(workerUrl);
} catch (err) {
console.warn(
logger.warn(
{ err: (err as Error).message },
'Worker pool creation failed, using sequential fallback:',
(err as Error).message,
);
}
}
@ -339,7 +340,7 @@ export async function runChunkedParseAndResolve(
exportedTypeMap,
);
if (isDev && enrichedCount > 0) {
console.log(
logger.info(
`🔗 E1: Seeded ${enrichedCount} cross-file receiver types (chunk ${chunkIdx + 1})`,
);
}
@ -538,7 +539,7 @@ export async function runChunkedParseAndResolve(
const rcStats = ctx.getStats();
const total = rcStats.cacheHits + rcStats.cacheMisses;
const hitRate = total > 0 ? ((rcStats.cacheHits / total) * 100).toFixed(1) : '0';
console.log(
logger.info(
`🔍 Resolution cache: ${rcStats.cacheHits} hits, ${rcStats.cacheMisses} misses (${hitRate}% hit rate)`,
);
}
@ -554,15 +555,15 @@ export async function runChunkedParseAndResolve(
bindingAccumulator.finalize();
const enriched = enrichExportedTypeMap(bindingAccumulator, graph, exportedTypeMap);
if (isDev && enriched > 0) {
console.log(
logger.info(
`🔗 Worker TypeEnv enrichment: ${enriched} fixpoint-inferred exports added to ExportedTypeMap`,
);
}
} catch (enrichErr) {
if (isDev) {
console.warn(
logger.warn(
{ err: (enrichErr as Error).message },
'Post-fallback finalize/enrich failed during cleanup:',
(enrichErr as Error).message,
);
}
}
@ -571,7 +572,7 @@ export async function runChunkedParseAndResolve(
if (!hasSynthesized) {
const synthesized = synthesizeWildcardImportBindings(graph, ctx);
if (isDev && synthesized > 0) {
console.log(
logger.info(
`🔗 Synthesized ${synthesized} additional wildcard import bindings (Go/Ruby/C++/Swift/Python)`,
);
}

5
gitnexus/src/core/ingestion/pipeline-phases/processes.ts
View file

@ -19,6 +19,7 @@ import { processProcesses, type ProcessDetectionResult } from '../process-proces
import { generateId } from '../../../lib/utils.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
export interface ProcessesOutput {
processResult: ProcessDetectionResult;
}
@ -67,7 +68,7 @@ export const processesPhase: PipelinePhase<ProcessesOutput> = {
);
if (isDev) {
console.log(
logger.info(
`🔄 Process detection: ${processResult.stats.totalProcesses} processes found (${processResult.stats.crossCommunityCount} cross-community)`,
);
}
@ -167,7 +168,7 @@ export const processesPhase: PipelinePhase<ProcessesOutput> = {
}
}
if (isDev && linked > 0) {
console.log(`🔗 Linked ${linked} Route/Tool nodes to execution flows`);
logger.info(`🔗 Linked ${linked} Route/Tool nodes to execution flows`);
}
}

7
gitnexus/src/core/ingestion/pipeline-phases/routes.ts
View file

@ -32,6 +32,7 @@ import { generateId } from '../../../lib/utils.js';
import { readFileContents } from '../filesystem-walker.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
const EXPO_NAV_PATTERNS = [
/router\.(push|replace|navigate)\(\s*['"`]([^'"`]+)['"`]/g,
/<Link\s+[^>]*href=\s*['"`]([^'"`]+)['"`]/g,
@ -174,7 +175,7 @@ export const routesPhase: PipelinePhase<RoutesOutput> = {
}
if (isDev) {
console.log(
logger.info(
`🗺️ Route registry: ${routeRegistry.size} routes${duplicateRoutes > 0 ? ` (${duplicateRoutes} duplicate URLs skipped)` : ''}`,
);
}
@ -224,7 +225,7 @@ export const routesPhase: PipelinePhase<RoutesOutput> = {
linkedCount++;
}
if (isDev && linkedCount > 0) {
console.log(
logger.info(
`🛡️ Linked ${mwPath} middleware [${mwLabel.join(', ')}] to ${linkedCount} routes`,
);
}
@ -290,7 +291,7 @@ export const routesPhase: PipelinePhase<RoutesOutput> = {
processNextjsFetchRoutes(ctx.graph, allFetchCalls, routeURLToFile, consumerContents);
if (isDev) {
console.log(
logger.info(
`🔗 Processed ${allFetchCalls.length} fetch() calls against ${routeRegistry.size} routes`,
);
}

5
gitnexus/src/core/ingestion/pipeline-phases/runner.ts
View file

@ -15,6 +15,7 @@
import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
/**
* Validate that the phases form a valid dependency graph (no cycles, all deps present).
* Returns phases in topological execution order.
@ -176,7 +177,7 @@ export async function runPipeline(
const start = Date.now();
if (isDev) {
console.log(`▶ Phase: ${phase.name}`);
logger.info(`▶ Phase: ${phase.name}`);
}
// Only expose declared dependencies — prevents hidden coupling to undeclared phases.
@ -220,7 +221,7 @@ export async function runPipeline(
});
if (isDev) {
console.log(`✓ Phase: ${phase.name} (${durationMs}ms)`);
logger.info(`✓ Phase: ${phase.name} (${durationMs}ms)`);
}
}

3
gitnexus/src/core/ingestion/pipeline-phases/tools.ts
View file

@ -16,6 +16,7 @@ import { generateId } from '../../../lib/utils.js';
import { readFileContents } from '../filesystem-walker.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
export interface ToolDef {
name: string;
filePath: string;
@ -104,7 +105,7 @@ export const toolsPhase: PipelinePhase<ToolsOutput> = {
}
if (isDev) {
console.log(`🔧 Tool registry: ${toolDefs.length} tools detected`);
logger.info(`🔧 Tool registry: ${toolDefs.length} tools detected`);
}
}

7
gitnexus/src/core/ingestion/process-processor.ts
View file

@ -17,6 +17,7 @@ import { calculateEntryPointScore, isTestFile } from './entry-point-scoring.js';
import { SupportedLanguages } from 'gitnexus-shared';
import { isDev } from './utils/env.js';
import { logger } from '../logger.js';
// ============================================================================
// CONFIGURATION
// ============================================================================
@ -319,13 +320,13 @@ const findEntryPoints = (
// DEBUG: Log top candidates with new scoring details
if (sorted.length > 0 && isDev) {
console.log(`[Process] Top 10 entry point candidates (new scoring):`);
logger.info(`[Process] Top 10 entry point candidates (new scoring):`);
sorted.slice(0, 10).forEach((c, i) => {
const node = graph.getNode(c.id);
const exported = node?.properties.isExported ? '✓' : '✗';
const shortPath = node?.properties.filePath?.split('/').slice(-2).join('/') || '';
console.log(` ${i + 1}. ${node?.properties.name} [exported:${exported}] (${shortPath})`);
console.log(` score: ${c.score.toFixed(2)} = [${c.reasons.join(' × ')}]`);
logger.info(` ${i + 1}. ${node?.properties.name} [exported:${exported}] (${shortPath})`);
logger.info(` score: ${c.score.toFixed(2)} = [${c.reasons.join(' × ')}]`);
});
}

3
gitnexus/src/core/ingestion/scope-extractor-bridge.ts
View file

@ -28,6 +28,7 @@ import type { ParsedFile } from 'gitnexus-shared';
import { extract as extractScope } from './scope-extractor.js';
import type { LanguageProvider } from './language-provider.js';
import { logger } from '../logger.js';
/** Callback used to report scope-extraction warnings to the host (worker or direct). */
export type ScopeBridgeWarn = (message: string) => void;
@ -53,7 +54,7 @@ export function extractParsedFile(
err instanceof Error ? err.message : String(err)
}`;
if (onWarn !== undefined) onWarn(message);
else console.warn(message);
logger.warn(message);
return undefined;
}
}

5
gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts
View file

@ -38,6 +38,7 @@ import { runScopeResolution } from './run.js';
import { SCOPE_RESOLVERS } from './registry.js';
import { isDev, isSemanticModelValidatorEnabled } from '../../utils/env.js';
import { logger } from '../../../logger.js';
export interface ScopeResolutionOutput {
/** True when at least one language ran. */
readonly ran: boolean;
@ -144,7 +145,7 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
resolutionConfig,
onWarn: (msg) => {
if (isSemanticModelValidatorEnabled()) {
console.warn(`[scope-resolution:${lang}] ${msg}`);
logger.warn(`[scope-resolution:${lang}] ${msg}`);
}
},
},
@ -162,7 +163,7 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
});
if (isDev) {
console.log(
logger.info(
`[scope-resolution:${lang}] ${stats.filesProcessed} files → ${stats.importsEmitted} IMPORTS + ${stats.referenceEdgesEmitted} reference edges (${stats.resolve.unresolved} unresolved sites, ${stats.referenceSkipped} skipped)`,
);
}

3
gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts
View file

@ -41,6 +41,7 @@ import { emitImportEdges } from '../graph-bridge/imports-to-edges.js';
import type { ScopeResolver } from '../contract/scope-resolver.js';
import { buildWorkspaceResolutionIndex } from '../workspace-index.js';
import { logger } from '../../../logger.js';
interface RunScopeResolutionInput {
readonly graph: KnowledgeGraph;
/**
@ -279,7 +280,7 @@ export function runScopeResolution(
if (PROF) {
const tEnd = process.hrtime.bigint();
const ns = (a: bigint, b: bigint): number => Number(b - a) / 1_000_000;
console.warn(
logger.warn(
`[scope-resolution prof] extract=${ns(tStart, tExtract).toFixed(0)}ms` +
` finalize=${ns(tExtract, tFinalize).toFixed(0)}ms` +
` propagate=${ns(tFinalize, tPropagate).toFixed(0)}ms` +

3
gitnexus/src/core/ingestion/type-env.ts
View file

@ -24,6 +24,7 @@ import {
import type { SemanticModel } from './model/index.js';
import type { NodeLabel } from 'gitnexus-shared';
import { logger } from '../logger.js';
/**
* Per-file scoped type environment: maps (scope, variableName) → typeName.
* Scope-aware: variables inside functions are keyed by function name,
@ -769,7 +770,7 @@ const resolveFixpointBindings = (
if (iter === MAX_FIXPOINT_ITERATIONS - 1 && process.env.GITNEXUS_DEBUG) {
const unresolved = pendingItems.length - resolved.size;
if (unresolved > 0) {
console.warn(
logger.warn(
`[type-env] fixpoint hit iteration cap (${MAX_FIXPOINT_ITERATIONS}), ${unresolved} items unresolved`,
);
}

3
gitnexus/src/core/ingestion/utils/max-file-size.ts
View file

@ -1,5 +1,6 @@
import { TREE_SITTER_MAX_BUFFER } from '../constants.js';
import { logger } from '../../logger.js';
/** Default threshold (512 KB). Files larger than this are skipped by the walker. */
export const DEFAULT_MAX_FILE_SIZE_BYTES = 512 * 1024;
@ -11,7 +12,7 @@ const warned = new Set<string>();
const warnOnce = (key: string, message: string): void => {
if (warned.has(key)) return;
warned.add(key);
console.warn(message);
logger.warn(message);
};
/**

19
gitnexus/src/core/ingestion/vue-sfc-extractor.ts
View file

@ -23,7 +23,24 @@ interface ScriptBlock {
lang: string;
}
const SCRIPT_RE = /<script(\s[^>]*)?>([^]*?)<\/script>/g;
// Closing-tag pattern accepts:
// - whitespace before `>` — `</script >`, `</script\t\n>`
// - attribute-like junk after `script` — `</script foo="bar">`,
// `</script\t\n bar>`
// - any case — `</SCRIPT>`, `</Script>`
//
// HTML5 parses `</script foo>` as a valid close tag (attributes on
// close tags are ignored by the parser but still terminate the script
// block). A strict `<\/script\s*>` would miss those forms and let a
// crafted Vue file hide content from this extractor — exactly the
// CodeQL `js/bad-tag-filter` failure mode (the published test cases
// it checks include `</script foo="bar">` and `</script\t\n bar>`).
//
// `[^>]*` after `</script` accepts everything up to the next `>`,
// matching the HTML parser's actual close-tag behaviour. The `i` flag
// covers the case axis. PR #1330 CI surfaced both the case and
// attribute axes; this expression closes both at once.
const SCRIPT_RE = /<script(\s[^>]*)?>([^]*?)<\/script[^>]*>/gi;
const TEMPLATE_COMPONENT_RE = /<([A-Z][A-Za-z0-9]+)/g;
// Greedy: matches from the first <template> to the *last* </template>.
// This is intentional — nested <template v-slot:...> tags are valid Vue

9
gitnexus/src/core/ingestion/workers/parse-worker.ts
View file

@ -85,6 +85,7 @@ import type { LanguageProvider } from '../language-provider.js';
import type { ParsedFile } from 'gitnexus-shared';
import { extractParsedFile } from '../scope-extractor-bridge.js';
import { logger } from '../../logger.js';
// ============================================================================
// Types for serializable results
// ============================================================================
@ -1385,7 +1386,7 @@ const processFileGroup = (
if (parentPort) {
parentPort.postMessage({ type: 'warning', message });
} else {
console.warn(message);
logger.warn(message);
}
return;
}
@ -1414,7 +1415,7 @@ const processFileGroup = (
bufferSize: getTreeSitterBufferSize(parseContent),
});
} catch (err) {
console.warn(
logger.warn(
`Failed to parse file ${file.path}: ${err instanceof Error ? err.message : String(err)}`,
);
continue;
@ -1427,7 +1428,7 @@ const processFileGroup = (
try {
matches = query.matches(tree.rootNode);
} catch (err) {
console.warn(
logger.warn(
`Query execution failed for ${file.path}: ${err instanceof Error ? err.message : String(err)}`,
);
continue;
@ -1447,7 +1448,7 @@ const processFileGroup = (
file.path,
(message) => {
if (parentPort) parentPort.postMessage({ type: 'warning', message });
else console.warn(message);
else logger.warn(message);
},
tree,
);

33
gitnexus/src/core/ingestion/workers/worker-pool.ts
View file

@ -3,6 +3,7 @@ import os from 'node:os';
import fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { logger } from '../../logger.js';
export interface WorkerPool {
/**
* Dispatch items across workers. Items are split into bounded jobs, each job
@ -297,11 +298,18 @@ export const createWorkerPool = (
splitDepth: job.splitDepth + 1,
timeoutMs: nextTimeout,
};
console.warn(
`Worker ${workerIndex} parse job idle timeout after ${job.timeoutMs / 1000}s ` +
`(${job.items.length} items, ${job.estimatedBytes} bytes, last progress: ${lastProgress}). ` +
`Splitting into ${first.items.length}/${second.items.length} item jobs with ` +
`${nextTimeout / 1000}s timeout.`,
logger.warn(
{
workerIndex,
timeoutSec: job.timeoutMs / 1000,
items: job.items.length,
estimatedBytes: job.estimatedBytes,
lastProgress,
firstSplitItems: first.items.length,
secondSplitItems: second.items.length,
nextTimeoutSec: nextTimeout / 1000,
},
`Worker ${workerIndex} parse job idle timeout. Splitting into ${first.items.length}/${second.items.length} item jobs.`,
);
// Preserve intuitive retry order; final result order is still enforced by startIndex sort.
jobs.unshift(first, second);
@ -310,10 +318,15 @@ export const createWorkerPool = (
const nextAttempt = job.attempt + 1;
if (nextAttempt <= poolOptions.maxTimeoutRetries) {
console.warn(
`Worker ${workerIndex} parse job idle timeout after ${job.timeoutMs / 1000}s ` +
`(single item, attempt ${nextAttempt}/${poolOptions.maxTimeoutRetries + 1}). ` +
`Retrying with ${nextTimeout / 1000}s timeout.`,
logger.warn(
{
workerIndex,
timeoutSec: job.timeoutMs / 1000,
attempt: nextAttempt,
maxAttempts: poolOptions.maxTimeoutRetries + 1,
nextTimeoutSec: nextTimeout / 1000,
},
`Worker ${workerIndex} parse job idle timeout (single item). Retrying with ${nextTimeout / 1000}s timeout.`,
);
jobs.unshift({
...job,
@ -402,7 +415,7 @@ export const createWorkerPool = (
reportProgress();
} else if (msg.type === 'warning') {
resetIdleTimer();
console.warn(msg.message);
logger.warn(msg.message);
} else if (msg.type === 'sub-batch-done') {
waitingForFlush = true;
resetIdleTimer();

3
gitnexus/src/core/lbug/extension-loader.ts
View file

@ -1,6 +1,7 @@
import { spawn } from 'child_process';
import { fileURLToPath } from 'node:url';
import { LBUG_MAX_DB_SIZE } from './lbug-config.js';
import { logger } from '../logger.js';
const DEFAULT_EXTENSION_INSTALL_TIMEOUT_MS = 15_000;
const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/;
@ -188,7 +189,7 @@ export class ExtensionManager {
const policy = opts.policy ?? this.options.policy ?? resolvePolicyFromEnv();
const timeoutMs =
opts.installTimeoutMs ?? this.options.installTimeoutMs ?? getExtensionInstallTimeoutMs();
const warn = this.options.warn ?? console.warn;
const warn = this.options.warn ?? ((msg: string) => logger.warn(msg));
if (policy === 'never') {
this.markUnavailable(name, label, 'extension install policy is "never"', warn);

65
gitnexus/src/core/lbug/lbug-adapter.ts
View file

@ -19,11 +19,15 @@ import type { CachedEmbedding } from '../embeddings/types.js';
import { extensionManager, type ExtensionEnsureOptions } from './extension-loader.js';
import {
closeLbugConnection,
isDbBusyError,
isOpenRetryExhausted,
openLbugConnection,
waitForWindowsHandleRelease,
type LbugConnectionHandle,
} from './lbug-config.js';
import { isVectorExtensionSupportedByPlatform } from '../platform/capabilities.js';
import { logger } from '../logger.js';
// ---------------------------------------------------------------------------
// Relationship CSV splitting — extracted for testability (PR #818)
// ---------------------------------------------------------------------------
@ -184,21 +188,6 @@ const DB_LOCK_RETRY_ATTEMPTS = 3;
/** Base back-off in ms between BUSY retries (multiplied by attempt number). */
const DB_LOCK_RETRY_DELAY_MS = 500;
/**
* Return true when the error message indicates that another process holds
* an exclusive lock on the LadybugDB file (e.g. `gitnexus analyze` or
* `gitnexus serve` running at the same time).
*/
export const isDbBusyError = (err: unknown): boolean => {
const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
return (
msg.includes('busy') ||
msg.includes('lock') ||
msg.includes('already in use') ||
msg.includes('could not set lock')
);
};
/**
* Return true when the error message indicates a write was attempted against
* a read-only LadybugDB connection. The MCP query pool opens DBs read-only,
@ -251,7 +240,11 @@ export const withLbugDb = async <T>(dbPath: string, operation: () => Promise<T>)
});
} catch (err) {
lastError = err;
if (!isDbBusyError(err) || attempt === DB_LOCK_RETRY_ATTEMPTS) {
// Skip outer retry when the inner open-retry already exhausted: the
// ~1.5s open-time budget was just spent, repeating the full reset+
// reopen cycle would only add 4-5s of tail latency without changing
// the outcome (both layers consult the same isDbBusyError matcher).
if (!isDbBusyError(err) || isOpenRetryExhausted(err) || attempt === DB_LOCK_RETRY_ATTEMPTS) {
throw err;
}
// Close stale connection inside the session lock to prevent race conditions
@ -329,8 +322,17 @@ const doInitLbug = async (dbPath: string) => {
await conn.query(schemaQuery);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
if (!msg.includes('already exists')) {
console.warn(`⚠️ Schema creation warning: ${msg.slice(0, 120)}`);
// Suppression list:
// - "already exists": expected idempotent re-create on existing DBs
// - "could not set lock on file": LadybugDB v0.16.1 emits this on
// Windows when CREATE NODE TABLE runs against a path that was
// just opened (the WAL handle from a fresh Database briefly
// contests the table's first-write lock). The table is created
// anyway and any genuine cross-process lock contention surfaces
// on the next operation via withLbugDb's retry. Logging it here
// would just be noise in CI.
if (!msg.includes('already exists') && !isDbBusyError(err)) {
logger.warn(`⚠️ Schema creation warning: ${msg.slice(0, 120)}`);
}
}
}
@ -683,7 +685,7 @@ export const insertNodeToLbug = async (
return false;
} catch (e: any) {
// Node may already exist or other error
console.error(`Failed to insert ${label} node:`, e.message);
logger.error({ err: e.message }, `Failed to insert ${label} node:`);
return false;
}
};
@ -1010,14 +1012,14 @@ export const fetchExistingEmbeddingHashes = async (
const nodeId = r.nodeId ?? r[0];
if (nodeId) map.set(nodeId, STALE_HASH_SENTINEL);
}
console.log(
logger.info(
`[embed] ${map.size} nodes in legacy DB (missing chunk-aware columns) — all treated as stale`,
);
return map;
} catch (fallbackErr: any) {
const fallbackMsg = fallbackErr?.message ?? '';
if (isMissingColumnOrTableError(fallbackMsg)) {
console.log(
logger.info(
`[embed] CodeEmbedding table not yet present — full embedding run (${fallbackMsg})`,
);
return undefined;
@ -1063,6 +1065,9 @@ export const flushWAL = async (): Promise<void> => {
*/
export const safeClose = async (): Promise<void> => {
await flushWAL();
// Capture before close — currentDbPath stays set so the Windows post-close
// probe below knows which file to wait on.
const closingDbPath = currentDbPath;
if (conn) {
try {
// eslint-disable-next-line no-restricted-syntax -- sole authorised close site
@ -1081,6 +1086,24 @@ export const safeClose = async (): Promise<void> => {
}
db = null;
}
// Windows: libuv reports `db.close()` resolved before the kernel has
// released the file handle. A subsequent `new Database(samePath)` in
// the same process can race the release. The probe (lbug-config.ts)
// forces any residual lock to surface as EBUSY/EPERM/EACCES so the
// open-time retry absorbs the lag.
if (process.platform === 'win32' && closingDbPath) {
const released = await waitForWindowsHandleRelease(closingDbPath);
if (!released) {
// Probe exhausted with a lock code still in flight. The next
// openLbugConnection will absorb whatever residual lag remains, but
// a chronic warning helps operators spot AV interference (Windows
// Defender holding the file far past the 250ms budget).
logger.warn(
{ dbPath: closingDbPath },
'⚠️ LadybugDB file handle still locked after close (Windows). If this repeats, check antivirus/Defender exclusions for the GitNexus storage directory.',
);
}
}
};
export const closeLbug = async (): Promise<void> => {

266
gitnexus/src/core/lbug/lbug-config.ts
View file

@ -1,3 +1,6 @@
import fs from 'fs/promises';
import os from 'os';
import path from 'path';
import type lbug from '@ladybugdb/core';
/**
@ -42,10 +45,23 @@ export const LBUG_MAX_DB_SIZE: number = (() => {
return 16 * 1024 * 1024 * 1024;
})();
/** Matches WAL corruption errors from the LadybugDB engine. */
const WAL_CORRUPTION_RE = /corrupt(ed)?\s+wal|invalid\s+wal\s+record|wal.*corrupt|checksum.*wal/i;
export const WAL_RECOVERY_SUGGESTION =
'WAL corruption detected. Run `gitnexus analyze` to rebuild the index.';
export function isWalCorruptionError(err: unknown): boolean {
if (!err) return false;
const msg = err instanceof Error ? err.message : String(err);
return WAL_CORRUPTION_RE.test(msg);
}
type LbugModule = typeof lbug;
export interface LbugDatabaseOptions {
readOnly?: boolean;
throwOnWalReplayFailure?: boolean;
}
export interface LbugConnectionHandle {
@ -53,20 +69,200 @@ export interface LbugConnectionHandle {
conn: lbug.Connection;
}
/**
* Return true when the error message indicates that a LadybugDB file lock
* could not be acquired — either at construction time
* (`new lbug.Database(...)` raises from `local_file_system.cpp`) or during
* a query (another writer holds the exclusive lock).
*
* Lives here (not in `lbug-adapter.ts`) so both the construction-time
* retry (`openWithLockRetry` in this file) and the query-time retry
* (`withLbugDb` in `lbug-adapter.ts`) consult the same matcher. Callers
* import directly from this module — no re-export to keep in sync.
*/
export const isDbBusyError = (err: unknown): boolean => {
const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
// `lock` already subsumes `could not set lock`; the broader term is kept
// because graph-DB transient errors include "deadlock", "lock contention",
// and the LadybugDB native module's "could not set lock on file" — all of
// which deserve a retry. If a non-transient lock-shaped error ever
// surfaces (e.g., "lock file missing" during recovery), tighten this
// matcher rather than raising the retry budget.
return msg.includes('busy') || msg.includes('lock') || msg.includes('already in use');
};
export function createLbugDatabase(
lbugModule: LbugModule,
databasePath: string,
options: LbugDatabaseOptions = {},
): lbug.Database {
return new lbugModule.Database(
// .d.ts declares fewer args than the native constructor accepts.
return new (lbugModule.Database as any)(
databasePath,
0,
false,
0, // bufferManagerSize
false, // enableCompression (pinned for v0.16.0)
options.readOnly ?? false,
LBUG_MAX_DB_SIZE,
);
true, // autoCheckpoint
-1, // checkpointThreshold
options.throwOnWalReplayFailure ?? true,
true, // enableChecksums
) as lbug.Database;
}
// ─── Lock-busy retry tuning knobs ───────────────────────────────────────────
//
// All four GitNexus retry pairs that touch native LadybugDB locks live with
// a comment cross-reference here so an SRE tuning Windows flakes finds them
// in one grep:
//
// 1. OPEN_LOCK_RETRY_ATTEMPTS / OPEN_LOCK_RETRY_DELAY_MS (this file)
// → `new lbug.Database()` constructor lock failures
// 2. HANDLE_RELEASE_PROBE_ATTEMPTS / HANDLE_RELEASE_PROBE_DELAY_MS (this file)
// → post-close fs.open probe to absorb Windows handle-release lag
// 3. DB_LOCK_RETRY_ATTEMPTS / DB_LOCK_RETRY_DELAY_MS (lbug-adapter.ts withLbugDb)
// → query-time busy/lock retry around already-open connections
//
// `new lbug.Database()` calls into the native module which performs an
// OS-level exclusive lock on `<dbPath>`. On Windows that lock can fail
// for reasons specific to the OS (Defender briefly opens new files,
// libuv handle release lags the JS-side close). 5 attempts × 100ms
// linear back-off (max sleep 100+200+300+400 = 1s, plus 5 ctor RTTs
// of 10–50ms each = ~1.0–1.2s worst case) clears the typical
// AV-scanner hold without masking real cross-process conflicts.
//
// Source: https://github.com/LadybugDB/ladybug/blob/v0.16.1/src/common/file_system/local_file_system.cpp#L126
const OPEN_LOCK_RETRY_ATTEMPTS = 5;
const OPEN_LOCK_RETRY_DELAY_MS = 100;
const HANDLE_RELEASE_PROBE_ATTEMPTS = 5;
const HANDLE_RELEASE_PROBE_DELAY_MS = 50;
const HANDLE_RELEASE_LOCK_CODES = new Set(['EBUSY', 'EPERM', 'EACCES']);
/**
* Test-fixture directory prefixes recognized by `isTestFixturePath`.
*
* IMPORTANT: this list must stay in sync with the prefixes passed to
* `createTempDir` in `gitnexus/test/helpers/test-db.ts` and the prefixes
* used by `withTestLbugDB` (`gitnexus/test/helpers/test-indexed-db.ts`).
* If you add a new test that passes a custom prefix to `createTempDir`,
* add it here too — otherwise the stale-sidecar sweep silently won't
* fire for that fixture and CI flakes return.
*
* The default `createTempDir('gitnexus-test-')` and the lbug variant
* `'gitnexus-lbug-'` cover today's call sites.
*/
const TEST_FIXTURE_PREFIXES = ['gitnexus-lbug-', 'gitnexus-test-'];
/**
* Marker symbol attached to lock errors after `openWithLockRetry` exhausts
* its budget. `withLbugDb`'s outer query-time retry consults this so it
* does not re-retry a path that just spent up to ~1.5s in the open-time
* loop — preventing 6s tail latencies (3× outer × 5× inner attempts).
*
* The symbol is internal to GitNexus; consumers should treat the underlying
* error message as the user-visible signal.
*/
export const LBUG_OPEN_RETRY_EXHAUSTED = Symbol.for('gitnexus.lbug.openRetryExhausted');
export const isOpenRetryExhausted = (err: unknown): boolean => {
if (err === null || err === undefined || typeof err !== 'object') return false;
return (err as { [LBUG_OPEN_RETRY_EXHAUSTED]?: boolean })[LBUG_OPEN_RETRY_EXHAUSTED] === true;
};
const tagOpenRetryExhausted = (err: unknown): unknown => {
if (err && typeof err === 'object') {
(err as { [LBUG_OPEN_RETRY_EXHAUSTED]?: boolean })[LBUG_OPEN_RETRY_EXHAUSTED] = true;
}
return err;
};
/**
* True when `dbPath` resolves to a recognized test fixture under the OS
* temp directory. Used to gate the stale-sidecar sweep so production
* paths never have their `.wal` / `.lock` files deleted.
*
* Defensive shape:
* - `path.resolve` normalizes `..` segments before the prefix check, so
* `<tmp>/gitnexus-lbug-x/../../etc/passwd` is rejected.
* - The tmpRoot check trims any trailing separator returned by some
* Windows TMP configurations (`C:\Users\X\Temp\`) so the startsWith
* comparison stays correct.
* - Only the IMMEDIATE parent directory is matched against the prefix
* list. An ancestor walk would let a tmpdir whose own basename starts
* with `gitnexus-lbug-` accept arbitrary nested paths under it.
*/
const isTestFixturePath = (dbPath: string): boolean => {
const tmpRoot = os.tmpdir().replace(new RegExp(`${path.sep === '\\' ? '\\\\' : path.sep}+$`), '');
const resolved = path.resolve(dbPath);
if (!resolved.startsWith(tmpRoot + path.sep) && resolved !== tmpRoot) return false;
const parentBase = path.basename(path.dirname(resolved));
return TEST_FIXTURE_PREFIXES.some((p) => parentBase.startsWith(p));
};
/** Exported only for direct unit testing — production callers use `openWithLockRetry`. */
export const _isTestFixturePathForTest = isTestFixturePath;
const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
/**
* Attempt to remove stale `.wal` / `.lock` sidecars that a previous aborted
* test run may have left behind. Best-effort: ENOENT is normal, anything
* else is swallowed so the caller's retry can surface the original error.
*/
const sweepStaleSidecars = async (dbPath: string): Promise<void> => {
for (const suffix of ['.wal', '.lock']) {
try {
await fs.unlink(dbPath + suffix);
} catch {
/* missing sidecar or permission error — let the open retry surface it */
}
}
};
/**
* Run `construct` with bounded retries when `new lbug.Database(...)` throws
* a busy/lock error. The original (loop-captured) error is preferred over
* any post-sweep error so triage sees the real LadybugDB lock message.
* On exhaustion the rethrown error is tagged via
* `LBUG_OPEN_RETRY_EXHAUSTED` so the outer query-time retry in
* `withLbugDb` skips re-retrying a freshly-exhausted path.
*/
const openWithLockRetry = async (
construct: () => lbug.Database,
dbPath: string,
): Promise<lbug.Database> => {
let originalLockError: unknown;
for (let attempt = 1; attempt <= OPEN_LOCK_RETRY_ATTEMPTS; attempt++) {
try {
return construct();
} catch (err) {
if (!isDbBusyError(err)) throw err;
originalLockError = err;
if (attempt === OPEN_LOCK_RETRY_ATTEMPTS) break;
await sleep(OPEN_LOCK_RETRY_DELAY_MS * attempt);
}
}
// Final defense: only for recognized test fixtures, sweep stale sidecars
// (a prior aborted test run can leave a `.wal` lock that survives the
// tmp dir cleanup). Production paths never reach this branch — the guard
// requires the immediate parent dir to match a test prefix AND the
// resolved path to live under the OS temp directory.
if (isTestFixturePath(dbPath)) {
await sweepStaleSidecars(dbPath);
try {
return construct();
} catch {
// Intentionally do NOT overwrite originalLockError. The user-actionable
// signal is "we exhausted lock retries" — a different error from the
// post-sweep attempt is less useful than the lock failure that drove
// the sweep in the first place.
}
}
throw tagOpenRetryExhausted(originalLockError);
};
export async function openLbugConnection(
lbugModule: LbugModule,
databasePath: string,
@ -74,7 +270,10 @@ export async function openLbugConnection(
): Promise<LbugConnectionHandle> {
let db: lbug.Database | undefined;
try {
db = createLbugDatabase(lbugModule, databasePath, options);
db = await openWithLockRetry(
() => createLbugDatabase(lbugModule, databasePath, options),
databasePath,
);
return { db, conn: new lbugModule.Connection(db) };
} catch (err) {
if (db) await db.close().catch(() => {});
@ -86,3 +285,60 @@ export async function closeLbugConnection(handle: LbugConnectionHandle): Promise
await handle.conn.close().catch(() => {});
await handle.db.close().catch(() => {});
}
/**
* Probe `dbPath` AND its `.wal` sidecar after `db.close()` so any
* residual native file handle surfaces as EBUSY/EPERM/EACCES and the
* bounded retry absorbs the release lag. Windows-only — Linux/macOS do
* not exhibit this race.
*
* Both files matter. Empirically, on rapid open→close→reopen cycles the
* main `dbPath` handle releases first; the `.wal` handle from the
* previous Database lingers and the new Database's first write (CREATE
* NODE TABLE during schema init) fails with "Could not set lock on
* file". Probing both makes safeClose actually return when the kernel
* is fully done with the path.
*
* Returns `true` when both probes succeeded (or skipped on non-lock
* errors / missing files). Returns `false` when either probe exhausted
* its budget with a lock code still in flight.
*
* Defensive shape:
* - Opens read+write (`'r+'`) so the probe actually surfaces exclusive
* locks held by the previous Database. A read-only probe (`'r'`) is
* insufficient — Windows will grant read access while the previous
* handle's exclusive write lock is still in flight, which lets
* `safeClose` return before the next CREATE NODE TABLE can lock the
* file.
* - `try/finally` around `handle.close()` guarantees no fd leak even
* if close itself throws.
*/
export const waitForWindowsHandleRelease = async (dbPath: string): Promise<boolean> => {
const mainReleased = await probeSinglePath(dbPath);
const walReleased = await probeSinglePath(dbPath + '.wal');
return mainReleased && walReleased;
};
const probeSinglePath = async (filePath: string): Promise<boolean> => {
for (let attempt = 1; attempt <= HANDLE_RELEASE_PROBE_ATTEMPTS; attempt++) {
let handle: fs.FileHandle | undefined;
try {
handle = await fs.open(filePath, 'r+');
return true;
} catch (err) {
const code = (err as NodeJS.ErrnoException | undefined)?.code;
if (!code || !HANDLE_RELEASE_LOCK_CODES.has(code)) return true; // ENOENT / unrelated → not our problem
if (attempt === HANDLE_RELEASE_PROBE_ATTEMPTS) return false;
await sleep(HANDLE_RELEASE_PROBE_DELAY_MS * attempt);
} finally {
if (handle) {
try {
await handle.close();
} catch {
/* swallow — caller cannot do anything useful with a probe-close failure */
}
}
}
}
return false;
};

88
gitnexus/src/core/lbug/pool-adapter.ts
View file

@ -18,7 +18,7 @@
import fs from 'fs/promises';
import lbug from '@ladybugdb/core';
import { loadFTSExtension } from './lbug-adapter.js';
import { createLbugDatabase } from './lbug-config.js';
import { createLbugDatabase, isWalCorruptionError } from './lbug-config.js';
/** Per-repo pool: one Database, many Connections */
interface PoolEntry {
@ -84,9 +84,21 @@ const MAX_CONNS_PER_REPO = 8;
let idleTimer: ReturnType<typeof setInterval> | null = null;
/** Saved real stdout/stderr write — used to silence native module output without race conditions */
export const realStdoutWrite = process.stdout.write.bind(process.stdout);
export const realStderrWrite = process.stderr.write.bind(process.stderr);
// Stdout-capture state lives in `gitnexus/src/mcp/stdio-capture.ts` — a leaf
// module with zero non-`node:` imports. We re-export the same symbols here
// so the existing test mock seam (`gitnexus/src/mcp/core/lbug-adapter.ts`
// re-exports * from this file, and 8+ test files use that path with
// `vi.mock(...)`) continues to work without churn. The source of truth is
// the leaf module; this re-export is a compatibility shim.
//
// Why the leaf module exists: Codex's adversarial review on PR #1383 found
// that putting this state in pool-adapter.ts pulled `@ladybugdb/core` into
// `cli/mcp.ts`'s static-import closure (via stdio-context → pool-adapter →
// @ladybugdb/core), corrupting stdout in the pre-sentinel window. Routing
// through the leaf breaks that chain.
export { realStdoutWrite, realStderrWrite, setActiveStdoutWrite } from '../../mcp/stdio-capture.js';
import { getActiveStdoutWrite, realStderrWrite } from '../../mcp/stdio-capture.js';
let stdoutSilenceCount = 0;
/** True while pre-warming connections — prevents watchdog from prematurely restoring stdout */
let preWarmActive = false;
@ -209,6 +221,7 @@ let activeQueryCount = 0;
*/
export function silenceStdout(): void {
if (stdoutSilenceCount++ === 0) {
// eslint-disable-next-line no-restricted-syntax -- silencing infrastructure; replacement is a no-op
process.stdout.write = (() => true) as any;
}
}
@ -216,7 +229,8 @@ export function silenceStdout(): void {
export function restoreStdout(): void {
if (--stdoutSilenceCount <= 0) {
stdoutSilenceCount = 0;
process.stdout.write = realStdoutWrite;
// eslint-disable-next-line no-restricted-syntax -- restoring the active stdout-write handler is the silencing API contract
process.stdout.write = getActiveStdoutWrite();
}
}
@ -227,7 +241,8 @@ export function restoreStdout(): void {
setInterval(() => {
if (stdoutSilenceCount > 0 && !preWarmActive && activeQueryCount === 0) {
stdoutSilenceCount = 0;
process.stdout.write = realStdoutWrite;
// eslint-disable-next-line no-restricted-syntax -- watchdog recovery for stuck silencing
process.stdout.write = getActiveStdoutWrite();
}
}, 1000).unref();
@ -248,6 +263,46 @@ const WAITER_TIMEOUT_MS = 15_000;
const LOCK_RETRY_ATTEMPTS = 3;
const LOCK_RETRY_DELAY_MS = 2000;
async function openReadOnlyDatabase(dbPath: string): Promise<lbug.Database> {
let db: lbug.Database | undefined;
silenceStdout();
try {
db = createLbugDatabase(lbug, dbPath, {
readOnly: true,
throwOnWalReplayFailure: false,
});
await db.init();
return db;
} catch (err) {
if (db) await db.close().catch(() => {});
throw err;
} finally {
restoreStdout();
}
}
/**
* Quarantine the .wal file and retry opening the database.
* Used when the initial open fails with a WAL corruption error.
*/
async function tryQuarantineAndReopen(dbPath: string, repoId: string): Promise<lbug.Database> {
const walPath = dbPath + '.wal';
const quarantineName = `${walPath}.corrupt.${Date.now()}-${Math.random().toString(36).slice(2)}`;
try {
await fs.rename(walPath, quarantineName);
} catch {
throw new Error(
`LadybugDB WAL corruption detected for ${repoId}. ` +
`Run \`gitnexus analyze\` to rebuild the index. (quarantine failed)`,
);
}
realStderrWrite(
`GitNexus: LadybugDB WAL quarantined for ${repoId}; graph may be stale. ` +
`Run \`gitnexus analyze\` to rebuild the index.\n`,
);
return await openReadOnlyDatabase(dbPath);
}
/** Deduplicates concurrent initLbug calls for the same repoId */
const initPromises = new Map<string, Promise<void>>();
@ -304,16 +359,29 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<void> {
// avoids lock conflicts when `gitnexus analyze` is writing.
let lastError: Error | null = null;
for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) {
silenceStdout();
try {
const db = createLbugDatabase(lbug, dbPath, { readOnly: true });
restoreStdout();
const db = await openReadOnlyDatabase(dbPath);
shared = { db, refCount: 0, ftsLoaded: false };
dbCache.set(dbPath, shared);
break;
} catch (err: any) {
restoreStdout();
lastError = err instanceof Error ? err : new Error(String(err));
if (isWalCorruptionError(lastError)) {
try {
const db = await tryQuarantineAndReopen(dbPath, repoId);
shared = { db, refCount: 0, ftsLoaded: false };
dbCache.set(dbPath, shared);
break;
} catch (retryErr) {
throw new Error(
`LadybugDB WAL corruption detected for ${repoId}. ` +
`Run \`gitnexus analyze\` to rebuild the index. ` +
`(${retryErr instanceof Error ? retryErr.message : String(retryErr)})`,
);
}
}
const isLockError =
lastError.message.includes('Could not set lock') || lastError.message.includes('lock');
if (!isLockError || attempt === LOCK_RETRY_ATTEMPTS) break;

375
gitnexus/src/core/logger.ts Normal file
View file

@ -0,0 +1,375 @@
/**
* Centralized structured logger for GitNexus.
*
* Wraps `pino` so the rest of the codebase imports from one place. Pino's
* NDJSON output is structurally log-injection-resistant (CWE-117 / CodeQL
* `js/log-injection`): each record is a single JSON object on its own line,
* with all string field values JSON-escaped. This replaces hand-rolled
* sanitizers (see PR #1329 history) that had recurring edge-case gaps
* (undefined Error.message, U+2028/U+2029, ANSI/C0).
*
* Usage:
* import { logger, createLogger } from '../core/logger.js';
* logger.warn({ groupDir }, 'msg');
* const childLogger = createLogger('bridge-db', { debugEnvVar: 'GITNEXUS_DEBUG_BRIDGE' });
*
* Operator semantics:
* - Default level: 'info' (matches pino default; preserves visibility of
* existing `console.log` migrations)
* - When `opts.debugEnvVar` is set and that env var is truthy at
* createLogger time, that named child logs at level 'debug'
* - Output is NDJSON in production / CI / vitest. pino-pretty is used only
* when stdout is a TTY AND CI is unset AND VITEST is unset, so test
* and pipeline output stay parseable.
*
* Test capture:
* The exported `logger` singleton is a Proxy that forwards every call to a
* lazily-built pino instance. Tests use `_captureLogger()` to redirect that
* inner instance to a memory stream so they can assert on records the
* production code logged. See `gitnexus/test/unit/logger.test.ts` for the
* pattern.
*/
import pino, { type Logger, type LoggerOptions, type DestinationStream } from 'pino';
import { Writable } from 'node:stream';
import { createRequire } from 'node:module';
export interface CreateLoggerOptions {
/** When set, this env var (truthy at construction time) bumps level to 'debug'. */
debugEnvVar?: string;
/** Override destination stream — primarily for tests. */
destination?: DestinationStream;
}
function isTruthyEnv(value: string | undefined): boolean {
if (!value) return false;
const v = value.toLowerCase();
return v !== '' && v !== '0' && v !== 'false' && v !== 'no' && v !== 'off';
}
function shouldUsePretty(): boolean {
// Logger writes to stderr (fd 2) so CLI data on stdout (fd 1) stays clean.
// Pretty-print only when stderr is a TTY and not in CI/test environments.
return (
process.stderr.isTTY === true &&
!isTruthyEnv(process.env.CI) &&
!isTruthyEnv(process.env.VITEST)
);
}
/**
* Default pino destination — writes to stderr (fd 2) so CLI commands can
* keep stdout (fd 1) clean for tool data output (#324). Pino defaults to
* stdout; we override here.
*
* `sync: false` (SonicBoom buffered writes) so logger calls don't issue a
* blocking `write(2)` syscall on every record. Hot paths (parse-impl,
* ingestion phases, per-query backend calls) pay the cost without it.
*
* The buffered-write trade-off is record loss on hard exit. We mitigate via:
* - A `process.on('beforeExit')` hook below that calls `flushSync()` on
* normal exits.
* - The exported `flushLoggerSync()` helper, which entry-point shutdown
* handlers (SIGINT/SIGTERM) MUST call before `process.exit(N)` so
* in-flight buffered records still reach stderr.
* - `pino.final(...)` integration in `uncaughtException` / `unhandledRejection`
* handlers (see `gitnexus/src/cli/serve.ts` and `gitnexus/src/server/api.ts`).
*
* Skipped under `VITEST` so vitest's between-test cleanup doesn't fight
* `_captureLogger()`'s lifecycle. Tests use an in-memory destination via
* `_captureLogger()` and never reach this branch.
*/
let _dest: ReturnType<typeof pino.destination> | undefined;
function defaultDestination(): DestinationStream {
if (_dest) return _dest;
_dest = pino.destination({ dest: 2, sync: false });
return _dest;
}
/**
* Flush any buffered records on the default destination. Entry-point
* shutdown handlers (`SIGINT` / `SIGTERM`) MUST call this before
* `process.exit(N)` — otherwise async-buffered records are lost on hard
* exit. No-op when the destination hasn't been constructed yet (logger
* module imported but never emitted) or when called from `_captureLogger`
* test mode (tests use an in-memory destination).
*/
export function flushLoggerSync(): void {
if (!_dest) return;
try {
_dest.flushSync();
} catch {
// Defend against a destination that has already been closed (e.g.,
// double-flush on rapid shutdown). Losing the flush attempt is the
// correct trade-off vs. throwing during shutdown.
}
}
/**
* Idempotent registration: `process.on('beforeExit')` flushes the buffered
* destination before normal exit. Skipped under VITEST to avoid interfering
* with `_captureLogger()`'s lifecycle and vitest's per-worker cleanup.
*/
let _flushHookInstalled = false;
function installFlushHook(): void {
if (_flushHookInstalled) return;
if (isTruthyEnv(process.env.VITEST)) return;
_flushHookInstalled = true;
process.on('beforeExit', () => {
flushLoggerSync();
});
}
/**
* Probe whether `pino-pretty` is resolvable from this module. Cached for
* the lifetime of the process — the resolve cost only happens once, and
* the one-time stderr warning on miss only fires once.
*
* Production installs ship pino-pretty as a runtime dependency (see
* gitnexus/package.json). The probe is the safety net for `--omit=optional`,
* `--no-package-lock` style installs and for any environment where the
* module turns out to be missing for reasons we can't predict — pino's
* own transport-resolution path resolves the target lazily at FIRST log
* write, so without this probe a missing module would throw deep inside
* the pino call site rather than at logger construction.
*/
let _prettyAvailable: boolean | null = null;
const _require = createRequire(import.meta.url);
function isPrettyAvailable(): boolean {
if (_prettyAvailable !== null) return _prettyAvailable;
try {
_require.resolve('pino-pretty');
_prettyAvailable = true;
} catch {
_prettyAvailable = false;
// One-time stderr warning so operators learn why TTY output is plain
// NDJSON instead of pretty-printed. Use realStderrWrite-style direct
// write — going through `logger` here would recurse.
process.stderr.write(
'[gitnexus:logger] pino-pretty unavailable; falling back to NDJSON on stderr\n',
);
}
return _prettyAvailable;
}
/**
* @internal Test-only reset for the pino-pretty availability cache. Lets
* unit tests exercise both resolve outcomes within the same vitest worker.
*/
export function _resetPrettyAvailableCache(): void {
_prettyAvailable = null;
}
/**
* Build the pino-pretty transport options. Internal — exported only so unit
* tests can exercise the probe path without going through `shouldUsePretty()`
* (which is structurally false under vitest).
*/
export function _tryBuildPrettyTransport(): LoggerOptions['transport'] | undefined {
if (!isPrettyAvailable()) return undefined;
return {
target: 'pino-pretty',
options: {
// Route to stderr (fd 2) so pretty output doesn't contaminate
// CLI tool data on stdout (fd 1). pino-pretty's default is fd 1,
// which would interleave with `gitnexus query | jq` output.
destination: 2,
colorize: true,
translateTime: 'SYS:HH:MM:ss.l',
ignore: 'pid,hostname',
},
};
}
/**
* Pino accepts `'fatal' | 'error' | 'warn' | 'info' | 'debug' | 'trace' | 'silent'`.
* Anything else is silently ignored at runtime; we narrow here so a typo in
* the env var produces the documented default rather than masking the issue.
*/
const PINO_LEVELS = new Set(['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent']);
function resolveBaseLevel(): string {
const fromEnv = process.env.GITNEXUS_LOG_LEVEL;
if (fromEnv && PINO_LEVELS.has(fromEnv.toLowerCase())) {
return fromEnv.toLowerCase();
}
return 'info';
}
function buildBaseOptions(): LoggerOptions {
const opts: LoggerOptions = {
level: resolveBaseLevel(),
base: undefined,
};
if (shouldUsePretty()) {
const transport = _tryBuildPrettyTransport();
if (transport) opts.transport = transport;
}
return opts;
}
/**
* Create a named child logger. When `opts.destination` is provided it bypasses
* the default stdout sink (useful for test capture). When `opts.debugEnvVar` is
* set and truthy at call time, the child runs at 'debug' level.
*/
export function createLogger(name: string, opts?: CreateLoggerOptions): Logger {
const debugRequested = opts?.debugEnvVar ? isTruthyEnv(process.env[opts.debugEnvVar]) : false;
if (opts?.destination) {
return pino(
{ level: debugRequested ? 'debug' : 'info', base: undefined, name },
opts.destination,
);
}
const base = buildBaseOptions();
// When using a transport (pino-pretty), pino manages the destination
// internally and we cannot pass one explicitly. When transport is absent,
// route to stderr so stdout stays clean for CLI data output.
let root: Logger;
if (base.transport) {
root = pino({ ...base, level: debugRequested ? 'debug' : base.level });
} else {
root = pino({ ...base, level: debugRequested ? 'debug' : base.level }, defaultDestination());
// The default destination is buffered (`sync: false`); register the
// graceful-exit flush hook now that we know the destination will be
// used. Idempotent — runs at most once per process. Skipped under
// VITEST so test cleanup doesn't fight `_captureLogger`.
installFlushHook();
}
return root.child({ name });
}
/* ------------------------------------------------------------------ */
/* Default singleton (Proxy-backed for test capture) */
/* ------------------------------------------------------------------ */
let _activeDestination: DestinationStream | undefined;
let _cached: Logger | undefined;
function _getInner(): Logger {
if (_cached) return _cached;
// Always go through createLogger so future defaults (serializers, redaction,
// formatters) apply uniformly. The destination override is honored when set
// by `_captureLogger()` below.
_cached = createLogger(
'gitnexus',
_activeDestination ? { destination: _activeDestination } : undefined,
);
return _cached;
}
/**
* Default singleton logger (`name: 'gitnexus'`). Backed by a Proxy so test
* capture (`_captureLogger()`) can redirect output without breaking modules
* that already imported the singleton at module-load time.
*/
export const logger = new Proxy({} as Logger, {
get(_target, prop) {
const inner = _getInner();
// Reflect.get keeps symbol-keyed lookups (e.g. Symbol.toPrimitive) intact;
// a `prop as string` cast would silently coerce them to the wrong key.
const value = Reflect.get(inner as object, prop, inner);
if (typeof value === 'function') {
return (value as (...a: unknown[]) => unknown).bind(inner);
}
return value;
},
}) as Logger;
/**
* Shape of a parsed pino record. `level`, `time`, and `msg` are always
* present; `name` is set when emitted from a named child logger; arbitrary
* additional fields appear when callers pass a structured first arg.
*
* Exported so test helpers and downstream skills can type-narrow capture
* results without inline `Record<string, unknown>` casts.
*/
export interface PinoLogRecord {
level: number;
time: number;
msg: string;
name?: string;
[key: string]: unknown;
}
/**
* In-memory Writable used by `_captureLogger()` and by tests that build
* their own pino destination. Exported so the shape lives in one place
* (previously duplicated between this module and `logger.test.ts`).
*
* `text()` and `records()` are convenience helpers test code calls. They
* don't appear in production hot paths — only test destinations capture
* here — so the surface is intentionally small.
*/
export class MemoryWritable extends Writable {
chunks: string[] = [];
_write(chunk: Buffer | string, _enc: BufferEncoding, cb: (err?: Error | null) => void): void {
this.chunks.push(typeof chunk === 'string' ? chunk : chunk.toString('utf-8'));
cb();
}
/** Concatenate every captured write back into a single string. */
text(): string {
return this.chunks.join('');
}
/** Parse captured writes as one NDJSON record per non-empty line. */
records(): PinoLogRecord[] {
return this.text()
.split('\n')
.filter((l) => l.length > 0)
.map((l) => JSON.parse(l) as PinoLogRecord);
}
}
export interface LoggerCapture {
records(): PinoLogRecord[];
text(): string;
restore(): void;
}
/**
* Test helper. Redirects the default `logger` singleton to an in-memory
* stream and returns a capture object plus a restore function.
*
* Pattern:
* let cap: LoggerCapture;
* beforeEach(() => { cap = _captureLogger(); });
* afterEach(() => { cap.restore(); });
* it('warns', () => {
* fnUnderTest();
* expect(cap.records().some(r => r.msg?.includes('clamping'))).toBe(true);
* });
*
* Not a public API; underscore-prefixed and called only from test code.
* Throws if a previous capture is still active — see the body for context.
*/
export function _captureLogger(): LoggerCapture {
// Guard against double-capture: forgetting `restore()` between two
// `_captureLogger()` calls silently abandoned the previous capture and
// corrupted logger state for the rest of the vitest worker. Throwing here
// surfaces the bug at the moment of misuse instead of as inscrutable
// missing-records assertions in unrelated tests.
if (_activeDestination !== undefined) {
throw new Error(
'_captureLogger: a previous capture is still active — call restore() before starting a new one.',
);
}
const w = new MemoryWritable();
_activeDestination = w;
_cached = undefined;
return {
records: () =>
w.chunks
.join('')
.split('\n')
.filter((l) => l.length > 0)
.map((l) => JSON.parse(l) as PinoLogRecord),
text: () => w.chunks.join(''),
restore: () => {
_activeDestination = undefined;
_cached = undefined;
},
};
}

11
gitnexus/src/core/tree-sitter/parser-loader.ts
View file

@ -2,6 +2,7 @@ import Parser from 'tree-sitter';
import { createRequire } from 'node:module';
import { SupportedLanguages } from 'gitnexus-shared';
import { logger } from '../logger.js';
const _require = createRequire(import.meta.url);
/**
@ -175,8 +176,14 @@ const logFailure = (key: string, result: LoadResult): void => {
logged.add(key);
const message = `[gitnexus] ${result.note} (${result.error.message})`;
if (result.severity === 'error') console.error(message);
else console.warn(message);
// Severity routes to the correct pino level. Both go to stderr (pino's
// default destination), so MCP stdio framing is preserved either way —
// the level tag drives log filtering, not channel selection.
if (result.severity === 'error') {
logger.error(message);
} else {
logger.warn(message);
}
};
export const resolveLanguageKey = (language: SupportedLanguages, filePath?: string): string =>

3
gitnexus/src/core/wiki/cursor-client.ts
View file

@ -10,6 +10,7 @@
import { spawn, execSync } from 'child_process';
import type { LLMResponse, CallLLMOptions } from './llm-client.js';
import { logger } from '../logger.js';
export interface CursorConfig {
model?: string;
workingDirectory?: string;
@ -21,7 +22,7 @@ function isVerbose(): boolean {
function verboseLog(...args: unknown[]): void {
if (isVerbose()) {
console.log('[cursor-cli]', ...args);
logger.info({ args }, '[cursor-cli]');
}
}

9
gitnexus/src/core/wiki/llm-client.ts
View file

@ -1,3 +1,4 @@
import { logger } from '../logger.js';
/**
* LLM Client for Wiki Generation
*
@ -85,8 +86,10 @@ export function isAzureProvider(baseUrl: string): boolean {
const { hostname } = new URL(baseUrl);
return hostname.endsWith('.openai.azure.com') || hostname.endsWith('.services.ai.azure.com');
} catch {
// If URL is malformed, fall back to substring check
return baseUrl.includes('.openai.azure.com') || baseUrl.includes('.services.ai.azure.com');
// Malformed URL — refuse to call this Azure rather than fall back to a
// substring check, which is bypassable by `https://evil.com/?u=.openai.azure.com`
// (CodeQL js/incomplete-url-substring-sanitization).
return false;
}
}
@ -135,7 +138,7 @@ export async function callLLM(
// Warn when using Azure legacy deployment URL without api-version
if (azure && !config.apiVersion && config.baseUrl.includes('/deployments/')) {
console.warn(
logger.warn(
'[gitnexus] Warning: Azure legacy deployment URL detected but no api-version set. Add --api-version 2024-10-21 or use the v1 API format.',
);
}

8
gitnexus/src/mcp/compatible-stdio-transport.ts
View file

@ -4,6 +4,7 @@ import type {
TransportSendOptions,
} from '@modelcontextprotocol/sdk/shared/transport.js';
import { JSONRPCMessageSchema, type JSONRPCMessage } from '@modelcontextprotocol/sdk/types.js';
import { withMcpWrite } from './stdio-context.js';
export type StdioFraming = 'content-length' | 'newline';
@ -232,7 +233,12 @@ export class CompatibleStdioServerTransport implements Transport {
this._stdout.on('error', onError);
if (this._stdout.write(payload)) {
// Tag the write with the MCP transport context so the sentinel
// (server.ts createStdoutSentinel Proxy) recognizes it as a legitimate
// JSON-RPC frame and passes it through to the real stdout instead of
// redirecting to stderr.
const writeOk = withMcpWrite(() => this._stdout.write(payload));
if (writeOk) {
this._stdout.removeListener('error', onError);
resolve();
} else {

49
gitnexus/src/mcp/core/embedder.ts
View file

@ -12,9 +12,14 @@ import {
httpEmbedQuery,
} from '../../core/embeddings/http-client.js';
import { resolveEmbeddingConfig } from '../../core/embeddings/config.js';
import { applyHfEnvOverrides } from '../../core/embeddings/hf-env.js';
import {
applyHfEnvOverrides,
isHfDownloadFailure,
withHfDownloadRetry,
} from '../../core/embeddings/hf-env.js';
import { silenceStdout, restoreStdout, realStderrWrite } from '../../core/lbug/pool-adapter.js';
import { logger } from '../../core/logger.js';
// Model config
const MODEL_ID = 'Snowflake/snowflake-arctic-embed-xs';
@ -51,7 +56,7 @@ export const initEmbedder = async (): Promise<FeatureExtractionPipeline> => {
applyHfEnvOverrides(env);
const embeddingConfig = resolveEmbeddingConfig();
console.error('GitNexus: Loading embedding model (first search may take a moment)...');
logger.info('GitNexus: Loading embedding model (first search may take a moment)...');
const devicesToTry: Array<'dml' | 'cuda' | 'cpu'> =
embeddingConfig.device === 'dml' || embeddingConfig.device === 'cuda'
@ -68,23 +73,39 @@ export const initEmbedder = async (): Promise<FeatureExtractionPipeline> => {
silenceStdout();
process.stderr.write = (() => true) as any;
try {
embedderInstance = await (pipeline as any)('feature-extraction', MODEL_ID, {
device: device,
dtype: 'fp32',
session_options: {
logSeverityLevel: 3,
intraOpNumThreads: embeddingConfig.threads,
interOpNumThreads: 1,
executionMode: 'sequential',
},
});
embedderInstance = await withHfDownloadRetry(() =>
pipeline('feature-extraction', MODEL_ID, {
device: device,
dtype: 'fp32',
session_options: {
logSeverityLevel: 3,
intraOpNumThreads: embeddingConfig.threads,
interOpNumThreads: 1,
executionMode: 'sequential',
},
}),
);
} finally {
restoreStdout();
process.stderr.write = realStderrWrite;
}
console.error(`GitNexus: Embedding model loaded (${device})`);
logger.info({ device }, 'GitNexus: Embedding model loaded');
return embedderInstance!;
} catch {
} catch (deviceError) {
// Network errors and circuit-open errors are not device-specific —
// they will fail the same way on every device. Rethrow immediately
// with actionable HF_ENDPOINT guidance rather than silently falling
// back to the next device.
const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError);
if (isHfDownloadFailure(errMsg)) {
const endpointHint = process.env.HF_ENDPOINT
? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.`
: `huggingface.co may be unreachable from your network.\n` +
` Set HF_ENDPOINT to a mirror and retry:\n` +
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`;
throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`);
}
if (device === 'cpu') throw new Error('Failed to load embedding model');
}
}

8
gitnexus/src/mcp/core/lbug-adapter.ts
View file

@ -1,5 +1,11 @@
/**
* LadybugDB connection pool — re-exported from core.
* Prefer importing from `../../core/lbug/pool-adapter.js` in new code.
*
* KEEP THIS FILE. It is intentionally a shim re-export of
* `../../core/lbug/pool-adapter.js`. The MCP test suite uses this path as
* a vi.mock seam so unit tests can stub LadybugDB without affecting other
* importers of `core/lbug/pool-adapter.js` (which is shared with the
* analyze pipeline). New non-test code MAY import from `pool-adapter.js`
* directly, but the shim must continue to exist for the mock seam to work.
*/
export * from '../../core/lbug/pool-adapter.js';

92
gitnexus/src/mcp/local/local-backend.ts
View file

@ -16,6 +16,7 @@ import {
isLbugReady,
isWriteQuery,
} from '../../core/lbug/pool-adapter.js';
import { isWalCorruptionError, WAL_RECOVERY_SUGGESTION } from '../../core/lbug/lbug-config.js';
export { isWriteQuery };
// Embedding imports are lazy (dynamic import) to avoid loading onnxruntime-node
// at MCP server startup — crashes on unsupported Node ABI versions (#89)
@ -40,7 +41,8 @@ import {
isVectorExtensionSupportedByPlatform,
} from '../../core/platform/capabilities.js';
import { PhaseTimer } from '../../core/search/phase-timer.js';
import { checkStaleness, checkCwdMatch } from '../../core/git-staleness.js';
import { checkStalenessAsync, checkCwdMatch } from '../../core/git-staleness.js';
import { logger } from '../../core/logger.js';
// AI context generation is CLI-only (gitnexus analyze)
// import { generateAIContextFiles } from '../../cli/ai-context.js';
@ -164,29 +166,27 @@ const confidenceForRelType = (relType: string | undefined): number =>
/** Structured error logging for query failures — replaces empty catch blocks */
function logQueryError(context: string, err: unknown): void {
const msg = err instanceof Error ? err.message : String(err);
console.error(`GitNexus [${context}]: ${msg}`);
logger.error({ context, err: msg }, 'GitNexus query failed');
}
/**
* Structured per-query latency log for production aggregation (#553).
* Per-query latency telemetry for production aggregation (#553).
*
* Emitted on stderr — NOT stdout — because the MCP stdio transport uses
* stdout exclusively for JSON-RPC responses (#324), and the CLI e2e test
* `tool output goes to stdout via fd 1` asserts that stdout parses cleanly
* as JSON. Any `console.log` from inside a tool handler would corrupt the
* protocol. Matches the existing `logQueryError` convention above, which
* uses stderr for the same reason.
* Logged at `debug` level — timing is observability/telemetry, not an
* error. Operators wanting per-query timing set `GITNEXUS_LOG_LEVEL=debug`
* (or equivalent). Emitting at `error` level (the original migration
* artifact) caused alerting rules to fire on every successful query and
* inflated stderr noise for every MCP/CLI invocation.
*
* The `GitNexus [query:timing] …` prefix keeps lines greppable; the
* `phases` payload is JSON so log-scraping pipelines can parse it
* without custom format knowledge.
* Emitted via the project logger which routes to stderr — never stdout —
* because the MCP stdio transport uses stdout exclusively for JSON-RPC
* responses (#324) and the CLI e2e test `tool output goes to stdout via
* fd 1` asserts stdout parses cleanly as JSON.
*/
function logQueryTiming(query: string, phases: Record<string, number>): void {
const totalMs = phases.wall ?? Object.values(phases).reduce((a, b) => a + b, 0);
const truncated = query.length > 80 ? `${query.slice(0, 80)}…` : query;
console.error(
`GitNexus [query:timing] query=${JSON.stringify(truncated)} totalMs=${totalMs} phases=${JSON.stringify(phases)}`,
);
logger.debug({ query: truncated, totalMs, phases }, 'GitNexus query timing');
}
export interface CodebaseContext {
@ -287,7 +287,7 @@ export class LocalBackend {
// If kuzu exists but lbug doesn't, warn so the user knows to re-analyze.
const kuzu = await cleanupOldKuzuFiles(storagePath);
if (kuzu.found && kuzu.needsReindex) {
console.error(
logger.error(
`GitNexus: "${entry.name}" has a stale KuzuDB index. Run: gitnexus analyze ${entry.path}`,
);
}
@ -555,8 +555,15 @@ export class LocalBackend {
byRemote.set(h.remoteUrl, list);
}
return handles.map((h) => {
const stale = checkStaleness(h.repoPath, h.lastCommit);
// Check staleness for all repos in parallel instead of sequentially.
// Each check spawns an async `git rev-list` — with 200 repos the sync
// variant took ~50 s; parallel async brings it under a second (#1363).
const stalenessResults = await Promise.all(
handles.map((h) => checkStalenessAsync(h.repoPath, h.lastCommit)),
);
return handles.map((h, i) => {
const stale = stalenessResults[i];
const selfNorm = norm(h.repoPath);
const siblings = h.remoteUrl
? (byRemote.get(h.remoteUrl) ?? []).filter((e) => norm(e.repoPath) !== selfNorm)
@ -637,7 +644,7 @@ export class LocalBackend {
}
this.warnedSiblingDrift.add(cacheKey);
console.error(`GitNexus: ${match.hint}`);
logger.error(`GitNexus: ${match.hint}`);
}
// ─── Tool Dispatch ───────────────────────────────────────────────
@ -990,7 +997,10 @@ export class LocalBackend {
try {
bm25Results = await searchFTSFromLbug(query, limit, repo.id);
} catch (err: any) {
console.error('GitNexus: BM25/FTS search failed (FTS indexes may not exist) -', err.message);
logger.error(
{ err: err.message },
'GitNexus: BM25/FTS search failed (FTS indexes may not exist) -',
);
return { results: [], ftsUsed: false };
}
@ -1114,7 +1124,7 @@ export class LocalBackend {
// policy. Emitted once per `LocalBackend` instance lifetime to avoid
// noisy stderr on hot semantic-search paths (DoD §2.8).
this.warnedVectorUnsupported = true;
console.error(
logger.warn(
'GitNexus [query:vector]: VECTOR extension not supported on this platform; using exact scan fallback',
);
}
@ -1216,7 +1226,14 @@ export class LocalBackend {
const result = await executeQuery(repo.id, params.query);
return result;
} catch (err: any) {
return { error: err.message || 'Query failed' };
const msg = err.message || 'Query failed';
if (isWalCorruptionError(err)) {
return {
error: msg,
recoverySuggestion: WAL_RECOVERY_SUGGESTION,
};
}
return { error: msg };
}
}
@ -1670,6 +1687,30 @@ export class LocalBackend {
kind?: string;
include_content?: boolean;
},
): Promise<any> {
try {
return await this._contextImpl(repo, params);
} catch (err: any) {
const msg = (err instanceof Error ? err.message : String(err)) || 'Context query failed';
if (isWalCorruptionError(err)) {
return {
error: msg,
recoverySuggestion: WAL_RECOVERY_SUGGESTION,
};
}
throw err;
}
}
private async _contextImpl(
repo: RepoHandle,
params: {
name?: string;
uid?: string;
file_path?: string;
kind?: string;
include_content?: boolean;
},
): Promise<any> {
await this.ensureInitialized(repo.id);
@ -2431,6 +2472,7 @@ export class LocalBackend {
impactedCount: 0,
risk: 'UNKNOWN',
suggestion: 'The graph query failed — try gitnexus context <symbol> as a fallback',
...(isWalCorruptionError(err) ? { recoverySuggestion: WAL_RECOVERY_SUGGESTION } : {}),
};
}
}
@ -2982,8 +3024,14 @@ export class LocalBackend {
relationTypes: string[];
minConfidence: number;
includeTests: boolean;
signal?: AbortSignal;
},
): Promise<any | null> {
// Honor an already-aborted signal at the entry boundary as a fast
// path. Cooperative cancellation inside _runImpactBFS is out of
// scope — the caller's Promise.race against the same signal
// resolves the await regardless of how long this body runs.
if (opts.signal?.aborted) return null;
try {
await this.refreshRepos();
await this.ensureInitialized(repoId);

34
gitnexus/src/mcp/server.ts
View file

@ -24,7 +24,7 @@ import {
GetPromptRequestSchema,
} from '@modelcontextprotocol/sdk/types.js';
import { GITNEXUS_TOOLS } from './tools.js';
import { realStdoutWrite } from './core/lbug-adapter.js';
import { installGlobalStdoutSentinel } from './stdio-context.js';
import type { LocalBackend } from './local/local-backend.js';
import { getResourceDefinitions, getResourceTemplates, readResource } from './resources.js';
@ -287,20 +287,36 @@ Follow these steps:
export async function startMCPServer(backend: LocalBackend): Promise<void> {
const server = createMCPServer(backend);
// Use the shared stdout reference captured at module-load time by the
// lbug-adapter. Avoids divergence if anything patches stdout between
// module load and server start.
const _safeStdout = new Proxy(process.stdout, {
// Idempotent global sentinel install. cli/mcp.ts calls this first thing
// (before warnMissingOptionalGrammars / backend.init can emit to stdout);
// calling again here is a safety net for direct callers of startMCPServer
// (tests, future entry points). The transport's _safeStdout Proxy is a
// second layer that guarantees transport writes reach the sentinel even
// if anything else re-replaces process.stdout.write later. Tagged
// transport writes (wrapped in withMcpWrite by compatible-stdio-transport.send)
// pass through to the captured realStdoutWrite; untagged writes reaching
// the Proxy or process.stdout get redirected to stderr with the
// [mcp:stdout-redirect] prefix. See stdio-context.ts.
const sentinel = installGlobalStdoutSentinel();
const safeStdout = new Proxy(process.stdout, {
get(target, prop, receiver) {
if (prop === 'write') return realStdoutWrite;
if (prop === 'write') return sentinel.write;
const val = Reflect.get(target, prop, receiver);
return typeof val === 'function' ? val.bind(target) : val;
},
});
const transport = new CompatibleStdioServerTransport(process.stdin, _safeStdout);
const transport = new CompatibleStdioServerTransport(process.stdin, safeStdout);
await server.connect(transport);
// Graceful shutdown helper
// Surface the redirect counter on shutdown so users see the volume of
// stray writes even when individual payloads were truncated/suppressed.
process.on('exit', () => sentinel.flushSummary());
// Graceful shutdown helper. Pino's default destination is `sync: false`
// (buffered), so we must `flushLoggerSync()` before `process.exit` —
// otherwise records emitted during disconnect/close are lost. The flush
// is a no-op when the singleton was never used or when running under
// vitest. See `gitnexus/src/core/logger.ts`.
let shuttingDown = false;
const shutdown = async (exitCode = 0) => {
if (shuttingDown) return;
@ -311,6 +327,8 @@ export async function startMCPServer(backend: LocalBackend): Promise<void> {
try {
await server.close();
} catch {}
const { flushLoggerSync } = await import('../core/logger.js');
flushLoggerSync();
process.exit(exitCode);
};

61
gitnexus/src/mcp/stdio-capture.ts Normal file
View file

@ -0,0 +1,61 @@
/**
* Stdio capture — leaf module with zero non-`node:` imports.
*
* Owns the singleton state that the MCP stdout sentinel needs:
* - `realStdoutWrite` / `realStderrWrite`: process.stdout.write /
* process.stderr.write captured at module load, BEFORE anything else
* can rebind them.
* - `activeStdoutWrite`: the write handler that silenceStdout/restoreStdout
* cycles in pool-adapter restore to. Defaults to `realStdoutWrite`;
* `installGlobalStdoutSentinel` (in stdio-context.ts) registers the
* sentinel here at MCP startup so silence/restore preserves the sentinel.
*
* This module exists separately from `pool-adapter.ts` (which previously
* owned the same state) so that `cli/mcp.ts`'s static-import closure does
* NOT transitively pull in `@ladybugdb/core`. Codex's adversarial review on
* PR #1383 found that the prior structure left a pre-sentinel window where
* native-module init banners could reach raw stdout: `cli/mcp.ts` →
* `mcp/stdio-context.ts` → `core/lbug/pool-adapter.ts` → `@ladybugdb/core`.
* Routing the sentinel state through this leaf module breaks that chain.
*
* **Constraint:** keep this module a leaf. No non-`node:` imports — adding
* any would re-introduce the import-time stdout-corruption hazard.
*/
type StdoutWrite = typeof process.stdout.write;
/** Captured at module load, before any rebinding. */
// eslint-disable-next-line no-restricted-syntax -- this IS the captured-real-write infrastructure used by the MCP sentinel
export const realStdoutWrite: StdoutWrite = process.stdout.write.bind(process.stdout);
export const realStderrWrite: typeof process.stderr.write = process.stderr.write.bind(
process.stderr,
);
/**
* The function `restoreStdout` (and the watchdog) in pool-adapter restore
* *to* when un-silencing. Defaults to the captured real write; the MCP
* server registers its sentinel here at startMCPServer (via
* installGlobalStdoutSentinel) so silenceStdout cycles preserve the sentinel
* instead of unwinding to raw stdout.
*/
let activeStdoutWrite: StdoutWrite = realStdoutWrite;
/**
* Register a wrapper (e.g., the MCP sentinel) as the active stdout write.
* silenceStdout/restoreStdout cycles in pool-adapter will preserve the
* wrapper instead of unwinding to the raw realStdoutWrite. Returns the
* previous value so callers can chain or restore.
*/
export function setActiveStdoutWrite(fn: StdoutWrite): StdoutWrite {
const prev = activeStdoutWrite;
activeStdoutWrite = fn;
return prev;
}
/**
* Read the currently-active stdout write handler. Used by pool-adapter's
* restoreStdout and watchdog so silence/restore preserves the sentinel.
*/
export function getActiveStdoutWrite(): StdoutWrite {
return activeStdoutWrite;
}

183
gitnexus/src/mcp/stdio-context.ts Normal file
View file

@ -0,0 +1,183 @@
/**
* MCP Stdio Context — AsyncLocalStorage-tagged transport-write detection.
*
* The MCP stdio transport writes JSON-RPC frames to stdout. Per spec, the
* server MUST NOT write anything to stdout that is not a valid MCP message.
* Stray writes from dependency code corrupt the protocol and present to
* clients as a hung handshake or `MCP error -32000`.
*
* This module provides:
* - withMcpWrite(fn): runs fn inside an AsyncLocalStorage context tagged
* `mcp: true`. The transport wraps every send() in this so its writes
* are recognizable as legitimate.
* - isMcpWrite(): true when called inside withMcpWrite.
* - createStdoutSentinel({...}): a write function suitable for installing
* in a Proxy over process.stdout. Tagged writes pass through to the real
* stdout; untagged writes are redirected to stderr with a [mcp:stdout-redirect]
* prefix, truncated to maxBytes per redirect, and rate-limited to maxRedirects
* per process so a stray loop cannot flood client logs.
*
* The sentinel is correctness-by-construction: it identifies legitimate
* writes by *who* called write(), not by inspecting the bytes. A byte-shape
* heuristic ("starts with {, ends with \n") would falsely reject Content-Length
* frames (which start with C and end with }) and misclassify multi-chunk writes.
*/
import { AsyncLocalStorage } from 'node:async_hooks';
// Import from the leaf module, NOT `core/lbug/pool-adapter.js`. pool-adapter
// pulls in `@ladybugdb/core`, which would put the native module in
// `cli/mcp.ts`'s static-import closure — exactly the pre-sentinel window
// Codex's adversarial review flagged on PR #1383.
import { realStdoutWrite, realStderrWrite, setActiveStdoutWrite } from './stdio-capture.js';
interface McpWriteContext {
mcp: true;
}
const store = new AsyncLocalStorage<McpWriteContext>();
export function withMcpWrite<T>(fn: () => T): T {
return store.run({ mcp: true }, fn);
}
export function isMcpWrite(): boolean {
return store.getStore()?.mcp === true;
}
type WriteFn = typeof process.stdout.write;
export interface SentinelOptions {
realStdoutWrite: WriteFn;
realStderrWrite: WriteFn;
/** Maximum bytes of payload to surface per redirect. Defaults to 200. */
maxBytes?: number;
/** Maximum number of redirects per process before suppression. Defaults to 10. */
maxRedirects?: number;
}
export interface SentinelStats {
redirected: number;
suppressed: number;
}
export interface Sentinel {
write: WriteFn;
stats: () => SentinelStats;
flushSummary: () => void;
}
const REDIRECT_PREFIX = '[mcp:stdout-redirect] ';
const STARTUP_WARNING =
'[mcp:stdout-redirect] sentinel triggered — stray write redirected to stderr; subsequent redirects logged at exit\n';
function chunkToBuffer(chunk: any): Buffer {
if (chunk === undefined || chunk === null) return Buffer.alloc(0);
if (Buffer.isBuffer(chunk)) return chunk;
if (typeof chunk === 'string') return Buffer.from(chunk, 'utf8');
// Plain Uint8Array (e.g. from a TypedArray-using producer): copy bytes
// verbatim instead of falling through to String(chunk), which produces
// garbage like "1,2,3,...".
if (chunk instanceof Uint8Array) return Buffer.from(chunk);
return Buffer.from(String(chunk), 'utf8');
}
/**
* Node Writable.write contract: the completion callback, when present, is
* always the last argument. Match exactly that — don't try to peer past
* earlier arguments — so future overload shapes (e.g. an options object)
* do not silently break callback delivery.
*/
function extractCallback(rest: unknown[]): ((err?: Error | null) => void) | undefined {
const last = rest[rest.length - 1];
return typeof last === 'function' ? (last as (err?: Error | null) => void) : undefined;
}
export function createStdoutSentinel(opts: SentinelOptions): Sentinel {
const maxBytes = opts.maxBytes ?? 200;
const maxRedirects = opts.maxRedirects ?? 10;
let redirected = 0;
let suppressed = 0;
let warningEmitted = false;
const stderr = (s: string | Buffer) => opts.realStderrWrite(s);
const write: WriteFn = (chunk: any, ...rest: any[]): boolean => {
if (isMcpWrite()) {
return opts.realStdoutWrite(chunk, ...rest);
}
if (!warningEmitted) {
warningEmitted = true;
stderr(STARTUP_WARNING);
}
if (redirected < maxRedirects) {
redirected += 1;
const buf = chunkToBuffer(chunk);
const truncated = buf.length > maxBytes ? buf.subarray(0, maxBytes) : buf;
stderr(REDIRECT_PREFIX);
if (truncated.length > 0) stderr(truncated);
if (buf.length > maxBytes) {
stderr(` (+${buf.length - maxBytes} bytes truncated)`);
}
if (truncated.length === 0 || truncated[truncated.length - 1] !== 0x0a) {
stderr('\n');
}
} else {
suppressed += 1;
}
// Honor the Writable.write callback contract — fire async to match
// Node's "next-tick" semantics so callers never observe sync reentry.
const cb = extractCallback(rest);
if (cb) {
process.nextTick(() => cb(null));
}
return true;
};
return {
write,
stats: () => ({ redirected, suppressed }),
flushSummary: () => {
if (redirected === 0 && suppressed === 0) return;
stderr(
`[mcp:stdout-redirect] summary: ${redirected} redirected, ${suppressed} suppressed beyond cap\n`,
);
},
};
}
/**
* Install the sentinel as the global stdout interceptor — idempotent.
*
* Does three things in order:
* 1. Creates the sentinel from the captured `realStdoutWrite` / `realStderrWrite`.
* 2. Replaces `process.stdout.write` with `sentinel.write`.
* 3. Registers `sentinel.write` as the "active" handler in pool-adapter
* so silenceStdout/restoreStdout cycles preserve the sentinel
* instead of unwinding to raw stdout.
*
* Idempotent — callers may invoke it multiple times safely (cli/mcp.ts at
* the top of mcpCommand, and startMCPServer). The earliest caller wins;
* subsequent calls return the same sentinel handle. Call this BEFORE any
* other startup work that might emit to stdout: native module loads,
* `_require()`-style grammar detection, repo registry reads, embedder
* pipeline initialization. Anything written before the sentinel is in
* place reaches raw stdout uncaught.
*
* Returns the sentinel handle so the earliest caller can register
* `process.on('exit', sentinel.flushSummary)`.
*/
let _installedSentinel: Sentinel | null = null;
export function installGlobalStdoutSentinel(): Sentinel {
if (_installedSentinel) return _installedSentinel;
const sentinel = createStdoutSentinel({ realStdoutWrite, realStderrWrite });
// eslint-disable-next-line no-restricted-syntax -- installing the global sentinel is the API contract
process.stdout.write = sentinel.write;
setActiveStdoutWrite(sentinel.write);
_installedSentinel = sentinel;
return sentinel;
}

34
gitnexus/src/server/api.ts
View file

@ -27,8 +27,6 @@ import { isWriteQuery } from '../core/lbug/pool-adapter.js';
import { NODE_TABLES, type GraphNode, type GraphRelationship } from 'gitnexus-shared';
import { searchFTSFromLbug } from '../core/search/bm25-index.js';
import { hybridSearch } from '../core/search/hybrid-search.js';
// Embedding imports are lazy (dynamic import) to avoid loading onnxruntime-node
// at server startup — crashes on unsupported Node ABI versions (#89)
import { LocalBackend } from '../mcp/local/local-backend.js';
import { mountMCPEndpoints } from './mcp-http.js';
import { fork } from 'child_process';
@ -36,6 +34,7 @@ import { fileURLToPath, pathToFileURL } from 'url';
import { JobManager } from './analyze-job.js';
import { assertString, escapeRegExp, BadRequestError, createRouteLimiter } from './validation.js';
import { extractRepoName, getCloneDir, cloneOrPull } from './git-clone.js';
import { logger, flushLoggerSync } from '../core/logger.js';
const _require = createRequire(import.meta.url);
const pkg = _require('../../package.json');
@ -143,7 +142,7 @@ export const resolveWebDistDir = async (
return dir;
} catch (err: any) {
if (err?.code !== 'ENOENT') {
console.warn(`[serve] could not access web UI dir ${dir}:`, err.message);
logger.warn({ err: err.message }, `[serve] could not access web UI dir ${dir}:`);
}
}
}
@ -1490,7 +1489,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
});
})
.catch((err) => {
console.error('backend.init() failed after analyze:', err);
logger.error({ err }, 'backend.init() failed after analyze:');
jobManager.updateJob(job.id, {
status: 'failed',
error: 'Server failed to reload after analysis. Try again.',
@ -1522,7 +1521,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
j.retryCount++;
const delay = 1000 * Math.pow(2, j.retryCount - 1); // 1s, 2s
const lastErr = stderrChunks.trim().split('\n').pop() || '';
console.warn(
logger.warn(
`Analyze worker crashed (code ${code}), retry ${j.retryCount}/${MAX_WORKER_RETRIES} in ${delay}ms` +
(lastErr ? `: ${lastErr}` : ''),
);
@ -1790,7 +1789,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
// Global error handler — catch anything the route handlers miss
app.use((err: any, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
console.error('Unhandled error:', err);
logger.error({ err }, 'Unhandled error:');
res.status(500).json({ error: 'Internal server error' });
});
@ -1804,7 +1803,9 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
});
server.on('error', (err) => reject(err));
// Graceful shutdown — close Express + LadybugDB cleanly
// Graceful shutdown — close Express + LadybugDB cleanly. Pino's default
// destination is `sync: false` (buffered); `flushLoggerSync()` before
// `process.exit` so records emitted during cleanup reach stderr.
const shutdown = async () => {
console.log('\nShutting down...');
server.close();
@ -1813,22 +1814,33 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
await cleanupMcp();
await closeLbug();
await backend.disconnect();
const { flushLoggerSync } = await import('../core/logger.js');
flushLoggerSync();
process.exit(0);
};
process.once('SIGINT', shutdown);
process.once('SIGTERM', shutdown);
// Catch-all crash guards (mirrors startMCPServer in mcp/server.ts)
// Catch-all crash guards (mirrors startMCPServer in mcp/server.ts).
// Pino v10's default destination is buffered (`sync: false`) — call
// `flushLoggerSync()` after logging and before triggering shutdown
// so the crash record reaches stderr regardless of how cleanup goes.
// Worker-thread transports (pino-pretty under TTY) handle their own
// flush on process exit in v10. `pino.final` was removed in v10
// because the new transport architecture made it unnecessary.
let shuttingDown = false;
process.on('uncaughtException', (err) => {
console.error('GitNexus uncaughtException:', err?.stack || err);
logger.error({ err }, 'GitNexus uncaughtException');
flushLoggerSync();
if (!shuttingDown) {
shuttingDown = true;
shutdown().catch(() => {});
}
});
process.on('unhandledRejection', (reason: any) => {
console.error('GitNexus unhandledRejection:', reason?.stack || reason);
process.on('unhandledRejection', (reason: unknown) => {
// Availability-first: log the rejection without exiting.
const err = reason instanceof Error ? reason : new Error(String(reason));
logger.error({ err }, 'GitNexus unhandledRejection');
});
});
};

3
gitnexus/src/server/git-clone.ts
View file

@ -10,6 +10,7 @@ import path from 'path';
import os from 'os';
import fs from 'fs/promises';
import { isIP } from 'net';
import { logger } from '../core/logger.js';
/** Root directory for all cloned repositories. Targets must resolve inside this. */
const CLONE_ROOT = path.resolve(path.join(os.homedir(), '.gitnexus', 'repos'));
@ -446,7 +447,7 @@ function runGit(args: string[], cwd?: string): Promise<void> {
if (code === 0) resolve();
else {
// Log full stderr internally but don't expose it to API callers (SSRF mitigation)
if (stderr.trim()) console.error(`git ${args[0]} stderr: ${stderr.trim()}`);
if (stderr.trim()) logger.error(`git ${args[0]} stderr: ${stderr.trim()}`);
reject(new Error(`git ${args[0]} failed (exit code ${code})`));
}
});

3
gitnexus/src/server/mcp-http.ts
View file

@ -15,6 +15,7 @@ import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { createMCPServer } from '../mcp/server.js';
import type { LocalBackend } from '../mcp/local/local-backend.js';
import { randomUUID } from 'crypto';
import { logger } from '../core/logger.js';
interface MCPSession {
server: Server;
@ -87,7 +88,7 @@ export function mountMCPEndpoints(app: Express, backend: LocalBackend): () => Pr
app.all('/api/mcp', (req: Request, res: Response) => {
void handleMcpRequest(req, res).catch((err: any) => {
console.error('MCP HTTP request failed:', err);
logger.error({ err }, 'MCP HTTP request failed:');
if (res.headersSent) return;
res.status(500).json({
jsonrpc: '2.0',

7
gitnexus/test/helpers/test-db.ts
View file

@ -37,6 +37,13 @@ export async function cleanupTempDir(tmpDir: string): Promise<void> {
/**
* Create a temporary directory for LadybugDB tests.
* Returns the path and a cleanup function.
*
* IMPORTANT: when adding a new test that passes a custom `prefix`, also add
* the prefix to `TEST_FIXTURE_PREFIXES` in
* `gitnexus/src/core/lbug/lbug-config.ts`. The stale-sidecar sweep relies
* on the prefix list to recognize test fixtures; an unknown prefix means
* the sweep silently won't fire for that fixture and Windows CI flakes
* return.
*/
export async function createTempDir(prefix: string = 'gitnexus-test-'): Promise<TestDBHandle> {
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), prefix));

122
gitnexus/test/integration/cli/tool-no-index-stderr.test.ts Normal file
View file

@ -0,0 +1,122 @@
/**
* Regression test for the buffered-pino + hard-exit diagnostic-loss bug
* (Codex adversarial review on PR #1336, plan 002).
*
* Symptom before the fix: `gitnexus tool query <foo>` with no indexed
* repos exits non-zero with EMPTY stderr — the `logger.error()` call was
* routed through pino's `sync: false` SonicBoom buffer, and the
* subsequent synchronous `process.exit(1)` killed the process before the
* buffer could drain. Operators saw a silent failure.
*
* The fix routes user-facing CLI diagnostics through `cliError` (in
* `gitnexus/src/cli/cli-message.ts`), which writes plain text directly
* to `process.stderr` AND tees a structured pino record. Direct stderr
* writes don't go through the buffer, so they survive `process.exit`.
*
* This test spawns the built CLI in a child process and asserts the
* diagnostic line reaches stderr before exit. Without the fix it fails;
* with the fix it passes. Characterization-first contract, locked in
* end-to-end against `dist/`.
*/
import { describe, it, expect } from 'vitest';
import { spawn } from 'node:child_process';
import path from 'node:path';
import fs from 'node:fs';
import os from 'node:os';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(__dirname, '..', '..', '..');
const DIST_CLI = path.join(REPO_ROOT, 'dist', 'cli', 'index.js');
const CHILD_TIMEOUT_MS = process.env.CI ? 20_000 : 10_000;
interface ChildResult {
exitCode: number | null;
stdout: string;
stderr: string;
}
/**
* Spawn the built `gitnexus` CLI with arguments, wait for exit, and
* return captured streams + exit code. Pin GITNEXUS_HOME to a fresh
* empty temp dir so the LocalBackend init reliably finds zero indexed
* repos. Force NODE_OPTIONS empty to prevent host-environment overrides
* from changing buffer / heap behavior (plan 001 U3 added the buffered
* destination, which is what this test guards against).
*/
function runCli(args: string[]): Promise<ChildResult> {
const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-cli-no-index-'));
return new Promise<ChildResult>((resolve, reject) => {
const proc = spawn(process.execPath, [DIST_CLI, ...args], {
cwd: REPO_ROOT,
env: {
...process.env,
GITNEXUS_HOME: tmpHome,
NODE_OPTIONS: '',
// Force NDJSON path: pino-pretty only activates when stderr is a
// TTY and !CI && !VITEST. spawn() pipes stderr, so it's not a
// TTY in this child anyway, but the explicit env is defense-in-depth.
CI: '1',
},
stdio: ['pipe', 'pipe', 'pipe'],
});
const stdoutChunks: Buffer[] = [];
const stderrChunks: Buffer[] = [];
proc.stdout.on('data', (chunk: Buffer) => stdoutChunks.push(chunk));
proc.stderr.on('data', (chunk: Buffer) => stderrChunks.push(chunk));
const timer = setTimeout(() => {
proc.kill('SIGKILL');
reject(new Error(`child process exceeded ${CHILD_TIMEOUT_MS}ms timeout`));
}, CHILD_TIMEOUT_MS);
proc.on('close', (code) => {
clearTimeout(timer);
// Best-effort cleanup of the empty temp home; ignore errors so they
// don't mask test failures.
try {
fs.rmSync(tmpHome, { recursive: true, force: true });
} catch {
/* ignore */
}
resolve({
exitCode: code,
stdout: Buffer.concat(stdoutChunks).toString('utf8'),
stderr: Buffer.concat(stderrChunks).toString('utf8'),
});
});
proc.on('error', (err) => {
clearTimeout(timer);
reject(err);
});
});
}
describe('CLI tool query — diagnostic survives hard exit (plan 002)', () => {
it('emits the no-index diagnostic to stderr before exit code 1', async () => {
if (!fs.existsSync(DIST_CLI)) {
throw new Error(
`dist/cli/index.js missing — run \`npm run build\` first (or use \`npm run test:integration\` which builds via pretest:integration).`,
);
}
const result = await runCli(['query', 'whatever']);
// Without the plan-002 fix, stderr was empty. The diagnostic must be
// visible regardless of how `process.exit(1)` interacts with the
// buffered pino destination.
expect(result.stderr).toContain('No indexed repositories found');
expect(result.stderr).toContain('gitnexus analyze');
// Exit code stays 1 — we're only changing the message channel, not
// the failure semantics.
expect(result.exitCode).toBe(1);
// Stdout should not carry the diagnostic. CLI tool data is reserved
// for stdout (e.g., `gitnexus query | jq`); diagnostics are stderr.
expect(result.stdout).not.toContain('No indexed repositories found');
}, 30_000);
});

Some files were not shown because too many files have changed in this diff Show more