diff --git a/.github/workflows/build-tree-sitter-prebuilds.yml b/.github/workflows/build-tree-sitter-prebuilds.yml index b592ebe16..d6e93536d 100644 --- a/.github/workflows/build-tree-sitter-prebuilds.yml +++ b/.github/workflows/build-tree-sitter-prebuilds.yml @@ -123,7 +123,7 @@ jobs: matrix: ${{ steps.decide.outputs.matrix }} release_app: ${{ steps.relapp.outputs.configured }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 # need base history to diff recorded versions persist-credentials: false @@ -392,7 +392,7 @@ jobs: # and compiling them under emulation on the arm runners is slow. timeout-minutes: 45 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # this job uploads artifacts (artipacked) @@ -565,7 +565,7 @@ jobs: app-id: ${{ secrets.RELEASE_APP_ID }} private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: token: ${{ steps.app-token.outputs.token }} # On a (non-fork) PR, check out the PR's HEAD branch — not the merge ref — diff --git a/.github/workflows/ci-devcontainer.yml b/.github/workflows/ci-devcontainer.yml index 4a7eeb39a..73fd4e1ae 100644 --- a/.github/workflows/ci-devcontainer.yml +++ b/.github/workflows/ci-devcontainer.yml @@ -36,7 +36,7 @@ jobs: # persist-credentials: false — this job only reads (tests and syntax # checks) and never pushes. The setting keeps GITHUB_TOKEN out of # .git/config, which zizmor flags as the "artipacked" issue. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -57,7 +57,7 @@ jobs: # persist-credentials: false — this is a read-only build smoke that # never pushes. The setting keeps GITHUB_TOKEN out of .git/config, # which zizmor flags as the "artipacked" issue. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index a30371637..b8d813f9e 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -14,7 +14,7 @@ jobs: outputs: web_changed: ${{ steps.filter.outputs.web }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v3 @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index 1a2ac4728..1b7185d1b 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -13,7 +13,7 @@ jobs: # canceled prettier at the 5-minute job cap; lint needed 7m41s the same run. timeout-minutes: 10 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # tsc --noEmit reads source + gitnexus-shared/dist. Skip prepare/postinstall @@ -61,7 +61,7 @@ jobs: # run is cold again. timeout-minutes: 15 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus-web @@ -84,7 +84,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Validate workflow concurrency convention diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml index f1afd43b7..a16cb1103 100644 --- a/.github/workflows/ci-report.yml +++ b/.github/workflows/ci-report.yml @@ -125,7 +125,7 @@ jobs: - name: Checkout (for vitest config) if: steps.meta.outputs.skip != 'true' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: sparse-checkout: gitnexus/vitest.config.ts sparse-checkout-cone-mode: false diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index ee6f85511..e6d93f74b 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -36,7 +36,7 @@ jobs: # persist-credentials: false — runs tests + uploads a blob artifact; the # default-persisted token must not be capturable through it (zizmor # credential-persistence / artipacked audit). The job never pushes. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -99,7 +99,7 @@ jobs: env: GITNEXUS_REQUIRE_FTS: '1' steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -223,7 +223,7 @@ jobs: steps: # persist-credentials: false — runs tests only, never pushes (zizmor # credential-persistence / artipacked audit). - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -273,7 +273,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 20 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -317,7 +317,7 @@ jobs: # from a tarball and never pushes back; the token in .git/config would # be at risk of leaking through any future artifact-upload step # (zizmor artipacked audit). Disable upfront. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # Skip prepare/postinstall/build here. `npm pack` runs prepack, which @@ -430,7 +430,7 @@ jobs: steps: # persist-credentials: false — builds and import-links only, never pushes # (zizmor credential-persistence / artipacked audit). - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -496,7 +496,7 @@ jobs: # and never pushes; the default-persisted token in .git/config would be at # risk of leaking through an artifact upload (zizmor credential-persistence # / artipacked audit). Mirrors the packaged-install-smoke job below. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -907,10 +907,10 @@ jobs: timeout-minutes: 15 steps: # persist-credentials: false — runs tests only, never pushes. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: version: '0.11.23' python-version: '3.13' @@ -934,7 +934,7 @@ jobs: GITNEXUS_REQUIRE_FULL_SWEEP: '1' GITNEXUS_REQUIRE_CLAUDE_CANARY: '1' steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -944,7 +944,7 @@ jobs: cache-dependency-path: | gitnexus/package-lock.json gitnexus-shared/package-lock.json - - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: version: '0.11.23' python-version: '3.13' @@ -1002,10 +1002,10 @@ jobs: runs-on: windows-latest timeout-minutes: 15 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: version: '0.11.23' python-version: '3.13' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 0856c132b..ffc6c77e7 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -129,7 +129,7 @@ jobs: core.setOutput('code_review', isCodeReview ? 'true' : 'false'); - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.repo || github.repository }} ref: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.sha || '' }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 9075d8c90..b22dc7650 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -42,13 +42,13 @@ jobs: steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Don't leave GITHUB_TOKEN in .git/config for downstream steps to read. persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 + uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: languages: ${{ matrix.language }} queries: security-and-quality @@ -87,6 +87,6 @@ jobs: - '.github/scripts/fetch-lbug-fts-artifacts.mjs' - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 + uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/commit-fork-prebuilds.yml b/.github/workflows/commit-fork-prebuilds.yml index 3b6ef78d3..a45a3f5ba 100644 --- a/.github/workflows/commit-fork-prebuilds.yml +++ b/.github/workflows/commit-fork-prebuilds.yml @@ -145,7 +145,7 @@ jobs: # checkout (the same trust anchor as this workflow file). - name: Checkout identity verifier if: steps.meta.outputs.deliver == 'true' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false sparse-checkout: .github/scripts/verify-workflow-run-pr-identity.cjs @@ -171,7 +171,7 @@ jobs: # never written to .git/config on disk. - name: Checkout fork PR head if: steps.meta.outputs.deliver == 'true' && steps.verify.outcome == 'success' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ steps.verify.outputs.head_repo }} ref: ${{ steps.verify.outputs.head_sha }} diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 58740cd09..714627717 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 80e4588a8..0e43de45e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -103,7 +103,7 @@ jobs: # When triggered by workflow_call the caller passes the RC tag as an input; # we check out that tag so the Dockerfile and package.json match the built image. # For tag-push events github.ref is already the tag ref — no override needed. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs.tag || github.ref }} diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index 3f0c50f4e..861b37ff2 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -29,7 +29,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Full history needed for the on-push full-history scan; on PRs the # action diffs against the base ref so the cost is bounded by the PR. diff --git a/.github/workflows/gitnexus-review-agent.yml b/.github/workflows/gitnexus-review-agent.yml index 8ed65559e..5a4b4a2ea 100644 --- a/.github/workflows/gitnexus-review-agent.yml +++ b/.github/workflows/gitnexus-review-agent.yml @@ -303,7 +303,7 @@ jobs: - name: Checkout trusted workflow control plane id: checkout-control if: steps.context.outputs.ready == 'true' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ github.repository }} ref: ${{ steps.context.outputs.control_sha }} @@ -315,7 +315,7 @@ jobs: - name: Checkout exact PR head as passive data id: checkout-head if: steps.context.outputs.ready == 'true' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ steps.context.outputs.head_repo }} ref: ${{ steps.context.outputs.head_sha }} @@ -1294,7 +1294,7 @@ jobs: steps.claude-recheck.outcome == 'success' # Use the low-level base action: the high-level GitHub action can restore # project configuration from a moving base branch before invoking Claude. - uses: anthropics/claude-code-action/base-action@3553f84341b92da26052e28acf1aa898f9511f32 # v1 + uses: anthropics/claude-code-action/base-action@e0cf66d1d257526b5d07f141838c338921cb8455 # v1 env: CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: '1' CLAUDE_CODE_ADDITIONAL_DIRECTORIES_CLAUDE_MD: '0' @@ -1447,7 +1447,7 @@ jobs: if: >- steps.precheck.outputs.repair_reason != '' && steps.repair-recheck.outcome == 'success' - uses: anthropics/claude-code-action/base-action@3553f84341b92da26052e28acf1aa898f9511f32 # v1 + uses: anthropics/claude-code-action/base-action@e0cf66d1d257526b5d07f141838c338921cb8455 # v1 env: CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: '1' CLAUDE_CODE_ADDITIONAL_DIRECTORIES_CLAUDE_MD: '0' diff --git a/.github/workflows/gitnexus-skill-evolution.yml b/.github/workflows/gitnexus-skill-evolution.yml index 03b3655ca..867ce49a3 100644 --- a/.github/workflows/gitnexus-skill-evolution.yml +++ b/.github/workflows/gitnexus-skill-evolution.yml @@ -252,7 +252,7 @@ jobs: exit 1 fi - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 @@ -265,7 +265,7 @@ jobs: gitnexus/package-lock.json gitnexus-shared/package-lock.json - - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: version: '0.11.23' python-version: '3.13' diff --git a/.github/workflows/grammar-update-monitor.yml b/.github/workflows/grammar-update-monitor.yml index 7380af269..7cc8313a1 100644 --- a/.github/workflows/grammar-update-monitor.yml +++ b/.github/workflows/grammar-update-monitor.yml @@ -44,7 +44,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/impact-pdg-mutation-report.yml b/.github/workflows/impact-pdg-mutation-report.yml index 42903acb9..e09970da4 100644 --- a/.github/workflows/impact-pdg-mutation-report.yml +++ b/.github/workflows/impact-pdg-mutation-report.yml @@ -37,7 +37,7 @@ jobs: # artifact and never pushes; the default-persisted token in .git/config # must not be capturable through that upload (zizmor credential-persistence # / artipacked audit). Mirrors ci-tests.yml. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/pr-autofix-apply.yml b/.github/workflows/pr-autofix-apply.yml index f612d3d31..78e90cced 100644 --- a/.github/workflows/pr-autofix-apply.yml +++ b/.github/workflows/pr-autofix-apply.yml @@ -336,7 +336,7 @@ jobs: # Push auth is provided inline at push time via the URL. - name: Checkout PR head if: steps.locate.outputs.found == 'true' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ steps.locate.outputs.head_repo }} ref: ${{ steps.locate.outputs.head_sha }} diff --git a/.github/workflows/pr-autofix-publish.yml b/.github/workflows/pr-autofix-publish.yml index 0fffad2ff..360b30986 100644 --- a/.github/workflows/pr-autofix-publish.yml +++ b/.github/workflows/pr-autofix-publish.yml @@ -131,7 +131,7 @@ jobs: # check-run SHA cannot be an unverified artifact field. # Mismatch => fail loud BEFORE any sticky/check-run side effect. - name: Checkout identity verifier - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false sparse-checkout: .github/scripts/verify-workflow-run-pr-identity.cjs diff --git a/.github/workflows/pr-autofix.yml b/.github/workflows/pr-autofix.yml index a74152049..2fec543aa 100644 --- a/.github/workflows/pr-autofix.yml +++ b/.github/workflows/pr-autofix.yml @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # PR head commit (not the synthetic merge ref) — we need the # exact tree the contributor pushed so suggestions line up. diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 57f50c4fd..4cf9b7c55 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -162,7 +162,7 @@ jobs: should_run: ${{ steps.decide.outputs.should_run }} head_sha: ${{ steps.decide.outputs.head_sha }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 fetch-tags: true @@ -332,7 +332,7 @@ jobs: # on the RC path. - name: Checkout (RC) if: needs.route.outputs.mode == 'rc' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 fetch-tags: true @@ -349,7 +349,7 @@ jobs: - name: Checkout (stable) if: needs.route.outputs.mode == 'stable' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # No `token:` — actions/checkout uses GITHUB_TOKEN by default. Stable # path performs no git pushes; the default scope is sufficient. with: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index d22add2ff..92e62f6ff 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -53,6 +53,6 @@ jobs: retention-days: 5 - name: Upload to Security tab - uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 + uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: sarif_file: results.sarif diff --git a/.github/workflows/skill-sync.yml b/.github/workflows/skill-sync.yml index d029a79a8..db2a8befc 100644 --- a/.github/workflows/skill-sync.yml +++ b/.github/workflows/skill-sync.yml @@ -47,7 +47,7 @@ jobs: timeout-minutes: 15 steps: # persist-credentials: false — runs a read-only test, never pushes. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 diff --git a/.github/workflows/tree-sitter-upgrade-readiness.yml b/.github/workflows/tree-sitter-upgrade-readiness.yml index 88ee0bd42..1a038d26e 100644 --- a/.github/workflows/tree-sitter-upgrade-readiness.yml +++ b/.github/workflows/tree-sitter-upgrade-readiness.yml @@ -52,7 +52,7 @@ jobs: report: ${{ steps.readiness.outputs.report }} exit_code: ${{ steps.readiness.outputs.exit_code }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/triage-sweep.yml b/.github/workflows/triage-sweep.yml index 4f39f94f2..b7c2e1d52 100644 --- a/.github/workflows/triage-sweep.yml +++ b/.github/workflows/triage-sweep.yml @@ -59,7 +59,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: sparse-checkout: .github/scripts/triage sparse-checkout-cone-mode: false diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index ce1a1ca96..f8c52b104 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -53,7 +53,7 @@ jobs: uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Build image (load locally for scan) - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . file: ${{ matrix.image.dockerfile }} @@ -76,7 +76,7 @@ jobs: exit-code: '0' - name: Upload to Security tab - uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 + uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: sarif_file: trivy-${{ matrix.image.name }}.sarif category: trivy-${{ matrix.image.name }} diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index c3aeb5fa8..03ce6b804 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -31,7 +31,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -53,7 +53,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -76,7 +76,7 @@ jobs: continue-on-error: true - name: Upload SARIF - uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 + uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: sarif_file: zizmor.sarif category: zizmor diff --git a/README.md b/README.md index bfcca5117..a761716e7 100644 --- a/README.md +++ b/README.md @@ -616,6 +616,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | | `GITNEXUS_EMBEDDING_RETRY_TIMEOUTS` | unset | When truthy (`1`/`true`/`yes`), per-attempt HTTP embedding timeouts (`TimeoutError` on fetch or body read) go through the bounded `GITNEXUS_EMBEDDING_MAX_ATTEMPTS` retry loop instead of failing the job. Any other value leaves it off, so cloud/default timeouts remain terminal. | Local accelerators that drop a device lock when the client disconnects and succeed on the next request (observed with FastFlowLM on Ryzen AI). | | `GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS` | `180000` (3 minutes) | Per-request IPC timeout for local embedding sidecar embed batches. On overrun the parent SIGKILLs the sidecar child and rejects the batch. Init still uses the HF download budget (`HF_DOWNLOAD_TIMEOUT_MS` × attempts), not this knob. | Large embed batches or slow local ONNX inference cause sidecar request timeouts during `analyze --embeddings`, `embeddings sync`, serve, or MCP. | +| `GITNEXUS_VECTOR_MAX_DISTANCE` | `0.6` for CLI/MCP `query`; `0.5` for standalone semantic search | Maximum cosine distance for vector hits, applied to both indexed search and exact-scan fallback. Hits must have `distance < cutoff`. Unset/blank uses the default; non-numeric, non-finite, zero, or negative values warn and use the default; finite values above `2` warn and clamp to `2`. | Paraphrased queries have weak semantic recall with your embedding model. See [Vector cutoff tuning](#vector-cutoff-tuning). | | `GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS` | unset | When truthy (`1`/`true`/`yes`), forces in-process cache-guard validation once a batch has ≥128 requests. In-process mode also auto-selects when `packageRoot`/`buildRoot` fail `W_OK` with `EACCES`/`EROFS`. Otherwise those large batches use a Node subprocess probe. Batches under 128 always stay in-process. | Trusted or read-only installs where two identity subprocess spawns per analyze dominate wall time; leave unset to keep the default isolation path on writable trees. | | `GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO` | on (unset) | Memoizes `resolveDefGraphId` per `nodeLookup` instance (WeakMap). Enabled by default. Set to `0`/`false`/`off`/`no` to disable and recompute on every call (debug / bisect memo bugs). | Suspecting stale graph-id resolution after a lookup rebuild, or comparing memo vs uncached cost on a large index. | | `GITNEXUS_AUTH_TOKEN` | unset | Bearer token required when `eval-server` binds beyond loopback. May also be read from `.env.local` or `.env`; shell values take precedence. | Exposing the evaluation HTTP tools to a container, VM, or LAN. | @@ -655,6 +656,22 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_PUBLIC_ORIGIN` | unset | The single browser origin `serve` is reached through, added to the CORS allowlist and to the write-route origin guard. A wildcard bind (`0.0.0.0`) has no host identity, so without this the server's own UI is refused. **Setting it currently refuses to start:** `serve` has no authentication, requests carrying no `Origin` header already reach `POST /api/analyze` and `DELETE /api/repo`, and this is the setting that would admit browser writes on top of that. Matching rules for when the gate lifts: the hostname must match exactly, and so must the scheme. A value with no scheme (`app.example.com`) means `https`, since a bare host comes from platform service discovery and those terminate TLS; spell out `http://app.example.com` for plain HTTP. An explicit port must match; with no port, any port on that hostname is accepted. Anything that is not one reachable host (a list, `*`, a bare port number, a `:0` port, a trailing dot) warns at startup and allows nothing. | `gitnexus serve` runs behind a reverse proxy or on a wildcard bind, and the UI's index/delete requests return `origin_not_allowed`. | | `GITNEXUS_TRUST_PROXY` | `loopback, linklocal, uniquelocal` | Express `trust proxy` value — which upstream hops may set `X-Forwarded-*`, and so what the per-IP rate limiter reads as the client IP. Set it to the exact number of proxies you control. Every hop past that is one more entry of the chain the caller gets to write. `false`/`no`/`off` (and a `0` hop count) trust no hop; a proxy list Express can compile (`loopback`, `10.0.0.0/8, 127.0.0.1`) names them instead. `true`/`yes`/`on` is **rejected**: it reads the client-controlled leftmost `X-Forwarded-For` entry, so a spoofed chain earns a fresh rate-limit key per request, and express-rate-limit rejects it too (`ERR_ERL_PERMISSIVE_TRUST_PROXY`). Counts above `16` are rejected as well, as a sanity ceiling rather than a safety boundary. Any invalid value warns and falls back to the default. Bind non-loopback with this unset and `serve` warns: a load balancer outside the private ranges is untrusted, so every request keys to the balancer and the per-IP limit becomes one shared limit. | `serve` sits behind a load balancer outside the private ranges (AWS ALB, Cloudflare, CGNAT), where every request otherwise collapses to the proxy hop and rate limiting goes global. | +### Vector cutoff tuning + +`GITNEXUS_VECTOR_MAX_DISTANCE` controls which vector candidates enter hybrid search. Cosine distance is lower for more similar vectors; the appropriate cutoff depends on the embedding model and repository. A higher cutoff can recover useful semantic matches, but can also admit less relevant hits and change the fused ranking. The default is `0.6` for CLI and MCP `query`, and `0.5` for the standalone semantic-search function. + +For an index that already has embeddings, compare the default with `0.8` and `1.0` on representative queries, including paraphrases, exact symbol names, and queries that should have no relevant result: + +```bash +GITNEXUS_VECTOR_MAX_DISTANCE=0.8 gitnexus query "how are expired sessions removed" --limit 10 +``` + +Check both whether the expected symbols appear and where they rank. In [#3457](https://github.com/abhigyanpatwari/GitNexus/issues/3457), the reporter's `voyage-code-4` sample found 20 of 32 paraphrased targets at `0.6`, 25 at `0.8`, and 28 at `1.0`. Those values are a tuning starting point for that model, not universal recommendations: one paraphrased target fell from first to twelfth at `1.0`. The sample covered eight repositories, recorded no raw distances, used agent-written queries after reading the code, and did not send Voyage's `input_type`. + +Set the variable in the process that runs the search. For MCP, add it to the server's launch environment and restart the server; for `gitnexus serve`, set it in that process's environment and restart it. Setting it only during `analyze`, or exporting it in another shell, does not configure an existing server. Changing the cutoff requires no reindex. + +The cutoff only filters available vector candidates. It cannot add missing embeddings or repair a mismatch between the indexing and query-time embedding configuration; see the [embeddings runbook](RUNBOOK.md#embeddings). A cutoff of `2` is very permissive, but still rejects distance exactly `2` and retains the search candidate limits. +
diff --git a/RUNBOOK.md b/RUNBOOK.md index 2f2c06f14..e2e1de94c 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -87,6 +87,14 @@ npx gitnexus analyze --force If it recurs, the cause is almost always environmental rather than a code defect: check free disk space on the volume holding `.gitnexus/`, make sure no second `analyze` is running against the same repo (both use `.gitnexus/csv` for staging), then run `npx gitnexus doctor`. The check compares in-memory relationship totals (including streamed rows) against what the DB hands back, and is deliberately skipped on incremental runs, where the two counts are not comparable. +**Weak semantic recall despite existing embeddings:** If paraphrased queries look like keyword-only search, the distance cutoff may be too strict for the embedding model. CLI and MCP `query` default to `0.6`. From the indexed repository, try a broader cutoff: + +```bash +GITNEXUS_VECTOR_MAX_DISTANCE=0.8 npx gitnexus query "how are expired sessions removed" --limit 10 +``` + +Compare target inclusion and ranking against the default; a broader cutoff also admits less relevant hits. Set the variable in the MCP/serve launch environment and restart that process when tuning a server. A cutoff change needs no reindex and setting it only during `analyze` does not persist it. If the index has no vectors, generate embeddings first; if the embedding model or dimensions differ between indexing and querying, align that configuration and regenerate vectors. See [Vector cutoff tuning](README.md#vector-cutoff-tuning) for validation rules and the limited `voyage-code-4` evidence from #3457. + **Large repos:** Analyze may skip or limit embedding work when node counts are very high; watch CLI output. --- diff --git a/docker-server.test.mjs b/docker-server.test.mjs index 6d2a9f6c2..309edf3b5 100644 --- a/docker-server.test.mjs +++ b/docker-server.test.mjs @@ -331,8 +331,8 @@ const respondOk = (_req, res) => { // // upstream request handler, replaceable mid-test via `ctx.handler`; // null points the proxy at a port nothing ever listens on -// listenAfterMs bind the upstream this late, so the first attempt(s) hit -// ECONNREFUSED (a single-instance restart window) +// listenAfterMs bind the upstream this late after the first refused attempt +// (a single-instance restart window) // schemeless drop http:// from GITNEXUS_UPSTREAM_URL, the way Render's // `fromService: { property: hostport }` yields it // env extra environment for docker-server.mjs @@ -366,18 +366,15 @@ async function withProxy( }) : null; - // A late (or never) bind needs its port reserved up front; otherwise let the - // OS assign one at listen time. - const upstreamPort = - server && listenAfterMs === 0 - ? await new Promise((r) => server.listen(0, '127.0.0.1', () => r(server.address().port))) - : await getFreePort(); - const bindTimer = - server && listenAfterMs > 0 - ? setTimeout(() => server.listen(upstreamPort, '127.0.0.1'), listenAfterMs) - : null; - + // Keep the upstream port bound until the proxy port is chosen. Releasing it + // sooner lets the OS assign both services the same port and proxy to itself. + const reservation = server ?? createServer(); + const upstreamPort = await new Promise((r) => + reservation.listen(0, '127.0.0.1', () => r(reservation.address().port)), + ); const port = await getFreePort(); + let bindTimer = null; + const target = `127.0.0.1:${upstreamPort}`; const proc = spawnServerWithEnv(dir, port, { GITNEXUS_UPSTREAM_URL: schemeless ? target : `http://${target}`, @@ -390,18 +387,25 @@ async function withProxy( ...env, }); proc.stderr.setEncoding('utf8'); - proc.stderr.on('data', (chunk) => { + const collectStderr = (chunk) => { ctx.stderr += chunk; - }); + // Process startup must not consume the restart window or skip the retry. + if (server && listenAfterMs > 0 && !bindTimer && ctx.stderr.includes('ECONNREFUSED; retry')) { + bindTimer = setTimeout(() => server.listen(upstreamPort, '127.0.0.1'), listenAfterMs); + } + }; + proc.stderr.on('data', collectStderr); try { await waitForServer(port); + if (!server || listenAfterMs > 0) await new Promise((r) => reservation.close(r)); await fn(port, ctx); } finally { + proc.stderr.off('data', collectStderr); if (bindTimer) clearTimeout(bindTimer); await killAndWait(proc); - if (server?.listening) { - server.closeAllConnections?.(); - await new Promise((r) => server.close(r)); + if (reservation.listening) { + reservation.closeAllConnections?.(); + await new Promise((r) => reservation.close(r)); } await rm(dir, { recursive: true, force: true }); } @@ -661,8 +665,8 @@ it('returns 502 when the upstream is unreachable', async () => { // -- Connection-retry across an upstream restart window --------------------- // -// `listenAfterMs: 400` binds the upstream late, so the first attempt hits -// ECONNREFUSED and must be retried — a single-instance restart. The default 3 +// `listenAfterMs: 400` binds the upstream 400ms after the first ECONNREFUSED, +// so the request must be retried — a single-instance restart. The default 3 // attempts (backoff 250ms, 500ms) span ~750ms, so a retry lands after the bind. it('retries a connection-refused POST and succeeds once the upstream is up', async () => { @@ -676,6 +680,7 @@ it('retries a connection-refused POST and succeeds once the upstream is up', asy assert.match(res.body, /"ok":true/); assert.equal(ctx.calls, 1, 'upstream must run the job exactly once (no double-execute)'); assert.equal(ctx.body, '{"repo":"x"}', 'buffered body replayed intact'); + assert.match(ctx.stderr, /ECONNREFUSED; retry/, 'the restart gap must exercise a retry'); }); }); diff --git a/eval/tests/test_workflow_bench.py b/eval/tests/test_workflow_bench.py index 51212ceda..4b197d9dc 100644 --- a/eval/tests/test_workflow_bench.py +++ b/eval/tests/test_workflow_bench.py @@ -185,7 +185,7 @@ def test_eval_ci_uses_locked_uv_and_blocking_native_containment_jobs(): step for step in containment["steps"] if str(step.get("uses", "")).startswith("actions/setup-node@") ) claude_lock = json.loads((repo_root / ".github" / "claude-canary-runtime" / "package-lock.json").read_text()) - setup_uv = "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990" + setup_uv = "astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9" assert workflow.count(setup_uv) >= 3 assert workflow.count("version: '0.11.23'") >= 3 assert workflow.count("uv run --locked --extra dev python -m pytest") >= 3 diff --git a/eval/uv.lock b/eval/uv.lock index ff06e6830..07e2f172f 100644 --- a/eval/uv.lock +++ b/eval/uv.lock @@ -1992,11 +1992,11 @@ wheels = [ [[package]] name = "oauthlib" -version = "3.3.1" +version = "4.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7a/d8/a1bcc8ba112a627f8ffbdc212a78ce18d3ac07e91a5ca65d27918eee25a1/oauthlib-4.0.0.tar.gz", hash = "sha256:efb274799819440f95b4ab3b818869f1ce9ae26c5beacba0201d1a1b76b54f86", size = 187232, upload-time = "2026-09-28T06:01:18.77Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, + { url = "https://files.pythonhosted.org/packages/d9/f4/78229a1066068ca14fc60fb26cf7381cabe4382261392b90e5f9552722d4/oauthlib-4.0.0-py3-none-any.whl", hash = "sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd", size = 159715, upload-time = "2026-09-28T06:01:17.008Z" }, ] [[package]] @@ -2522,11 +2522,11 @@ wheels = [ [[package]] name = "pyjwt" -version = "2.13.0" +version = "2.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +sdist = { url = "https://files.pythonhosted.org/packages/02/a5/5197bfd06417837ac079921c66fa6393f1dea3557272a263cebfef69e432/pyjwt-2.15.0.tar.gz", hash = "sha256:b11c5f9791d7bf51c2b39a81ed669f6b2dbbd669df2942f6c60167e9e3d1abe4", size = 120513, upload-time = "2026-09-23T16:56:00.689Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, + { url = "https://files.pythonhosted.org/packages/e8/55/40e45bf052ee8ee12a4dfd785519660f8effa7b065442b91646ec6828619/pyjwt-2.15.0-py3-none-any.whl", hash = "sha256:7a3742debf6b879e912dbb9819ceec1594be812452b78c5f2e2dfc56564954f8", size = 33680, upload-time = "2026-09-23T16:55:59.241Z" }, ] [package.optional-dependencies] @@ -3306,11 +3306,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index d904db1f6..4fc259a7b 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -9,16 +9,16 @@ "version": "0.0.0", "dependencies": { "@langchain/anthropic": "^1.5.8", - "@langchain/core": "^1.2.8", + "@langchain/core": "^1.2.13", "@langchain/google-genai": "^2.3.1", - "@langchain/langgraph": "^1.4.14", + "@langchain/langgraph": "^1.4.18", "@langchain/ollama": "^1.3.0", "@langchain/openai": "^1.5.13", "@sigma/edge-curve": "^3.1.0", "@tailwindcss/vite": "^4.3.3", "axios": "^1.20.0", "d3": "^7.9.0", - "dompurify": "^3.4.15", + "dompurify": "^3.4.16", "gitnexus-shared": "file:../gitnexus-shared", "graphology": "^0.26.0", "graphology-indices": "^0.17.0", @@ -26,9 +26,9 @@ "graphology-layout-forceatlas2": "^0.10.1", "graphology-layout-noverlap": "^0.4.2", "graphology-utils": "^2.3.0", - "i18next": "^26.3.6", + "i18next": "^26.4.2", "i18next-browser-languagedetector": "^8.2.1", - "langchain": "^1.5.11", + "langchain": "^1.5.14", "lru-cache": "^11.5.3", "lucide-react": "^1.46.0", "mermaid": "^11.17.2", @@ -53,7 +53,7 @@ "@testing-library/react": "^16.3.3", "@testing-library/user-event": "^14.6.7", "@types/dompurify": "^3.2.0", - "@types/node": "^26.5.1", + "@types/node": "^26.6.2", "@types/react": "^19.3.0", "@types/react-dom": "^19.3.0", "@types/react-syntax-highlighter": "^15.5.13", @@ -1083,9 +1083,9 @@ } }, "node_modules/@langchain/core": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.11.tgz", - "integrity": "sha512-8yuWLLloTSYA453akm2JSadOVa8kGDY8v+kTzQ6kTY6aIETTEIxgysjZWyKrWQLo3UazctsSoGJ8JrdCGFL4/w==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.13.tgz", + "integrity": "sha512-ADGTxZ84n3civruUX1LlTOdua/PDGoni2U6TmKTX7hvRU2Jlepu8vLJI4Wnwj45KUKhUCrW94Il12Q2bxE3e8A==", "license": "MIT", "dependencies": { "@cfworker/json-schema": "^4.0.2", @@ -1116,13 +1116,13 @@ } }, "node_modules/@langchain/langgraph": { - "version": "1.4.14", - "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.14.tgz", - "integrity": "sha512-uWAdRYTllfKCnTrlyovExPJCHJwcf3Wl2LzUlnaqsT7Rmoo3aCeYtq/7MV/Pw4q11motG8pR8bjr6T6V8Pe1gQ==", + "version": "1.4.18", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.18.tgz", + "integrity": "sha512-yrMMJ9hk2NVMD2xU2WoVrgFawAU6s/RzEl/dX9BhlyxZHazo1wHY35z4K2BIBzTUh4z3giCzrUoqRAqW2CWpWg==", "license": "MIT", "dependencies": { "@langchain/langgraph-checkpoint": "^1.1.5", - "@langchain/langgraph-sdk": "~1.10.2", + "@langchain/langgraph-sdk": "~1.12.0", "@langchain/protocol": "^0.0.19", "@standard-schema/spec": "1.1.0" }, @@ -1147,9 +1147,9 @@ } }, "node_modules/@langchain/langgraph-sdk": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.10.2.tgz", - "integrity": "sha512-86qsfdBZWu1ZgywLN8AThU/jXi9rjPDZPWcTJp4SA1A/L62ypTNoSXbvtiwZt1odokXccYTxK1XWS8tmVdvEmw==", + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.12.0.tgz", + "integrity": "sha512-F3AOZjKZRUGmE3FzY+RaVZI6WzXOkwnuF7jqgto6rDPSnO9OdVdYGvwGDi3jjRGf5xLoDtX2dpsR9z5KptU+5A==", "license": "MIT", "dependencies": { "@langchain/protocol": "^0.0.19", @@ -1194,9 +1194,9 @@ } }, "node_modules/@langchain/langgraph-sdk/node_modules/p-timeout": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", - "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.2.tgz", + "integrity": "sha512-prbX4Z3YszrFNgH+MW5Zoeq3baXrMtP/MQnFeET90UB/GtGcGDQ5Usg9OCy6ETjTTntOw1SL2z9fMPUppN3Guw==", "license": "MIT", "engines": { "node": ">=20" @@ -2064,9 +2064,9 @@ "license": "MIT" }, "node_modules/@ts-morph/common/node_modules/brace-expansion": { - "version": "1.1.18", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", - "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", "dev": true, "license": "MIT", "dependencies": { @@ -2438,9 +2438,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.5.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.1.tgz", - "integrity": "sha512-CzNm2FezW4VR/LjG6yUdiEgLE/rAQ9Slj5gCu/C2VrdcW7I0ahNZ8DRbHT7zOZ6r3ONgd/bsQIeSaoDGrd1C6g==", + "version": "26.6.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.2.tgz", + "integrity": "sha512-X1P21scMv4zGKLYqjdGjaKa7COa0RKVYYZZN/NfvLQ1JegxFhdhpZG/Lyn8AXx6CDUavKAd11v6BvfpkDByK8g==", "devOptional": true, "license": "MIT", "dependencies": { @@ -3410,9 +3410,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", "dev": true, "license": "MIT", "dependencies": { @@ -4291,9 +4291,9 @@ "peer": true }, "node_modules/dompurify": { - "version": "3.4.15", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.15.tgz", - "integrity": "sha512-EUBjM+B+lkDE41iE82DDSCfkoPGfXx8IxFxPMjNzm/Uk4xDet77rTN9wqlxlVg71kK7XGuUMv6wUxJUwwv+Xyw==", + "version": "3.4.16", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.16.tgz", + "integrity": "sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -5108,9 +5108,9 @@ } }, "node_modules/i18next": { - "version": "26.3.6", - "resolved": "https://registry.npmjs.org/i18next/-/i18next-26.3.6.tgz", - "integrity": "sha512-Bu5Z2nAXgfVyM8xvW3jk9EKRIuX37PudsrBViThNFx7CR7aaYTpP01cxNB/E4c4UUzTDiAZRstEhsRfPOL/8xA==", + "version": "26.4.2", + "resolved": "https://registry.npmjs.org/i18next/-/i18next-26.4.2.tgz", + "integrity": "sha512-RX+R0VLg13IbvRuJSxnqykUFS9vQZTl8wYpWPCIUDWVrSGjsQywB5Y+pjzrkboxGAuYfJZVH1InFTdgBdxq6ug==", "funding": [ { "type": "individual", @@ -5513,9 +5513,9 @@ "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" }, "node_modules/langchain": { - "version": "1.5.11", - "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.5.11.tgz", - "integrity": "sha512-6Sx9N5ylAJ11WrP1QnJLSIo75UABZbshzTJgG28H4mXuGcDk+w+7ZaNLkmGIh9sy/3PZcYS8UrI2rvH6A8NYIg==", + "version": "1.5.14", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.5.14.tgz", + "integrity": "sha512-/orHDk5xbNSIJc9UhwmxFYHXbr/3RaBeQ3zX0xovEWfPg7Lmj8EJCMOXIUTCe2UtsFgfbyWtlwh51NM562LnZQ==", "license": "MIT", "dependencies": { "@langchain/langgraph": "^1.4.13", @@ -5527,7 +5527,7 @@ "node": ">=20" }, "peerDependencies": { - "@langchain/core": "^1.2.10" + "@langchain/core": "^1.2.13" } }, "node_modules/langsmith": { diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index 787e24949..a45921117 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -19,16 +19,16 @@ }, "dependencies": { "@langchain/anthropic": "^1.5.8", - "@langchain/core": "^1.2.8", + "@langchain/core": "^1.2.13", "@langchain/google-genai": "^2.3.1", - "@langchain/langgraph": "^1.4.14", + "@langchain/langgraph": "^1.4.18", "@langchain/ollama": "^1.3.0", "@langchain/openai": "^1.5.13", "@sigma/edge-curve": "^3.1.0", "@tailwindcss/vite": "^4.3.3", "axios": "^1.20.0", "d3": "^7.9.0", - "dompurify": "^3.4.15", + "dompurify": "^3.4.16", "gitnexus-shared": "file:../gitnexus-shared", "graphology": "^0.26.0", "graphology-indices": "^0.17.0", @@ -36,9 +36,9 @@ "graphology-layout-forceatlas2": "^0.10.1", "graphology-layout-noverlap": "^0.4.2", "graphology-utils": "^2.3.0", - "i18next": "^26.3.6", + "i18next": "^26.4.2", "i18next-browser-languagedetector": "^8.2.1", - "langchain": "^1.5.11", + "langchain": "^1.5.14", "lru-cache": "^11.5.3", "lucide-react": "^1.46.0", "mermaid": "^11.17.2", @@ -63,7 +63,7 @@ "@testing-library/react": "^16.3.3", "@testing-library/user-event": "^14.6.7", "@types/dompurify": "^3.2.0", - "@types/node": "^26.5.1", + "@types/node": "^26.6.2", "@types/react": "^19.3.0", "@types/react-dom": "^19.3.0", "@types/react-syntax-highlighter": "^15.5.13", diff --git a/gitnexus/bench/emit-persistence/baselines-streaming.json b/gitnexus/bench/emit-persistence/baselines-streaming.json index 0fabf5a02..837d78d1f 100644 --- a/gitnexus/bench/emit-persistence/baselines-streaming.json +++ b/gitnexus/bench/emit-persistence/baselines-streaming.json @@ -1,5 +1,6 @@ { - "fingerprint": "dadb6eb19a751c3d315b7c53effc6605f89e0a766b01cd5a2ae611b3720d0d55", + "fingerprint": "9408aad13feb6a65c6ce031a3264b72d1afdb1fdd1e984666fe30c3ecc24d66f", "_rebaselined_3161_static_gated_column": "Same change as baselines.json's `_rebaselined_3161_static_gated_column`: every relationship row now ends in a `staticGated` cell (`0` here, no PDG edge is a gated call), so each of the 36,000 `rel_BasicBlock_BasicBlock.csv` data rows grew by 2 bytes (`...,\"T\",0` -> `...,\"T\",0,0`) while the 7,200 BasicBlock rows are unchanged. Evidence is the inverse operation: stripping the trailing `,0` from the rel rows and re-hashing the sorted bb + rel rows gives EXACTLY the prior baseline 381de8dede253140953775c290bf9a25f82ebf3cc8ecb5250ae06a63f648b089. byte_identical_nodes / byte_identical_edges stayed true and resident_basic_blocks stayed 0 while the fingerprint gate was red, so the streamed sink still matches the whole-graph emit byte for byte and the RSS bound holds. Prior 381de8dede253140953775c290bf9a25f82ebf3cc8ecb5250ae06a63f648b089 -> dadb6eb19a751c3d315b7c53effc6605f89e0a766b01cd5a2ae611b3720d0d55.", - "_note": "Byte-identity + bounded-retention gate for streaming/chunked PDG emit (#2202). fingerprint = sha256 of the sorted, header-stripped BasicBlock + PDG-edge data rows of the canonical synthetic set. --check also asserts the streamed PdgEmitSink output is byte-identical to the whole-graph streamAllCSVsToDisk emit (byte_identical_nodes/edges) and that the in-memory graph retains 0 BasicBlocks (resident_basic_blocks === 0, the O(chunk) RSS bound). Regenerate via `node --import tsx bench/emit-persistence/measure-streaming.mjs`." + "_note": "Byte-identity + bounded-retention gate for streaming/chunked PDG emit (#2202). fingerprint = sha256 of the sorted, header-stripped BasicBlock + PDG-edge data rows of the canonical synthetic set. --check also asserts the streamed PdgEmitSink output is byte-identical to the whole-graph streamAllCSVsToDisk emit (byte_identical_nodes/edges) and that the in-memory graph retains 0 BasicBlocks (resident_basic_blocks === 0, the O(chunk) RSS bound). Regenerate via `node --import tsx bench/emit-persistence/measure-streaming.mjs`.", + "_rebaselined_3442_ladybug_null_cells": "Same NULL serialization change as baselines.json (#3442). Compared the whole-graph output with the parent writer: file set, row order and all other bytes match. The fingerprinted BasicBlock and PDG-edge files contain exactly 38,400 quoted empty cells changed to bare cells (14,400 BasicBlock cells and 24,000 edge cells); restoring their quotes reproduces the exact prior fingerprint dadb6eb19a751c3d315b7c53effc6605f89e0a766b01cd5a2ae611b3720d0d55. The new fingerprint is 9408aad13feb6a65c6ce031a3264b72d1afdb1fdd1e984666fe30c3ecc24d66f. Both streamed/whole byte-identity flags remain true and resident_basic_blocks remains 0." } diff --git a/gitnexus/bench/emit-persistence/baselines.json b/gitnexus/bench/emit-persistence/baselines.json index 9c04fb248..374ae2067 100644 --- a/gitnexus/bench/emit-persistence/baselines.json +++ b/gitnexus/bench/emit-persistence/baselines.json @@ -1,5 +1,5 @@ { - "fingerprint": "d3c3a53ead47663816344369dc0e501f31d60c462039de190ab6f5361037bd79", + "fingerprint": "2f5c46506cb8dfb79a65a276646ace766cc7645fabab8b383c89f5f0270cc30b", "_rebaselined_objective_c_node_tables": "Objective-C support adds Protocol and Category node tables to csv-generator.ts's MULTI_LANG_TYPES, so this deterministic 2,400-entity synthetic emit now creates 38 CSV files rather than 36. The two additions, category.csv and protocol.csv, are both expected 55-byte header-only files because the synthetic graph contains no Objective-C nodes. Re-emitting the graph and recomputing the fingerprint after excluding exactly those two files yields the exact prior baseline 011485e5180c005be9dfec7ac8dc4e3bb0fcfc4a680a16dcae2cf2d8af5ef7e2, proving all 36 pre-existing files retain their bytes, routing, and within-file order. Prior 011485e5180c005be9dfec7ac8dc4e3bb0fcfc4a680a16dcae2cf2d8af5ef7e2 -> d3c3a53ead47663816344369dc0e501f31d60c462039de190ab6f5361037bd79. The timing guard remains within budget: local scaling_ratio 0.965 against 1.8 and elapsed_ms_large 69.24ms against 1000ms.", "_rebaselined_3161_static_gated_column": "Every relationship row gained a trailing `staticGated` BOOLEAN column (RELATION_SCHEMA in src/core/lbug/schema.ts, REL_CSV_HEADER + buildRelRow in csv-generator.ts, the fallback CREATE in lbug-adapter.ts), written as `0` for every edge that does not carry the flag and `1` for a Zig call site inside a comptime-false branch (PR #3161). Unlike the earlier header-only rebaselines this one touches ROWS, so the evidence is the inverse operation rather than a file-set diff: re-emitting this bench's 2,400-entity graph on the branch produces the same 36 CSV files; exactly 3 of them differ from a copy with the new column stripped (`rel_File_Class.csv` 156228 -> 151416 bytes, `rel_File_Function.csv` 334008 -> 324396, `rel_Function_Function.csv` 185028 -> 180216: one header field plus `,0` per row, 4,800 / 9,600 / 4,800 rows, 2 bytes each), the other 33 files are byte-identical, and the per-file fingerprint over the stripped copies is EXACTLY the prior baseline 72096279092d4f118de7e179333705c19c9aff2664f77d71a7da48cd9f73fb5a. So no row moved between pair files and no row reordered; the only bytes that changed are the appended column. Prior 72096279092d4f118de7e179333705c19c9aff2664f77d71a7da48cd9f73fb5a -> 011485e5180c005be9dfec7ac8dc4e3bb0fcfc4a680a16dcae2cf2d8af5ef7e2. Timing gates passed while the guard was red: scaling_ratio 0.82 against the 1.8 budget, elapsed_ms_large 185.54ms against the 1000ms backstop, so no throughput claim is being rebaselined away.", "scaling_budget": 1.8, @@ -9,5 +9,6 @@ "_rebaselined_2856_property_is_detail": "Third and last of the bench guards this branch left red. The Property node table gained an `isDetail` BOOLEAN column (see PROPERTY_SCHEMA in src/core/lbug/schema.ts), so `streamAllCSVsToDisk` writes one more header field and one more cell per Property row \u2014 csv-generator.ts `propertyHeader` and the `node.label === 'Property'` tail. Verified to be header-only drift rather than a change in what is emitted: dumping every CSV this bench produces on `origin/main` and on this branch and diffing per-file (filename, byte length, sha256) shows the file SET is identical at 35 CSVs on both sides, 34 of the 35 are byte-identical, and the sole difference is `property.csv` growing 68 -> 77 bytes, `id,name,filePath,startLine,endLine,content,description,declaredType` -> `...,declaredType,isDetail`. The synthetic graph has no Property nodes, so no ROW moved at all. That is the check that matters here: a row routed to the wrong pair file, or a within-file reordering, is what this fingerprint exists to catch, and neither happened. Prior 69e9182ae205183ade24c3d8ad5d7292aea677144b1cbe443dd631bc25b0cafe -> 4ee15e742a9839671a900df4f57c1c91196c64256c8cab2ac445bec605a092d5. Both timing gates passed unchanged while this was red (scaling_ratio 0.783 vs budget 1.8, elapsed_ms_large 229ms vs the 1000ms backstop), so no throughput claim is being rebaselined away.", "_rebaselined_3040_convex_endpoint_factory": "Const and Function gained a trailing convexEndpointFactory column. A deterministic 2,400-entity emit produced the same 35 CSV files and fingerprint c4d799c5336d616955b3530ba051b7dca300d1a0e412a66741cf2f27e04c533e. Removing the new Const and Function header fields plus the new trailing empty Function cell from each of 4,800 Function rows restored the exact prior fingerprint 4ee15e742a9839671a900df4f57c1c91196c64256c8cab2ac445bec605a092d5. No file or row moved or reordered. The measured scaling ratio remained 0.826 against the 1.8 budget and elapsed_ms_large was 307.75ms against the 1000ms backstop.", "_rebaselined_3107_route_runtime_evidence": "Route gained trailing runtimeConfirmed BOOLEAN, runtimeSource STRING, and runtimeStatus STRING columns in its schema, CSV header/rows, COPY statement, and graph API projection. The deterministic emit still produces the same 35 CSV files; the synthetic benchmark graph has no Route rows, so the only byte drift is the Route CSV header and no row moved or reordered. Prior c4d799c5336d616955b3530ba051b7dca300d1a0e412a66741cf2f27e04c533e -> 7b2ec01a110dcbc66868fba2c97714aaece8c3864c2eba00df68b5c027f034d2. While the guard was red, scaling_ratio was 1.044 against the 1.8 budget and elapsed_ms_large was 121.08ms against the 1000ms backstop.", - "_note": "fingerprint = sha256 over per-file digests (filename + sha256(file bytes)), entry list sorted \u2014 binds each emitted line to its file so a row routed to the WRONG pair file changes the hash, AND catches within-file row reordering (file bytes hashed as-written). Byte-identity gate for #2203 U2/U3. NOTE: a future change that legitimately reorders emit (without changing the node/edge SET) will trip --check; regenerate then, and record WHY in a `_rebaselined_` key alongside \u2014 bench/scope-capture/baselines.json sets that convention and it is what makes a regenerated hash reviewable. scaling_budget bounds (t_large/t_small)/(LARGE/SMALL): observed ~0.95-1.05 (linear); 1.8 tolerates disk-I/O timing noise on CI while still catching an O(n^2) re-regression (~4x). max_ms_large=1000ms is a coarse absolute backstop (observed ~200ms) that catches a gross uniform slowdown the ratio gate misses; generous so CI host noise won't flake it. Regenerate via `node --import tsx bench/emit-persistence/measure.mjs`." + "_note": "fingerprint = sha256 over per-file digests (filename + sha256(file bytes)), entry list sorted \u2014 binds each emitted line to its file so a row routed to the WRONG pair file changes the hash, AND catches within-file row reordering (file bytes hashed as-written). Byte-identity gate for #2203 U2/U3. NOTE: a future change that legitimately reorders emit (without changing the node/edge SET) will trip --check; regenerate then, and record WHY in a `_rebaselined_` key alongside \u2014 bench/scope-capture/baselines.json sets that convention and it is what makes a regenerated hash reviewable. scaling_budget bounds (t_large/t_small)/(LARGE/SMALL): observed ~0.95-1.05 (linear); 1.8 tolerates disk-I/O timing noise on CI while still catching an O(n^2) re-regression (~4x). max_ms_large=1000ms is a coarse absolute backstop (observed ~200ms) that catches a gross uniform slowdown the ratio gate misses; generous so CI host noise won't flake it. Regenerate via `node --import tsx bench/emit-persistence/measure.mjs`.", + "_rebaselined_3442_ladybug_null_cells": "LadybugDB 0.21.1 preserves quoted empty strings, so escapeCSVField now emits bare empty cells to retain the SQL NULL meaning of the 0.18.3 writer (#3442). Compared the canonical emit with the parent writer: all 38 file names and every row order match; exactly 31,200 quoted empty cells became bare cells in six files, with no other byte changes. Restoring quotes around those cells reproduces the exact prior fingerprint d3c3a53ead47663816344369dc0e501f31d60c462039de190ab6f5361037bd79. The new fingerprint is 2f5c46506cb8dfb79a65a276646ace766cc7645fabab8b383c89f5f0270cc30b. Timing budgets remain unchanged." } diff --git a/gitnexus/bench/incremental-write-integrity/README.md b/gitnexus/bench/incremental-write-integrity/README.md new file mode 100644 index 000000000..b5b14d881 --- /dev/null +++ b/gitnexus/bench/incremental-write-integrity/README.md @@ -0,0 +1,83 @@ +# Incremental node identity reconciliation + +From `gitnexus/`: + +```sh +node --import tsx bench/incremental-write-integrity/measure.cjs --check +node bench/incremental-write-integrity/reproduce.cjs +``` + +The first command measures the production reconciliation, including every +single-label scan and comparison of ID, name, path and source range. Construction, +CSV loading, checkpoints and correctness controls are outside the timed region. +Two warm-ups precede seven measured samples. Exact counts and SHA-256 fixture +fingerprints reject empty or incomplete work. `--check` gates normalized scaling +against the committed budget, rather than a machine-specific time limit. + +On Node v24.19.0, Linux x64, Xeon Platinum 8370C, the initial measurement was +583 ms for 16,384 nodes and 2,810 ms for 65,536 nodes per reconciliation. The +normalized scaling was 1.206 (budget 2). Analyze performs two reconciliations: +after graph COPY/checkpoint and after FTS/embedding work plus a final checkpoint, +before registration, freshness metadata or staging publication. + +The verifier covers retained nodes as well as the write set. A path/range filter +would let a corrupted field hide its own row; a count would miss swapped fields +or blank IDs. Folder lifecycle, preserved Community/Process layers and streamed +BasicBlock rows require other oracles and are excluded. Relationships and source +content are outside this identity check. + +## Native-only reduction + +`reproduce.cjs` uses only `@ladybugdb/core`, its native schema/query/COPY API, and +synthetic ASCII CSV files. There is no parser, parse cache, graph adapter, FTS, +relationship table, periodic checkpoint driver or concurrent writer. Compression +is disabled and COPY is serial, matching the production settings. + +It loads 8,192 Function rows, checkpoints, deletes 64 rows belonging to the first +and last owners, then copies back those same 64 tuples. An independent tuple-set +oracle validates the complete single-label scan before and after each operation. +It also compares affected scan rows with primary-key lookups after reopening. +Use `--keep` to retain the synthetic database and CSV files; the JSON output gives +their directory. `--require-corruption` is a diagnostic assertion, deliberately +not a CI requirement: it should fail when the native defect is fixed. + +With the original `@ladybugdb/core` 0.18.3, this reduction produced: + +| Phase | Rows | Incorrect tuples | +| ------------------------- | ----: | ---------------: | +| Initial COPY + checkpoint | 8,192 | 0 | +| DELETE | 8,128 | 0 | +| Incremental COPY | 8,192 | 3,936 | +| Explicit checkpoint | 8,192 | 3,936 | +| Read-only reopen | 8,192 | 3,936 | + +With `@ladybugdb/core` 0.21.1, the same reduction returns zero incorrect +tuples in every phase. The production reconciliation and negative controls +remain required; this result covers the reduced fixture, not a replay of the +original incident. + +For example, the scan returned an empty `id` at native offset 64, while a +primary-key lookup at that same offset returned +`Function:src/owner2.ts:fn64` with the correct name/path/range. Smaller 1,024, +2,048 and 4,096-row fixtures remained healthy in the same three-cycle adapter +probe. The affected tuple count varies with fixture size and scan shape. + +This isolates a native scan/lookup discrepancy introduced by incremental COPY +into a previously populated table. It survives checkpoint and reopen, but does +not establish physical byte loss or the precise C++ defect. The reported Unicode +change and FTS build are unnecessary for this reduction; this does not prove +that the original incident has exactly the same native cause. + +The measurement harness runs selective native writes too. An independent oracle +requires the production verifier to reject any inconsistent scan, both before +and after checkpoint/reopen; healthy scans must certify their complete identity +set. It does not quietly treat the native discrepancy as a successful graph. + +Missing-row and swapped-range negative controls separately verify that the +check fails closed. Publication tests inject damage after COPY and after FTS, +assert unchanged registry/freshness, preserve the live staged baseline, and +exercise automatic dirty-index recovery followed by a no-op run. +An in-place verification failure keeps a non-FTS dirty phase: an FTS-only +repair cannot clear the marker and recertify the inconsistent graph. +The same protection applies when FTS returns but analyzer finalization fails +before the final identity scan: recovery still rebuilds the graph. diff --git a/gitnexus/bench/incremental-write-integrity/baseline.json b/gitnexus/bench/incremental-write-integrity/baseline.json new file mode 100644 index 000000000..988196191 --- /dev/null +++ b/gitnexus/bench/incremental-write-integrity/baseline.json @@ -0,0 +1,14 @@ +{ + "version": 1, + "linear_scaling_budget": 2, + "sizes": [ + { + "nodes": 16384, + "fingerprint": "17aa2ded124af9cbd15a9fe13013410e92a478f88e63a2ad9859a05aacef8efe" + }, + { + "nodes": 65536, + "fingerprint": "a6ce82aa1d07804d16b916892dfb6ca0e6f5ce5ed606f47cb1321eb124c6ac44" + } + ] +} diff --git a/gitnexus/bench/incremental-write-integrity/measure.cjs b/gitnexus/bench/incremental-write-integrity/measure.cjs new file mode 100644 index 000000000..8db1d2d5b --- /dev/null +++ b/gitnexus/bench/incremental-write-integrity/measure.cjs @@ -0,0 +1,194 @@ +#!/usr/bin/env node +/** + * Native identity reconciliation overhead and scaling. + * node --import tsx bench/incremental-write-integrity/measure.cjs [--check] + * + * COPY, graph construction, fingerprints and correctness controls are untimed. + * Each timed sample includes all single-label scans and exact tuple checks. + * Counts/fingerprints reject empty output; timing gates use normalized scaling. + */ +const assert = require('node:assert/strict'); +const { mkdtemp, rm, readFile } = require('node:fs/promises'); +const { tmpdir, cpus } = require('node:os'); +const { join, resolve } = require('node:path'); +const { pathToFileURL } = require('node:url'); +const { createHash } = require('node:crypto'); +const ROOT = resolve(__dirname, '../..'); +const source = (file) => import(pathToFileURL(join(ROOT, file)).href); + +(async () => { + const { createKnowledgeGraph } = await source('src/core/graph/graph.ts'); + const { reconcileGraphNodeIdentities } = await source( + 'src/core/incremental/write-reconciliation.ts', + ); + const adapter = await source('src/core/lbug/lbug-adapter.ts'); + const baseline = JSON.parse(await readFile(join(__dirname, 'baseline.json'), 'utf8')); + assert.equal(baseline.version, 1); + assert.ok(Number.isFinite(baseline.linear_scaling_budget) && baseline.linear_scaling_budget > 0); + const results = []; + for (const size of baseline.sizes) { + assert.ok(Number.isSafeInteger(size.nodes) && size.nodes > 0); + assert.match(size.fingerprint, /^[a-f0-9]{64}$/); + const graph = createKnowledgeGraph(); + const hash = createHash('sha256'); + for (let i = 0; i < size.nodes; i++) { + const id = `Function:src/owner${Math.floor(i / 32)}.ts:fn${i}`; + const properties = { + name: `fn${i}`, + filePath: `src/owner${Math.floor(i / 32)}.ts`, + startLine: (i % 32) * 4, + endLine: (i % 32) * 4 + 2, + }; + graph.addNode({ id, label: 'Function', properties }); + hash.update( + JSON.stringify([ + id, + properties.name, + properties.filePath, + properties.startLine, + properties.endLine, + ]) + '\n', + ); + } + assert.equal(hash.digest('hex'), size.fingerprint); + const directory = await mkdtemp(join(tmpdir(), 'gnx-reconciliation-bench-')); + try { + await adapter.initLbug(join(directory, 'lbug'), { skipFts: true }); + await adapter.loadGraphToLbug( + graph, + directory, + directory, + undefined, + undefined, + undefined, + 'none', + ); + await adapter.tryFlushWAL(); + const samples = []; + let calls = 0; + const query = async (sql) => { + calls++; + return adapter.executeQuery(sql); + }; + for (let run = 0; run < 9; run++) { + calls = 0; + const start = performance.now(); + const receipt = await reconcileGraphNodeIdentities(graph, query, 'benchmark'); + const elapsed = performance.now() - start; + assert.equal(receipt.nodes, size.nodes); + assert.equal(calls, receipt.tables); + if (run >= 2) samples.push(elapsed); + } + const first = graph.iterNodes().next().value; + const alteredQuery = async (sql) => { + const rows = await adapter.executeQuery(sql); + if (sql.includes('(n:`Function`)')) rows.find((r) => r.id === first.id).startLine++; + return rows; + }; + await assert.rejects( + reconcileGraphNodeIdentities(graph, alteredQuery, 'negative-control'), + /startLine/, + ); + await assert.rejects( + reconcileGraphNodeIdentities(graph, async () => [], 'negative-control'), + /missing ID/, + ); + + // Native 0.18.3 can corrupt scan projections after this real write, even + // without FTS. Use an independent tuple-set oracle to require rejection + // whenever the scan is wrong; a future native fix can pass normally. + const { extractChangedSubgraph } = await source('src/core/incremental/subgraph-extract.ts'); + const files = new Set(['src/owner0.ts', `src/owner${Math.floor((size.nodes - 1) / 32)}.ts`]); + const wanted = new Set( + [...graph.iterNodes()].map((node) => + JSON.stringify([ + node.id, + node.properties.name, + node.properties.filePath, + node.properties.startLine, + node.properties.endLine, + ]), + ), + ); + const nativePhases = []; + const audit = async (phase) => { + const rows = await adapter.executeQuery( + 'MATCH (n:Function) RETURN n.id AS id, n.name AS name, ' + + 'n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine', + ); + const actual = new Set( + rows.map((r) => JSON.stringify([r.id, r.name, r.filePath, r.startLine, r.endLine])), + ); + const mismatched = [...wanted].filter((tuple) => !actual.has(tuple)).length; + const rejected = + rows.length !== size.nodes || mismatched > 0 || actual.size !== wanted.size; + if (rejected) { + await assert.rejects( + reconcileGraphNodeIdentities(graph, adapter.executeQuery, phase), + /Graph identity reconciliation/, + ); + } else { + assert.equal( + (await reconcileGraphNodeIdentities(graph, adapter.executeQuery, phase)).nodes, + size.nodes, + ); + } + nativePhases.push({ + phase, + rows: rows.length, + missing_tuples: mismatched, + verdict: rejected ? 'rejected' : 'certified', + }); + }; + await adapter.deleteNodesForFiles([...files]); + await adapter.loadGraphToLbug( + extractChangedSubgraph(graph, files), + directory, + directory, + undefined, + undefined, + undefined, + 'none', + ); + await audit('post-COPY'); + await adapter.tryFlushWAL(); + await audit('post-checkpoint'); + await adapter.closeLbug(); + await adapter.initLbug(join(directory, 'lbug'), { readOnly: true, skipFts: true }); + await audit('reopened'); + samples.sort((a, b) => a - b); + results.push({ + nodes: size.nodes, + queries: calls, + min_ms: +samples[0].toFixed(3), + median_ms: +samples[Math.floor(samples.length / 2)].toFixed(3), + fingerprint: size.fingerprint, + native_phases: nativePhases, + }); + } finally { + await adapter.closeLbug(); + await rm(directory, { recursive: true, force: true }); + } + } + const scaling = + results[1].median_ms / results[0].median_ms / (results[1].nodes / results[0].nodes); + const report = { + node: process.version, + platform: process.platform, + cpu: cpus()[0].model, + results, + normalized_scaling: +scaling.toFixed(3), + negative_controls: 'missing/swapped fields rejected', + native_sequences: + 'selective delete/COPY, checkpoint and read-only reopen checked against independent tuple oracle', + }; + process.stdout.write(JSON.stringify(report, null, 2) + '\n'); + if (process.argv.includes('--check')) + assert.ok( + scaling <= baseline.linear_scaling_budget, + `normalized scaling ${scaling} exceeds ${baseline.linear_scaling_budget}`, + ); +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/gitnexus/bench/incremental-write-integrity/reproduce.cjs b/gitnexus/bench/incremental-write-integrity/reproduce.cjs new file mode 100644 index 000000000..a270d3331 --- /dev/null +++ b/gitnexus/bench/incremental-write-integrity/reproduce.cjs @@ -0,0 +1,187 @@ +#!/usr/bin/env node +/** Native-only selective COPY scan discrepancy; no parser, graph adapter or FTS. */ +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const lbug = require('@ladybugdb/core'); + +(async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ladybug-copy-identity-')); + let database; + let connection; + const close = async () => { + // Retire this session before closing so failures or reopening cannot close it twice. + const activeConnection = connection; + const activeDatabase = database; + connection = undefined; + database = undefined; + const errors = []; + try { + await activeConnection?.close(); + } catch (error) { + errors.push(error); + } + try { + await activeDatabase?.close(); + } catch (error) { + errors.push(error); + } + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) throw new AggregateError(errors, 'Native resource cleanup failed'); + }; + const errors = []; + try { + const dbPath = path.join(directory, 'lbug'); + const count = 8192; + const tuple = (i) => [ + `Function:src/owner${Math.floor(i / 32)}.ts:fn${i}`, + `fn${i}`, + `src/owner${Math.floor(i / 32)}.ts`, + (i % 32) * 4, + (i % 32) * 4 + 2, + ]; + const indices = Array.from({ length: count }, (_, i) => i); + const expected = new Set(indices.map((i) => JSON.stringify(tuple(i)))); + const csv = (rows) => + 'id,name,filePath,startLine,endLine\n' + + rows + .map((i) => + tuple(i) + .map((value) => JSON.stringify(value)) + .join(','), + ) + .join('\n') + + '\n'; + await fs.writeFile(path.join(directory, 'full.csv'), csv(indices)); + await fs.writeFile( + path.join(directory, 'delta.csv'), + csv(indices.filter((i) => i < 32 || i >= count - 32)), + ); + + // Match GitNexus's uncompressed native constructor and serial COPY options. + const open = (readOnly) => + new lbug.Database( + dbPath, + 256 * 1024 * 1024, + false, + readOnly, + 4 * 1024 ** 3, + true, + 64 * 1024 * 1024, + true, + true, + ); + database = open(false); + connection = new lbug.Connection(database); + const query = async (cypher, params) => { + const result = params + ? await connection.execute(await connection.prepare(cypher), params) + : await connection.query(cypher); + try { + return await result.getAll(); + } finally { + await result.close(); + } + }; + const copy = (file) => + query( + `COPY Function FROM "${path.join(directory, file).replace(/\\/g, '/')}" ` + + `(HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`, + ); + const phases = []; + const scan = async (phase) => { + const rows = await query( + 'MATCH (n:Function) RETURN id(n) AS internalID, n.id AS id, n.name AS name, ' + + 'n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine', + ); + const key = (r) => JSON.stringify([r.id, r.name, r.filePath, r.startLine, r.endLine]); + const bad = rows.filter((r) => !expected.has(key(r))); + const actual = new Set(rows.map(key)); + const missing = [...expected].filter((value) => !actual.has(value)).length; + phases.push({ + phase, + rows: rows.length, + wrong_tuples: bad.length, + missing_tuples: missing, + examples: bad.slice(0, 2), + }); + return { rows, bad, missing }; + }; + await query( + 'CREATE NODE TABLE Function(id STRING, name STRING, filePath STRING, ' + + 'startLine INT64, endLine INT64, PRIMARY KEY(id))', + ); + await copy('full.csv'); + await query('CHECKPOINT'); + const before = await scan('baseline'); + assert.equal(before.rows.length, count); + assert.equal(before.missing, 0); + const baselineIdsByOffset = new Map(before.rows.map((r) => [r.internalID.offset, r.id])); + await query( + "MATCH (n:Function) WHERE n.filePath IN ['src/owner0.ts', 'src/owner255.ts'] DETACH DELETE n", + ); + const deleted = await scan('after-delete'); + assert.equal(deleted.rows.length, count - 64); + assert.equal(deleted.bad.length, 0); + await copy('delta.csv'); + const copied = await scan('after-copy'); + await query('CHECKPOINT'); + await scan('after-checkpoint'); + await close(); + database = open(true); + connection = new lbug.Connection(database); + const reopened = await scan('reopened'); + const pointLookups = []; + for (const row of reopened.bad.slice(0, 2)) { + const id = baselineIdsByOffset.get(row.internalID.offset); + if (!id) continue; + pointLookups.push({ + scan: row, + lookup: await query( + 'MATCH (n:Function {id: $id}) RETURN n.id AS id, n.name AS name, n.filePath AS filePath, ' + + 'n.startLine AS startLine, n.endLine AS endLine', + { id }, + ), + }); + } + process.stdout.write( + JSON.stringify( + { + native_version: lbug.VERSION, + node: process.version, + phases, + point_lookups: pointLookups, + ...(process.argv.includes('--keep') ? { directory } : {}), + }, + null, + 2, + ) + '\n', + ); + if (process.argv.includes('--require-corruption')) { + assert.ok( + copied.bad.length > 0 || copied.missing > 0 || copied.rows.length !== count, + 'Native failure did not reproduce; this is a diagnostic, not a passing CI invariant', + ); + } + } catch (error) { + errors.push(error); + } finally { + try { + await close(); + } catch (error) { + errors.push(error); + } + try { + if (!process.argv.includes('--keep')) + await fs.rm(directory, { recursive: true, force: true }); + } catch (error) { + errors.push(error); + } + } + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) throw new AggregateError(errors, 'Native benchmark failed'); +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index 4845f23fb..60ffca32d 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -119,7 +119,8 @@ "_rebaselined_3354_callable_alternatives": "#3354 callable alternatives: a callable chosen by a value-selecting source now flows every branch it can yield (`a ?? b`, `a || b`, `a or b`, `c ? a : b`, statement `if`, elvis), and an operator branch (`x == f || g`) stays opaque instead of seeding a qualified name. Verified by running the BASE (merge-base 233ca2849) and HEAD emitters over the SAME HEAD fixture corpus: every added or removed match is an `@callable-flow.*` match on one of those sources, and the pre-existing corpus is byte-identical (all other languages: zero delta). The rest of the drift is corpus growth from this PR's regression fixture, which this bench globs. Corpus growth: ruby-callable-alternatives/app.rb (+1 file, +58 groups). Emitter delta: +5 / -0: seeds for single-statement `if` / `elsif` branches (run_then, run_else, run_sweep, run_a, run_b); the multi-statement branch contributes nothing. capture_groups_fp 1358 -> 1416, fixture_count 91 -> 92; synthetic counts unchanged; scaling 1.04 < 1.5. Prior 1c8c9c4b54036fa24c2a81e39ea530e938645c856d369075e5f437da78218c57 -> 45e65d9fa8a9e5e905ddb596b179b77c86ed13ab5139c6b17ccaaf7315dfe46a." }, "swift": { - "fingerprint": "d56406c2645637042899cfcc8dc73f603d77caef0a9a2bac179ffbec848258a3", + "fingerprint": "2507ac75ba6fb47c272a5caa97b5e8b9f6ebc5098c15295d05c7bff3cabc321a", + "_rebaselined_3425_injected_closure": "Swift capture fixture growth: 83 to 85 fixtures, 1436 to 1553 capture groups; scaling budget unchanged.", "_rebaselined_3355_xcode_and_import_fixtures": "#3355 follow-up: new swift-xcode-targets fixture (four sources) and two sources added to swift-nested-packages (a Docs/Net folder decoy and an `import Net` caller). The capture query is unchanged; this is fixture-corpus growth only. capture_groups_fp 1393 -> 1436 and fixture_count 77 -> 83; synthetic scale counts remain 5012/16012. Prior aea33bf12f57561be7e3125929fb98cec7aed5a681e25aad435c7bb558747853 -> d56406c2645637042899cfcc8dc73f603d77caef0a9a2bac179ffbec848258a3; measured scaling 1.036 < 1.5.", "_rebaselined_3355_nested_packages": "#3355: new swift-nested-packages fixture (three nested Package.swift manifests, six sources) for nested-package module grouping. The capture query is unchanged; this is fixture-corpus growth only. capture_groups_fp 1333 -> 1393 and fixture_count 68 -> 77; synthetic scale counts remain 5012/16012. Prior c9fc553662f0db18027fba882e3ff730744f6153cc46eca7b00667fabd6a0df8 -> aea33bf12f57561be7e3125929fb98cec7aed5a681e25aad435c7bb558747853; measured scaling 1.023 < 1.5.", "scaling_budget": 1.5, @@ -187,7 +188,8 @@ "capture_groups_fp": 680 }, "typescript": { - "fingerprint": "77c9b4ea654123a64972db8348190ec250b669472b150db65ea8c7f20b355467", + "fingerprint": "a7972d87abd253583dafa37443bee8ddf5c635d41b4dfbb0c831efda71d82b47", + "_rebaselined_3446_mcp_tools_fixture": "#3446 adds typescript-mcp-tools/src/{handlers,server,tools}.ts: corpus growth only, with fixture_count 171 -> 174 and capture_groups_fp 2753 -> 2894. Excluding only that fixture directory restores the prior fingerprint 77c9b4ea654123a64972db8348190ec250b669472b150db65ea8c7f20b355467 exactly. The scope emitter, synthetic capture counts (4503/14403), scaling budget, and other language baselines are unchanged.", "_rebaselined_3190": "Capture matches now retain explicit ESM export/private evidence, including synthesized default HOCs; CommonJS surfaces remain undecided. Capture group counts unchanged. Scaling budget unchanged.", "scaling_budget": 1.5, "_rebaselined_2934_import_type_only": "#2934: `import-decomposer.ts` attaches a presence-only `@import.type-only` synthetic capture to specifiers `tsc` erases, so `check --cycles` can stop counting type-only edges as initialization cycles. DIGEST DRIFT ONLY, NOT A CAPTURE-SET CHANGE \u2014 the tag is added to import matches that already existed, never a new match, the same shape as the #2747 receiver-chain rebaseline. Every count is unchanged: capture_groups_fp 2414, fixture_count 155, capture_groups_small/large 4503/14403 (those measure the SYNTHETIC scaling source, which has no imports at all). The fingerprint moves because `canonicalizeMatch` in measure.mjs hashes every TAG on every match, synthetics included, so one extra presence-only tag on an existing match rewrites that match's canonical string. Attribution is exact, not inferred: neutralizing ONLY the `m['@import.type-only'] = \u2026` assignment in import-decomposer.ts and re-running returns the fingerprint to c2fbf8a89e5686dd\u2026 byte-for-byte, so nothing else in the TypeScript capture stream moved. All 14 other languages report ok. Scaling 0.997 < 1.5. NOTE ON THE CONTROL: javascript did not move (2026993b\u2026, 43 fixtures), but it is a WEAK control here \u2014 `import type` is TypeScript-only syntax, so a JS corpus cannot express the construct and could not have drifted either way. It evidences no collateral damage, not the correctness of the TS change; the exact-attribution check above is what does that. Prior c2fbf8a89e5686dd1ff3659b20d41d8b05ebcc9790356e3653ee0c8ca5d365c8 -> f719163eb03a447c9e40ca316a905dd76cee82192a75a403df478ebbdc13e98f.", diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 912ee05d5..5cc0fcbce 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "license": "PolyForm-Noncommercial-1.0.0", "dependencies": { - "@ladybugdb/core": "0.18.3", + "@ladybugdb/core": "0.21.1", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", "busboy": "^1.6.0", @@ -724,9 +724,9 @@ } }, "node_modules/@ladybugdb/core": { - "version": "0.18.3", - "resolved": "https://registry.npmjs.org/@ladybugdb/core/-/core-0.18.3.tgz", - "integrity": "sha512-XjpPKW4MrL28D2gYGTZuIjiEcPx12L21lx58QggrdrItw8o/e9Lmg/Ejoo4Kz08lZj+rIcC1Fu9thzIYOTUlJw==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core/-/core-0.21.1.tgz", + "integrity": "sha512-G0LWyn/dIqX9yHlyjpR12Rvou2o1dvCB9EkuzGa5ykCYbzur8EQhRUbi4fw3FlbckWvR6rK/tVbnFpBcRLwiHg==", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -735,17 +735,17 @@ "node-addon-api": "^6.0.0" }, "optionalDependencies": { - "@ladybugdb/core-darwin-arm64": "0.18.3", - "@ladybugdb/core-darwin-x64": "0.18.3", - "@ladybugdb/core-linux-arm64": "0.18.3", - "@ladybugdb/core-linux-x64": "0.18.3", - "@ladybugdb/core-win32-x64": "0.18.3" + "@ladybugdb/core-darwin-arm64": "0.21.1", + "@ladybugdb/core-darwin-x64": "0.21.1", + "@ladybugdb/core-linux-arm64": "0.21.1", + "@ladybugdb/core-linux-x64": "0.21.1", + "@ladybugdb/core-win32-x64": "0.21.1" } }, "node_modules/@ladybugdb/core-darwin-arm64": { - "version": "0.18.3", - "resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-arm64/-/core-darwin-arm64-0.18.3.tgz", - "integrity": "sha512-DGZTOlvSS4esEb1vTekY5IDoAvZAeYzR5cXVkECtQj9BVkk05zsvCAdTPo1Rz1BuI0qvqUVF+2WlIerI67iA2g==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-arm64/-/core-darwin-arm64-0.21.1.tgz", + "integrity": "sha512-TEFYNqBdbIojf1t29p3esgRhMDm56lD5eK91dwWFZJTVWAZoGXn+yLKs0NUQgj3lW2ZquuRTLZGglPhFEsDL1Q==", "cpu": [ "arm64" ], @@ -756,9 +756,9 @@ ] }, "node_modules/@ladybugdb/core-darwin-x64": { - "version": "0.18.3", - "resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-x64/-/core-darwin-x64-0.18.3.tgz", - "integrity": "sha512-Qp6j0CM/orBlK6KD0p/s4ofkIhNUwi1hdCgMw+fj81UHugWHkVLiYV4grRBdHhyplw+snchZpTxvfpxFbkG1Cw==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-x64/-/core-darwin-x64-0.21.1.tgz", + "integrity": "sha512-w9p3oKrqeMOeWpX3xdCc/hQJHfrZtpB9We+Ir+0qyls68dSQoeXXR1GZYzzTElOtxr4OM5USwYbVyxADx+CMZg==", "cpu": [ "x64" ], @@ -769,9 +769,9 @@ ] }, "node_modules/@ladybugdb/core-linux-arm64": { - "version": "0.18.3", - "resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-arm64/-/core-linux-arm64-0.18.3.tgz", - "integrity": "sha512-F9miYjBuS43I7uNG199FNMqwdHJ98WA6dU3v2SZCeLXmXCdRzmYcuHQWlbNr2Tba9CX58w2XvBZoUaXZKJ/yKQ==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-arm64/-/core-linux-arm64-0.21.1.tgz", + "integrity": "sha512-TodWmzOHxmmKvXXo9buBXGeyolxf/NqBUbo/LcR/tuMViKhGH8TBzIXi8DfVVwmsJ+CQCU065TvXgv2JoHAlZg==", "cpu": [ "arm64" ], @@ -782,9 +782,9 @@ ] }, "node_modules/@ladybugdb/core-linux-x64": { - "version": "0.18.3", - "resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-x64/-/core-linux-x64-0.18.3.tgz", - "integrity": "sha512-AfG5RDp/f/IDctDMpTAT5+2MYNtlWT191xiQNjSaWD4X85DhY3Dzps8Qu5VteIAPih5d6mmoaKGs8q0XIjfkFA==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-x64/-/core-linux-x64-0.21.1.tgz", + "integrity": "sha512-hCcLYv8ds0x4fkTaRkuc4f75KLdH9e0EV51ne6uBNdQSY7kyc0RgSoUG3utUGfFDMMkfgIJAeIxQY4ck9jQiXg==", "cpu": [ "x64" ], @@ -795,9 +795,9 @@ ] }, "node_modules/@ladybugdb/core-win32-x64": { - "version": "0.18.3", - "resolved": "https://registry.npmjs.org/@ladybugdb/core-win32-x64/-/core-win32-x64-0.18.3.tgz", - "integrity": "sha512-bHuFk0m9cnq0WGd9I4D8or8g6cC/BS58iatMtilqM3JpDPIQIFk6MQl6exL7P4xyWbkLwQgsrv2ToDnyoQNKvg==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core-win32-x64/-/core-win32-x64-0.21.1.tgz", + "integrity": "sha512-nBS1XSRJfpexdlhkezmEiRtkUt2qEDp4zBjIhH+oxgRdD7UsRz1Jyit8foJuZAmUsoQGukN5LO+YeyDU2WZTyA==", "cpu": [ "x64" ], @@ -854,9 +854,9 @@ } }, "node_modules/@nodable/entities": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", - "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.1.0.tgz", + "integrity": "sha512-LsS/DjHr+uDM647Gru/cA8+J3a3HfhttwCKLyuoyN7yXTFCxKBtSgMQBvrW9yNPa2/zDRUNuGPaH5QUFoa4arQ==", "funding": [ { "type": "github", @@ -1277,9 +1277,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.6.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.2.tgz", - "integrity": "sha512-X1P21scMv4zGKLYqjdGjaKa7COa0RKVYYZZN/NfvLQ1JegxFhdhpZG/Lyn8AXx6CDUavKAd11v6BvfpkDByK8g==", + "version": "26.6.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.3.tgz", + "integrity": "sha512-dsqMQQoeTLqu9wynDD00q573mNzso3IdQOAfHRJqLCcmCFPoGo9A1bDpUcv/9tnKpErQWv9uKeGfl37EIS02Yg==", "dev": true, "license": "MIT", "dependencies": { @@ -2025,9 +2025,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -2701,9 +2701,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", - "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", "funding": [ { "type": "github", @@ -2733,9 +2733,9 @@ } }, "node_modules/fast-xml-parser": { - "version": "5.11.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.1.tgz", - "integrity": "sha512-TBw6K/fxoQGGjCmZDw9w/ZwP3uDcnTM4YH/g+PFRWr8sbe5idXtxNN6vITh4+1ruCZaho6uBFurElsA7F0zzgw==", + "version": "5.11.2", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.2.tgz", + "integrity": "sha512-R9iDuNrQYeQut46cn2r2wHKn4HYzVDvDm5J1wW+koZewykv0yuO2HChTYeZtrULyHIDM9cj9TUXloCsueCQUog==", "funding": [ { "type": "github", @@ -2744,7 +2744,7 @@ ], "license": "MIT", "dependencies": { - "@nodable/entities": "^3.0.0", + "@nodable/entities": "^3.0.1", "fast-xml-builder": "^1.2.0", "is-unsafe": "^2.0.0", "path-expression-matcher": "^1.6.2", @@ -3130,9 +3130,9 @@ "license": "ISC" }, "node_modules/ip-address": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz", - "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", + "version": "10.7.2", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz", + "integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==", "license": "MIT", "engines": { "node": ">= 12" diff --git a/gitnexus/package.json b/gitnexus/package.json index c5ab6f3c6..1db831d0d 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -65,7 +65,7 @@ "version": "node scripts/sync-plugin-manifests.mjs" }, "dependencies": { - "@ladybugdb/core": "0.18.3", + "@ladybugdb/core": "0.21.1", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", "busboy": "^1.6.0", diff --git a/gitnexus/src/cli/detect-changes-format.ts b/gitnexus/src/cli/detect-changes-format.ts index 9e91cc330..9eece8e43 100644 --- a/gitnexus/src/cli/detect-changes-format.ts +++ b/gitnexus/src/cli/detect-changes-format.ts @@ -1,5 +1,6 @@ import { t } from './i18n/index.js'; import { formatSymbolLine } from './format-symbol.js'; +import { formatPathForTerminal } from './format-path.js'; type DetectChangesSummary = { changed_files?: number; @@ -32,6 +33,7 @@ type DetectChangesResult = { summary?: DetectChangesSummary; changed_symbols?: ChangedSymbol[]; affected_processes?: AffectedProcess[]; + unmapped_files?: string[]; }; export function formatDetectChangesResult(result: unknown): string { @@ -47,6 +49,13 @@ export function formatDetectChangesResult(result: unknown): string { // Both lead the output — a caveat printed after the summary is read too late. const notes: string[] = []; if (payload.partial) notes.push(t('tool.detectChanges.partial')); + if (payload.unmapped_files?.length) { + notes.push( + t('tool.detectChanges.unmappedSource', { + files: payload.unmapped_files.map(formatPathForTerminal).join(', '), + }), + ); + } // The plain truncation note reassures that the counts are whole. That is only // true when the run did NOT also degrade — `changed_count` sums the batches // that succeeded — so the two flags together get a different sentence. diff --git a/gitnexus/src/cli/embeddings-sync.ts b/gitnexus/src/cli/embeddings-sync.ts index 9848ef7c4..59cf9270a 100644 --- a/gitnexus/src/cli/embeddings-sync.ts +++ b/gitnexus/src/cli/embeddings-sync.ts @@ -4,7 +4,11 @@ import { LBUG_DIRECTORY } from '../storage/storage-constants.js'; import path from 'node:path'; import { cliInfo } from './cli-message.js'; import { getGitRoot } from '../storage/git.js'; -import { acquireIndexLock, requireExclusiveIndexLock } from '../storage/index-lock.js'; +import { + acquireIndexLock, + requireExclusiveIndexLock, + sweepStagingArtifacts, +} from '../storage/index-lock.js'; import { getStoragePaths, loadMeta, saveMeta } from '../storage/repo-manager.js'; import { closeLbug, @@ -50,12 +54,22 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = // Writes go to the slot's own graph. A shared-store checkout that reads an // immutable commit graph (#3352) takes a private copy first. const lbugPath = path.join(metaDir, LBUG_DIRECTORY); - const lock = await acquireIndexLock(metaDir); + const lock = await acquireIndexLock(metaDir, { sweep: false }); try { requireExclusiveIndexLock( lock, `Cannot acquire the index lock at ${metaDir}; refusing an unlocked embeddings sync.`, ); + // Sync writes the published graph and cannot recover a staged generation. + // Reject even malformed receipts before sweeping staging files or writing. + const recoveryCheckpoint = (await loadMeta(metaDir))?.embeddingCheckpoint; + if (recoveryCheckpoint && Object.hasOwn(recoveryCheckpoint, 'recovery')) { + throw new Error( + 'Cannot sync embeddings: the index checkpoint references staged embeddings. ' + + 'Run `gitnexus analyze` to recover them first.', + ); + } + sweepStagingArtifacts(metaDir); if (!(await ensurePrivateSharedGraph(metaDir, (m) => console.log(` ${m}`)))) { throw new Error('The shared graph this checkout reads is gone. Run gitnexus analyze first.'); } diff --git a/gitnexus/src/cli/format-path.ts b/gitnexus/src/cli/format-path.ts new file mode 100644 index 000000000..e7eab1f47 --- /dev/null +++ b/gitnexus/src/cli/format-path.ts @@ -0,0 +1,8 @@ +/** Quote control-bearing paths for terminal output; structured results retain raw paths. */ +export const formatPathForTerminal = (filePath: string): string => + /[\u0000-\u001f\u007f-\u009f]/.test(filePath) + ? JSON.stringify(filePath).replace( + /[\u007f-\u009f]/g, + (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`, + ) + : filePath; diff --git a/gitnexus/src/cli/group-status-format.ts b/gitnexus/src/cli/group-status-format.ts index f5d09c47e..bd94ccc9a 100644 --- a/gitnexus/src/cli/group-status-format.ts +++ b/gitnexus/src/cli/group-status-format.ts @@ -4,16 +4,18 @@ * against a real group, which is how `STALE (-1 commits behind)` went unnoticed * (#3256). */ +import type { StalenessStatus } from '../core/staleness-status.js'; /** The fields of a `groupStatus` repo row the index column reads. */ export interface GroupRepoIndexRow { indexStale: boolean; commitsBehind?: number; + status?: StalenessStatus; } /** - * The index column of a `group status` row. The output is unchanged except - * for one case: a count that is not a real count renders as `?`. + * The index column of a `group status` row. Diverged indexes differ from HEAD + * without a forward commit count; an unavailable count renders as `?`. * * `group/service.ts` has always reported a repo with no recorded commit as * `{ indexStale: true, commitsBehind: -1 }`. The previous `?? '?'` fallback @@ -21,6 +23,7 @@ export interface GroupRepoIndexRow { */ export const formatIndexStatusCell = (row: GroupRepoIndexRow): string => { if (!row.indexStale) return 'OK '; + if (row.status === 'diverged') return 'STALE (index differs from HEAD)'; const n = row.commitsBehind; const count = typeof n === 'number' && n >= 0 ? String(n) : '?'; return `STALE (${count} commits behind)`; diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 981071f64..8cedc843f 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -137,7 +137,9 @@ export const en = { 'tool.detectChanges.noOverlappingSymbols': 'Diff touched {{files}} file(s) but no indexed symbols overlap those hunks — not a clean tree.', 'tool.detectChanges.partial': - 'PARTIAL RESULT: a graph query failed, so changed symbols may be missing. Do not read this as a clean pre-commit check.', + 'PARTIAL RESULT: changed-symbol or process mapping is incomplete. Do not read this as a clean pre-commit check.', + 'tool.detectChanges.unmappedSource': + 'No symbols mapped for changed source files: {{files}}. Rebuild the index and inspect the diff; retry alone may not resolve missing or out-of-range symbols.', 'tool.detectChanges.truncated': 'LISTING CAPPED: the changed-symbol list was capped, so it does not name every changed symbol. The counts and risk level still cover all of them.', // The reassurance above is only true on its own. When the run also degraded, @@ -428,5 +430,5 @@ export const en = { 'help.identityCache.environment': '\nAnalyzer identity cache:\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir\n Operator-trusted persistent cache for warm cross-process status. The directory must pre-exist, be outside the GitNexus package/build roots, and contain no symlink or junction components. Defaults remain fail-closed on platforms without POSIX ownership APIs.', 'help.analyze.environment': - '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_STORAGE_PATH=/absolute/index Complete external index directory. Preserves the existing configuration semantics and overrides GITNEXUS_STORAGE_ROOT when both are set.\n GITNEXUS_STORAGE_ROOT=/absolute/root External index root; each repository uses an isolated -/ slot.\n GITNEXUS_CONTENT_RETENTION=full Source-text retention profile: full, symbol, or none. Default full.\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir Operator-trusted persistent analyzer identity cache; must pre-exist, be outside package/build roots, and contain no symlink/junction components.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N Max wait at pool shutdown for a retired worker still inside native code (terminated at its next safe point instead of aborting the process). Default 30000.\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N Per-file wall-clock budget for C++ capture extraction; on breach the file keeps partial captures with a warning. Default 20000.\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_EMBEDDING_RETRY_TIMEOUTS=1 Retry per-attempt HTTP embedding timeouts through GITNEXUS_EMBEDDING_MAX_ATTEMPTS (default off; timeouts stay terminal).\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n GITNEXUS_MAX_PROCESSES=N Process-detection process cap (positive integer). Replaces the dynamic max(20, round(symbols/10)) formula. Distinct from query-time IMPACT_MAX_CHUNKS.\n GITNEXUS_MAX_PROCESS_BRANCHING=N Process-detection per-node branching cap. Default 4.\n GITNEXUS_MAX_PROCESS_TRACE_DEPTH=N Process-detection DFS depth cap. Default 10.\n GITNEXUS_MAX_ENTRY_POINT_CANDIDATES=N Ranked entry-point candidate pool. Default 200. Raise when the warning names this knob; doubling is the usual first raise.\n\nCLI flags take precedence over `.gitnexusrc`, which takes precedence over env vars, which take precedence over built-in defaults.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).', + '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_STORAGE_PATH=/absolute/index Complete external index directory. Preserves the existing configuration semantics and overrides GITNEXUS_STORAGE_ROOT when both are set.\n GITNEXUS_STORAGE_ROOT=/absolute/root External index root; each repository uses an isolated -/ slot.\n GITNEXUS_CONTENT_RETENTION=full Source-text retention profile: full, symbol, or none. Default full.\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir Operator-trusted persistent analyzer identity cache; must pre-exist, be outside package/build roots, and contain no symlink/junction components.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N Max wait at pool shutdown for a retired worker still inside native code (terminated at its next safe point instead of aborting the process). Default 30000.\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N Per-file wall-clock budget for C++ capture extraction; on breach the file keeps partial captures with a warning. Default 20000.\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_EMBEDDING_RETRY_TIMEOUTS=1 Retry per-attempt HTTP embedding timeouts through GITNEXUS_EMBEDDING_MAX_ATTEMPTS (default off; timeouts stay terminal).\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for CLI/MCP query, 0.5 for standalone semantic search. Tune for your embedding model: higher values can improve recall but may reduce relevance. Set in the query/server environment; no reindex needed.\n GITNEXUS_MAX_PROCESSES=N Process-detection process cap (positive integer). Replaces the dynamic max(20, round(symbols/10)) formula. Distinct from query-time IMPACT_MAX_CHUNKS.\n GITNEXUS_MAX_PROCESS_BRANCHING=N Process-detection per-node branching cap. Default 4.\n GITNEXUS_MAX_PROCESS_TRACE_DEPTH=N Process-detection DFS depth cap. Default 10.\n GITNEXUS_MAX_ENTRY_POINT_CANDIDATES=N Ranked entry-point candidate pool. Default 200. Raise when the warning names this knob; doubling is the usual first raise.\n\nCLI flags take precedence over `.gitnexusrc`, which takes precedence over env vars, which take precedence over built-in defaults.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).', } as const; diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 606a9e4e6..f333188b7 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -128,7 +128,9 @@ export const zhCN = { 'tool.detectChanges.noOverlappingSymbols': 'diff 触及 {{files}} 个文件,但没有索引符号与这些 hunk 重叠 — 并非干净工作区。', 'tool.detectChanges.partial': - '结果不完整:图查询失败,可能遗漏已变更符号。请勿将其视为通过的提交前检查。', + '结果不完整:变更符号或流程映射不完整。请勿将其视为通过的提交前检查。', + 'tool.detectChanges.unmappedSource': + '变更源码文件未映射到符号:{{files}}。请重建索引并检查 diff;仅重试可能无法解决符号缺失或源码范围不匹配。', 'tool.detectChanges.truncated': '列表已截断:已变更符号列表被截断,未列出全部变更符号。计数与风险等级仍涵盖全部符号。', 'tool.detectChanges.truncatedDegraded': @@ -390,5 +392,5 @@ export const zhCN = { 'help.identityCache.environment': '\n分析器身份缓存:\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir\n 由操作员明确信任的持久缓存,用于跨进程快速查询状态。目录必须预先存在、位于 GitNexus 包/构建根目录之外,且路径中不得包含符号链接或 junction。缺少 POSIX 所有权 API 的平台默认保持故障关闭。', 'help.analyze.environment': - '\n环境变量:\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析(仍读取 .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值(KB)。默认 512,最大 32768。\n GITNEXUS_STORAGE_PATH=/absolute/index 完整外部索引目录。保留既有配置语义;与 GITNEXUS_STORAGE_ROOT 同时设置时优先使用。\n GITNEXUS_STORAGE_ROOT=/absolute/root 外部索引根目录;每个仓库使用独立的 <仓库名>-<规范路径哈希>/ 子目录。\n GITNEXUS_CONTENT_RETENTION=full 源码文本保留策略:full、symbol 或 none。默认 full。\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir 由操作员明确信任的持久分析器身份缓存;目录必须预先存在、位于包/构建根目录之外,且路径中不得包含符号链接或 junction。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时(毫秒)。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值(字节,默认 67108864 = 64 MiB;-1 保持 Ladybug 默认约 16 MiB)。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1,最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N 线程池关闭时等待仍在原生代码中的已退役 worker 的最长时间(到达安全点后再终止,避免进程级 abort)。默认 30000。\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N C++ 捕获提取的每文件墙钟预算;超出后该文件保留部分捕获并输出警告。默认 20000。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_EMBEDDING_RETRY_TIMEOUTS=1 将单次 HTTP 嵌入超时纳入 GITNEXUS_EMBEDDING_MAX_ATTEMPTS 重试(默认关闭,超时仍为终止错误)。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n GITNEXUS_VECTOR_MAX_DISTANCE=N 语义/向量搜索接受的最大余弦距离(0 < N <= 2;超出则钳制为 2)。MCP 默认 0.6,其他路径默认 0.5。\n GITNEXUS_MAX_PROCESSES=N 流程检测的流程数量上限(正整数)。覆盖动态的 max(20, round(symbols/10)) 公式。与查询时的 IMPACT_MAX_CHUNKS 无关。\n GITNEXUS_MAX_PROCESS_BRANCHING=N 流程检测的单节点分支上限。默认 4。\n GITNEXUS_MAX_PROCESS_TRACE_DEPTH=N 流程检测的 DFS 深度上限。默认 10。\n GITNEXUS_MAX_ENTRY_POINT_CANDIDATES=N 排序后的入口点候选池。默认 200。仅在警告点名该上限时提高;那时通常先翻倍。\n\nCLI 参数优先于 `.gitnexusrc`,后者优先于环境变量,环境变量优先于内置默认值。\n\n提示:`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录(#771)。', + '\n环境变量:\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析(仍读取 .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值(KB)。默认 512,最大 32768。\n GITNEXUS_STORAGE_PATH=/absolute/index 完整外部索引目录。保留既有配置语义;与 GITNEXUS_STORAGE_ROOT 同时设置时优先使用。\n GITNEXUS_STORAGE_ROOT=/absolute/root 外部索引根目录;每个仓库使用独立的 <仓库名>-<规范路径哈希>/ 子目录。\n GITNEXUS_CONTENT_RETENTION=full 源码文本保留策略:full、symbol 或 none。默认 full。\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir 由操作员明确信任的持久分析器身份缓存;目录必须预先存在、位于包/构建根目录之外,且路径中不得包含符号链接或 junction。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时(毫秒)。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值(字节,默认 67108864 = 64 MiB;-1 保持 Ladybug 默认约 16 MiB)。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1,最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N 线程池关闭时等待仍在原生代码中的已退役 worker 的最长时间(到达安全点后再终止,避免进程级 abort)。默认 30000。\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N C++ 捕获提取的每文件墙钟预算;超出后该文件保留部分捕获并输出警告。默认 20000。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_EMBEDDING_RETRY_TIMEOUTS=1 将单次 HTTP 嵌入超时纳入 GITNEXUS_EMBEDDING_MAX_ATTEMPTS 重试(默认关闭,超时仍为终止错误)。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n GITNEXUS_VECTOR_MAX_DISTANCE=N 语义/向量搜索接受的最大余弦距离(0 < N <= 2;超出则钳制为 2)。CLI/MCP query 默认 0.6,独立语义搜索默认 0.5。应根据嵌入模型调节:提高阈值可增加召回,但可能降低相关性。在查询/服务器进程的环境中设置,无需重新索引。\n GITNEXUS_MAX_PROCESSES=N 流程检测的流程数量上限(正整数)。覆盖动态的 max(20, round(symbols/10)) 公式。与查询时的 IMPACT_MAX_CHUNKS 无关。\n GITNEXUS_MAX_PROCESS_BRANCHING=N 流程检测的单节点分支上限。默认 4。\n GITNEXUS_MAX_PROCESS_TRACE_DEPTH=N 流程检测的 DFS 深度上限。默认 10。\n GITNEXUS_MAX_ENTRY_POINT_CANDIDATES=N 排序后的入口点候选池。默认 200。仅在警告点名该上限时提高;那时通常先翻倍。\n\nCLI 参数优先于 `.gitnexusrc`,后者优先于环境变量,环境变量优先于内置默认值。\n\n提示:`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录(#771)。', } satisfies EnglishMessages; diff --git a/gitnexus/src/cli/skill-gen.ts b/gitnexus/src/cli/skill-gen.ts index f1dd45c3b..ed7f58232 100644 --- a/gitnexus/src/cli/skill-gen.ts +++ b/gitnexus/src/cli/skill-gen.ts @@ -114,21 +114,21 @@ export const generateSkillFiles = async ( } } - if (!communityResult || !communityResult.memberships.length) { + const memberships = communityResult?.rawMemberships ?? communityResult?.memberships ?? []; + if (!communityResult || !memberships.length) { console.log('\n Skills: no communities detected, skipping skill generation'); return { skills: [], outputPath: outputDir }; } console.log('\n Generating repo-specific skills...'); - // Step 1: Build communities from memberships (not the filtered communities array). - // The community processor skips singletons from its communities array but memberships - // include ALL assignments. For repos with sparse CALLS edges, the communities array - // can be empty while memberships still has useful groupings. + // Step 1: Use raw assignments for the fallback when all communities were + // filtered as singletons. Same-folder aggregation can still produce skills + // for these sparse graphs without emitting dangling MEMBER_OF edges. const communities = communityResult.communities.length > 0 ? communityResult.communities - : buildCommunitiesFromMemberships(communityResult.memberships, graph, repoPath); + : buildCommunitiesFromMemberships(memberships, graph, repoPath); const aggregated = aggregateCommunities(communities); @@ -145,11 +145,8 @@ export const generateSkillFiles = async ( } // Step 3: Build lookup maps - const membershipsByComm = buildMembershipMap(communityResult.memberships); - const nodeIdToCommunityLabel = buildNodeCommunityLabelMap( - communityResult.memberships, - communities, - ); + const membershipsByComm = buildMembershipMap(memberships); + const nodeIdToCommunityLabel = buildNodeCommunityLabelMap(memberships, communities); // Step 4: Ensure the shared project-skill root exists. Never clear it: it // also contains user-authored and standard GitNexus skills. @@ -185,7 +182,7 @@ export const generateSkillFiles = async ( const entryPoints = gatherEntryPoints(members); // Gather execution flows - const flows = gatherFlows(community.rawIds, processResult?.processes || []); + const flows = gatherFlows(community.rawIds, members, processResult?.processes || []); // Gather cross-community connections const connections = gatherCrossConnections( @@ -529,14 +526,26 @@ const gatherEntryPoints = (members: MemberSymbol[]): MemberSymbol[] => { /** * @brief Gather execution flows touching this community * @param {string[]} rawIds - Raw community IDs for this aggregated community + * @param {MemberSymbol[]} members - Member symbols, including raw singleton assignments * @param {ProcessNode[]} processes - All detected processes - * @returns {ProcessNode[]} Processes whose communities intersect rawIds, sorted by stepCount + * @returns {ProcessNode[]} Processes matching the community IDs or member symbols, sorted by stepCount */ -const gatherFlows = (rawIds: string[], processes: ProcessNode[]): ProcessNode[] => { +const gatherFlows = ( + rawIds: string[], + members: MemberSymbol[], + processes: ProcessNode[], +): ProcessNode[] => { const rawIdSet = new Set(rawIds); + const memberIds = new Set(members.map((member) => member.id)); return processes - .filter((proc) => proc.communities.some((cid) => rawIdSet.has(cid))) + .filter( + (proc) => + proc.communities.some((cid) => rawIdSet.has(cid)) || + // Filtered singleton communities are absent from process metadata, + // but their symbols still participate in detected execution traces. + proc.trace.some((nodeId) => memberIds.has(nodeId)), + ) .sort((a, b) => b.stepCount - a.stepCount); }; diff --git a/gitnexus/src/cli/status.ts b/gitnexus/src/cli/status.ts index 26dd780b4..8316fc3ae 100644 --- a/gitnexus/src/cli/status.ts +++ b/gitnexus/src/cli/status.ts @@ -48,6 +48,7 @@ import { isFullSourceAvailable, } from '../core/content-retention.js'; import { t } from './i18n/index.js'; +import { formatPathForTerminal as formatDriftPath } from './format-path.js'; /** How many drifted paths the report names before summarizing the rest. */ const DRIFT_SAMPLE_LIMIT = 10; @@ -84,9 +85,6 @@ const describeContentDrift = (drift: IndexContentDrift | undefined) => { }; }; -/** Escape control characters in repo-relative paths before printing. */ -const formatDriftPath = (rel: string): string => - /[\u0000-\u001f\u007f]/.test(rel) ? JSON.stringify(rel) : rel; const printDriftDetail = (drift: Extract): void => { console.log( t('status.indexContentDrifted', { diff --git a/gitnexus/src/config/ignore-service.ts b/gitnexus/src/config/ignore-service.ts index ac2dc7704..6af0e479a 100644 --- a/gitnexus/src/config/ignore-service.ts +++ b/gitnexus/src/config/ignore-service.ts @@ -1,5 +1,13 @@ import ignore, { type Ignore } from 'ignore'; -import { existsSync } from 'fs'; +import { + closeSync, + constants as fsConstants, + existsSync, + fstatSync, + lstatSync, + openSync, + readFileSync, +} from 'fs'; import fs from 'fs/promises'; import nodePath from 'path'; import type { Path } from 'path-scurry'; @@ -531,9 +539,163 @@ const hasExplicitUnignore = (ig: Ignore, rel: string): boolean => { return false; }; +/** + * Read a nested `.gitignore` only if it is a regular file, not a symlink. + * + * git does not follow a symlinked `.gitignore` in the working tree, and + * reading one could pull rules from outside the repository. Where the + * platform supports it, the file is opened with O_NOFOLLOW (a symlink fails + * with ELOOP). Windows has no O_NOFOLLOW, so there the path is lstat'ed after + * opening and must be the same regular file as the open descriptor. Either + * way the content is read through the descriptor that was checked, never by + * path, so the file cannot be swapped between the check and the read. + * + * The open also passes O_NONBLOCK where it exists. Opening a FIFO for reading + * blocks in open(2) until a writer appears, so a `.gitignore` that is a FIFO + * would hang the scan before the isFile() check could reject it (glob's + * ignore callback is synchronous). The flag makes that open return at once + * and changes nothing for a regular file. Same reasoning as readBoundedFile + * in src/core/ingestion/asyncapi/document.ts. + */ +const readNestedGitignore = (filePath: string): string | null => { + const noFollow = fsConstants.O_NOFOLLOW; + const nonBlock = fsConstants.O_NONBLOCK; + const fd = openSync(filePath, fsConstants.O_RDONLY | (noFollow ?? 0) | (nonBlock ?? 0)); + try { + const stat = fstatSync(fd); + if (!stat.isFile()) return null; + if (noFollow === undefined) { + const link = lstatSync(filePath); + if (!link.isFile() || link.ino !== stat.ino || link.dev !== stat.dev) return null; + } + return readFileSync(fd, 'utf-8'); + } finally { + closeSync(fd); + } +}; + +/** + * Resolve `.gitignore` files below the repository root (#2675). + * + * `loadIgnoreRules` only reads the root `.gitignore`, so a monorepo package + * or checked-out submodule with its own `.gitignore` had its generated + * output indexed anyway. Each nested file is read lazily (glob's filter is + * synchronous) and cached per directory, and its patterns are matched + * against the path relative to that directory, like git does. + * + * Returns the effective decision when nested rules affect the path or an + * ancestor, and `undefined` otherwise. Root rules participate so a directory + * negation does not erase independent root exclusions for its children. + * The caller still gives `.gitnexusignore` its higher precedence. + */ +const createNestedGitignoreMatcher = ( + repoPath: string, + rootRules: Ignore | null, +): ((rel: string, isDirectory: boolean) => boolean | undefined) => { + const rulesFor = (dirRel: string): Ignore | null => { + let rules: Ignore | null = null; + const filePath = nodePath.join(repoPath, dirRel, '.gitignore'); + try { + const content = readNestedGitignore(filePath); + if (content !== null) rules = ignore().add(content); + } catch (err: unknown) { + const code = (err as NodeJS.ErrnoException).code; + if (code !== 'ENOENT' && code !== 'ENOTDIR' && code !== 'ELOOP') { + logger.warn(` Warning: could not read ${filePath}: ${(err as Error).message}`); + } + } + return rules; + }; + + interface Scope { + base: string; + rules: Ignore; + } + interface DirectoryContext { + scopes: Scope[]; + ignored: boolean; + nested: boolean; + } + + const relativeTo = (base: string, rel: string): string => + base ? rel.slice(base.length + 1) : rel; + const match = (scopes: Scope[], rel: string, isDirectory: boolean) => { + for (let i = scopes.length - 1; i >= 0; i--) { + const { base, rules } = scopes[i]; + const sub = relativeTo(base, rel); + const result = rules.test(isDirectory ? `${sub}/` : sub); + if (result.ignored || result.unignored) { + return { ignored: result.ignored, nested: base !== '' }; + } + } + return undefined; + }; + + const contexts = new Map([ + [ + '', + { + scopes: rootRules ? [{ base: '', rules: rootRules }] : [], + ignored: false, + nested: false, + }, + ], + ]); + + const contextFor = (dir: string): DirectoryContext => { + const cached = contexts.get(dir); + if (cached) return cached; + const parentDir = nodePath.posix.dirname(dir); + const parent = contextFor(parentDir === '.' ? '' : parentDir); + // A .gitignore inside an excluded directory cannot bring that directory + // back. Do not read rules below a parent that traversal would prune. + if (parent.ignored) { + contexts.set(dir, parent); + return parent; + } + + const result = match(parent.scopes, dir, true); + const context: DirectoryContext = { + scopes: parent.scopes, + ignored: result?.ignored ?? false, + nested: parent.nested || (result?.nested ?? false), + }; + if (!context.ignored) { + context.scopes = parent.scopes.map(({ base, rules }) => { + const sub = relativeTo(base, dir); + if (!rules.test(`${sub}/`).ignored) return { base, rules }; + // A deeper rule let us enter this directory. Clear only its inherited + // exclusion in the shallower layer; child rules must still be tested. + // Keep patterns in their original scope, and escape this literal path. + const literal = sub.replace(/[\\*?\[\]]/g, '\\$&'); + return { + base, + rules: ignore() + .add(rules) + .add({ pattern: `!/${literal}/` }), + }; + }); + const rules = rulesFor(dir); + if (rules) context.scopes.push({ base: dir, rules }); + } + contexts.set(dir, context); + return context; + }; + + return (rel: string, isDirectory: boolean): boolean | undefined => { + const parentDir = nodePath.posix.dirname(rel); + const parent = contextFor(parentDir === '.' ? '' : parentDir); + if (parent.ignored) return parent.nested ? true : undefined; + const result = match(parent.scopes, rel, isDirectory); + if (parent.nested || result?.nested) return result?.ignored ?? false; + return undefined; + }; +}; + /** * Create a glob-compatible ignore filter combining: * - .gitignore / .gitnexusignore patterns (via `ignore` package) + * - nested .gitignore files, scoped to their own directory (#2675) * - Hardcoded DEFAULT_IGNORE_LIST, IGNORED_EXTENSIONS, IGNORED_FILES * * Returns an IgnoreLike object for glob's `ignore` option, @@ -550,6 +712,13 @@ const hasExplicitUnignore = (ig: Ignore, rel: string): boolean => { */ export const createIgnoreFilter = async (repoPath: string, options?: IgnoreOptions) => { const ig = await loadIgnoreRules(repoPath, options); + const skipGitignore = options?.noGitignore ?? !!process.env.GITNEXUS_NO_GITIGNORE; + const nestedIgnores = skipGitignore ? null : createNestedGitignoreMatcher(repoPath, ig); + // A nested negation outranks the root .gitignore, as in git, but not the + // user's .gitnexusignore, so keep a matcher for that file on its own. + const nexusIgnore = nestedIgnores + ? await loadIgnoreRules(repoPath, { ...options, noGitignore: true, noGlobalIgnore: true }) + : null; return { ignored(p: Path): boolean { @@ -557,6 +726,23 @@ export const createIgnoreFilter = async (repoPath: string, options?: IgnoreOptio // native separators on Windows when called through glob. const rel = p.relative().replace(/\\/g, '/'); if (!rel) return false; + // Nested .gitignore files below the root (#2675). .gitnexusignore + // comes first, then the deepest nested .gitignore, which outranks the + // root .gitignore as in git. The nested matcher preserves independent + // root exclusions; a nested negation never rescues a hardcoded default. + // With no nested opinion the original order below applies unchanged. + if (nestedIgnores) { + if (nexusIgnore) { + if (hasExplicitUnignore(nexusIgnore, rel) && !ig?.ignores(rel)) return false; + if (nexusIgnore.ignores(rel)) return true; + } + const nested = nestedIgnores(rel, false); + if (nested === true) return true; + if (nested === false) { + if (ig && hasExplicitUnignore(ig, rel) && !ig.ignores(rel)) return false; + return shouldIgnorePath(rel); + } + } // User's .gitnexusignore negation takes precedence over hardcoded // rules (#771). If any ancestor or the path itself was explicitly // unignored AND no more-specific rule re-ignores this exact path, @@ -576,6 +762,22 @@ export const createIgnoreFilter = async (repoPath: string, options?: IgnoreOptio // list check below is defense-in-depth — do not remove `dot: false` // assuming this covers it. const rel = p.relative().replace(/\\/g, '/'); + // Nested .gitignore files below the root (#2675), same precedence as in + // `ignored` above. + if (nestedIgnores && rel) { + if (nexusIgnore) { + if (hasExplicitUnignore(nexusIgnore, rel) && !ig?.ignores(rel + '/')) { + return false; + } + if (nexusIgnore.ignores(rel + '/')) return true; + } + const nested = nestedIgnores(rel, true); + if (nested === true) return true; + if (nested === false) { + if (ig && hasExplicitUnignore(ig, rel) && !ig.ignores(rel + '/')) return false; + return isHardcodedIgnoredDirectoryAtPath(repoPath, nodePath.join(repoPath, rel)); + } + } // User's .gitnexusignore negation takes precedence (#771) — if the // user explicitly unignored this directory or any ancestor via a // !pattern rule, allow descent even if the directory name is in diff --git a/gitnexus/src/core/embeddings/staged-embedding-recovery-child.ts b/gitnexus/src/core/embeddings/staged-embedding-recovery-child.ts new file mode 100644 index 000000000..b1fb2cb73 --- /dev/null +++ b/gitnexus/src/core/embeddings/staged-embedding-recovery-child.ts @@ -0,0 +1,181 @@ +/** Isolated strict native reader. Never import this entrypoint into analyze. */ +import fs from 'node:fs'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { createLbugDatabase, toNativeSafePath } from '../lbug/lbug-config.js'; +import { FAMILY_SUFFIXES } from '../../storage/embedding-recovery.js'; +import { + abortCachedEmbeddingsBuilder, + createCachedEmbeddingsBuilder, + finalizeCachedEmbeddingsSnapshot, + ingestCachedEmbeddingRow, +} from './embedding-restore-spill.js'; +import type { StagedEmbeddingExport } from './staged-embedding-recovery.js'; + +/** Native replay and close may checkpoint; only give them disposable copies. */ +function copyRecoveryFamily(dbPath: string, exportDir: string): string { + const replayDir = fs.mkdtempSync(path.join(exportDir, 'replay-')); + const replayPath = path.join(replayDir, path.basename(dbPath)); + const noFollow = fs.constants.O_NOFOLLOW ?? 0; + const flags = fs.constants.O_RDONLY | noFollow | (fs.constants.O_NONBLOCK ?? 0); + const buffer = Buffer.allocUnsafe(1024 * 1024); + for (const suffix of FAMILY_SUFFIXES) { + const sourcePath = dbPath + suffix; + let entry: fs.BigIntStats; + try { + entry = fs.lstatSync(sourcePath, { bigint: true }); + } catch (error) { + if (suffix && (error as NodeJS.ErrnoException).code === 'ENOENT') continue; + throw error; + } + if (!entry.isFile()) throw new Error('staged embedding family is not a regular file'); + const source = fs.openSync(sourcePath, flags); + let destination: number | undefined; + try { + const opened = fs.fstatSync(source, { bigint: true }); + const current = fs.lstatSync(sourcePath, { bigint: true }); + if ( + !opened.isFile() || + !current.isFile() || + (noFollow === 0 && opened.ino === 0n) || + opened.dev !== entry.dev || + opened.ino !== entry.ino || + opened.dev !== current.dev || + opened.ino !== current.ino + ) { + throw new Error('staged embedding family changed while opening'); + } + destination = fs.openSync(replayPath + suffix, 'wx', 0o600); + let copied = 0n; + for (;;) { + const bytesRead = fs.readSync(source, buffer, 0, buffer.length, null); + if (bytesRead === 0) break; + fs.writeFileSync(destination, buffer.subarray(0, bytesRead)); + copied += BigInt(bytesRead); + } + const after = fs.fstatSync(source, { bigint: true }); + if ( + copied !== opened.size || + after.size !== opened.size || + after.mtimeNs !== opened.mtimeNs || + after.ctimeNs !== opened.ctimeNs + ) { + throw new Error('staged embedding family changed while copying'); + } + } finally { + try { + if (destination !== undefined) fs.closeSync(destination); + } finally { + fs.closeSync(source); + } + } + } + return replayPath; +} + +async function extract(): Promise { + const [dbPath, exportDir, dimensionsArg] = process.argv.slice(2); + const dimensions = Number(dimensionsArg); + if (!dbPath || !exportDir || !Number.isInteger(dimensions) || dimensions <= 0) { + throw new Error('invalid staged embedding extraction arguments'); + } + // The parent owns exportDir and reclaims it even after killing this child. + const replayPath = copyRecoveryFamily(dbPath, exportDir); + const builder = createCachedEmbeddingsBuilder({ inMemoryRowLimit: 0, spillDir: exportDir }); + const rejectedNodeIds = new Set(); + let db: lbug.Database | undefined; + let conn: lbug.Connection | undefined; + try { + // Avoid openLbugConnection's test-fixture lock sweep: a recovery source must + // never have its WAL removed, even when an external slot resembles a fixture. + db = createLbugDatabase(lbug, toNativeSafePath(replayPath), { throwOnWalReplayFailure: true }); + conn = new lbug.Connection(db); + const queried = await conn.query( + 'MATCH (e:CodeEmbedding) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding, e.contentHash AS contentHash', + ); + const results = Array.isArray(queried) ? queried : [queried]; + try { + if (results.length !== 1) throw new Error('unexpected staged embedding query result'); + const result = results[0]; + while (await result.hasNext()) { + const raw = await result.getNext(); + const rec = raw as Record & unknown[]; + const nodeId = rec.nodeId ?? rec[0]; + if (typeof nodeId !== 'string' || !nodeId) + throw new Error('invalid staged embedding node id'); + const chunkIndex = rec.chunkIndex ?? rec[1]; + const startLine = rec.startLine ?? rec[2]; + const endLine = rec.endLine ?? rec[3]; + const embedding = rec.embedding ?? rec[4]; + const contentHash = rec.contentHash ?? rec[5]; + const vector = + Array.isArray(embedding) || + (ArrayBuffer.isView(embedding) && !(embedding instanceof DataView)) + ? Array.from(embedding as ArrayLike) + : undefined; + if ( + !Number.isInteger(chunkIndex) || + Number(chunkIndex) < 0 || + !Number.isInteger(startLine) || + Number(startLine) < 0 || + !Number.isInteger(endLine) || + Number(endLine) < Number(startLine) || + typeof contentHash !== 'string' || + !contentHash || + !vector || + vector.length !== dimensions || + vector.some( + (value) => + typeof value !== 'number' || + !Number.isFinite(value) || + !Number.isFinite(Math.fround(value)), + ) + ) { + rejectedNodeIds.add(nodeId); + continue; + } + ingestCachedEmbeddingRow( + builder, + { nodeId, chunkIndex, startLine, endLine, embedding: vector, contentHash }, + true, + ); + } + } finally { + for (const result of results) await result.close(); + } + // Both closes must succeed. Suppressed native teardown errors are unsafe. + await conn.close(); + await db.close(); + const snapshot = finalizeCachedEmbeddingsSnapshot(builder); + if (snapshot.spill) fs.renameSync(snapshot.spill.path, path.join(exportDir, 'vectors.bin')); + const manifest: StagedEmbeddingExport = { + version: 1, + dimensions, + rows: snapshot.rows, + rejectedNodeIds: [...rejectedNodeIds], + }; + fs.writeFileSync(path.join(exportDir, 'manifest.json'), JSON.stringify(manifest), { + flag: 'wx', + mode: 0o600, + }); + } catch (err) { + abortCachedEmbeddingsBuilder(builder); + // Cleanup is best effort on a rejected source, never used to approve output. + try { + await conn?.close(); + } catch { + /* rejected */ + } + try { + await db?.close(); + } catch { + /* rejected */ + } + throw err; + } +} + +extract().catch((err: unknown) => { + process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); + process.exitCode = 1; +}); diff --git a/gitnexus/src/core/embeddings/staged-embedding-recovery.ts b/gitnexus/src/core/embeddings/staged-embedding-recovery.ts new file mode 100644 index 000000000..b295f4beb --- /dev/null +++ b/gitnexus/src/core/embeddings/staged-embedding-recovery.ts @@ -0,0 +1,244 @@ +/** Recover paid embedding rows without opening an interrupted native DB in analyze. */ +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + abortCachedEmbeddingsBuilder, + createCachedEmbeddingsBuilder, + EmbeddingSpillReader, + emptyCachedEmbeddingsSnapshot, + finalizeCachedEmbeddingsSnapshot, + ingestCachedEmbeddingRow, + materializeCachedEmbeddings, + type CachedEmbeddingMeta, + type CachedEmbeddingsSnapshot, +} from './embedding-restore-spill.js'; + +export interface StagedEmbeddingRecoveryOptions { + dimensions: number; + /** Active checkpoint window and any incomplete inherited restore groups. */ + excludedNodeIds?: Iterable; + timeoutMs?: number; +} + +export interface StagedEmbeddingExport { + version: 1; + dimensions: number; + rows: CachedEmbeddingMeta[]; + rejectedNodeIds: string[]; +} + +const RESTORE_BATCH_SIZE = 200; +const DEFAULT_EXTRACTION_TIMEOUT_MS = 120_000; + +/** A contiguous prefix is safe only when its node is outside every unsafe window. */ +export function validateRecoveredNodeGroups( + rows: readonly CachedEmbeddingMeta[], + excludedNodeIds: ReadonlySet = new Set(), +): Set { + const groups = new Map; invalid: boolean }>(); + for (const row of rows) { + let group = groups.get(row.nodeId); + if (!group) { + group = { hash: row.contentHash, ordinals: new Set(), invalid: false }; + groups.set(row.nodeId, group); + } + if ( + typeof row.nodeId !== 'string' || + !row.nodeId || + typeof row.contentHash !== 'string' || + !row.contentHash || + row.contentHash !== group.hash || + !Number.isInteger(row.chunkIndex) || + row.chunkIndex < 0 || + group.ordinals.has(row.chunkIndex) || + !Number.isInteger(row.startLine) || + row.startLine < 0 || + !Number.isInteger(row.endLine) || + row.endLine < row.startLine + ) + group.invalid = true; + group.ordinals.add(row.chunkIndex); + } + const accepted = new Set(); + for (const [nodeId, group] of groups) { + if (group.invalid || excludedNodeIds.has(nodeId)) continue; + // Unique ordinals with max n-1 and zero present have no holes. + if (!group.ordinals.has(0)) continue; + if ([...group.ordinals].some((ordinal) => ordinal >= group.ordinals.size)) continue; + accepted.add(nodeId); + } + return accepted; +} + +/** Whole recovered nodes replace whole published groups; vectors stay in bounded batches. */ +export function mergeRecoveredEmbeddings( + live: CachedEmbeddingsSnapshot, + recovered: CachedEmbeddingsSnapshot, +): CachedEmbeddingsSnapshot { + const builder = createCachedEmbeddingsBuilder({ inMemoryRowLimit: 0 }); + try { + appendSnapshotRows( + live, + live.rows.filter((row) => !recovered.embeddingNodeIds.has(row.nodeId)), + builder, + ); + appendSnapshotRows(recovered, recovered.rows, builder); + return finalizeCachedEmbeddingsSnapshot(builder); + } catch (err) { + abortCachedEmbeddingsBuilder(builder); + throw err; + } +} + +function appendSnapshotRows( + snapshot: CachedEmbeddingsSnapshot, + rows: readonly CachedEmbeddingMeta[], + builder: ReturnType, +): void { + const reader = snapshot.spill ? new EmbeddingSpillReader(snapshot.spill) : undefined; + try { + for (let i = 0; i < rows.length; i += RESTORE_BATCH_SIZE) { + const batch = materializeCachedEmbeddings( + snapshot, + rows.slice(i, i + RESTORE_BATCH_SIZE), + reader, + ); + for (const row of batch) + ingestCachedEmbeddingRow(builder, row as unknown as Record, true); + } + } finally { + reader?.close(); + } +} + +/** + * Caller must validate checkpoint identity, schema, exact generation, and hold the + * index lock. A subprocess contains native WAL replay/query/destructor failures. + * A failed strict open is never retried with WAL removed or validation disabled. + */ +export async function recoverStagedEmbeddings( + dbPath: string, + options: StagedEmbeddingRecoveryOptions, +): Promise { + if (!Number.isInteger(options.dimensions) || options.dimensions <= 0) { + throw new Error('invalid staged embedding dimensions'); + } + const dbStat = fs.lstatSync(dbPath); + if (!dbStat.isFile() || dbStat.isSymbolicLink()) + throw new Error('staged embedding DB is not a regular file'); + const exportDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-stage-export-')); + try { + await runExtractionChild(dbPath, exportDir, options); + const manifestPath = path.join(exportDir, 'manifest.json'); + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as StagedEmbeddingExport; + if ( + manifest.version !== 1 || + manifest.dimensions !== options.dimensions || + !Array.isArray(manifest.rows) || + !Array.isArray(manifest.rejectedNodeIds) || + manifest.rejectedNodeIds.some((id) => typeof id !== 'string') || + manifest.rows.some((row, i) => !row || row.vectorIndex !== i) + ) + throw new Error('invalid staged embedding export manifest'); + if (manifest.rows.length === 0) return emptyCachedEmbeddingsSnapshot(); + const spill = { + path: path.join(exportDir, 'vectors.bin'), + dims: options.dimensions, + rowCount: manifest.rows.length, + }; + const vectorStat = fs.lstatSync(spill.path); + if ( + !vectorStat.isFile() || + vectorStat.isSymbolicLink() || + vectorStat.size !== 12 + spill.rowCount * spill.dims * 4 + ) { + throw new Error('invalid staged embedding export size'); + } + const excluded = new Set(options.excludedNodeIds ?? []); + for (const nodeId of manifest.rejectedNodeIds) excluded.add(nodeId); + const accepted = validateRecoveredNodeGroups(manifest.rows, excluded); + const exported: CachedEmbeddingsSnapshot = { + rows: manifest.rows, + embeddings: [], + embeddingNodeIds: accepted, + spill, + }; + const builder = createCachedEmbeddingsBuilder({ inMemoryRowLimit: 0 }); + const reader = new EmbeddingSpillReader(spill); + try { + const rows = manifest.rows.filter((row) => accepted.has(row.nodeId)); + for (let i = 0; i < rows.length; i += RESTORE_BATCH_SIZE) { + for (const row of materializeCachedEmbeddings( + exported, + rows.slice(i, i + RESTORE_BATCH_SIZE), + reader, + )) { + if ( + row.embedding.length !== options.dimensions || + row.embedding.some((value) => !Number.isFinite(value)) + ) { + throw new Error('invalid staged embedding vector'); + } + ingestCachedEmbeddingRow(builder, row as unknown as Record, true); + } + } + return finalizeCachedEmbeddingsSnapshot(builder); + } catch (err) { + abortCachedEmbeddingsBuilder(builder); + throw err; + } finally { + reader.close(); + } + } finally { + fs.rmSync(exportDir, { recursive: true, force: true }); + } +} + +function runExtractionChild( + dbPath: string, + exportDir: string, + options: StagedEmbeddingRecoveryOptions, +): Promise { + const compiledPath = fileURLToPath( + new URL('./staged-embedding-recovery-child.js', import.meta.url), + ); + const sourcePath = compiledPath.replace(/\.js$/, '.ts'); + const childPath = fs.existsSync(compiledPath) ? compiledPath : sourcePath; + const args = childPath.endsWith('.ts') + ? ['--import', import.meta.resolve('tsx'), childPath] + : [childPath]; + args.push(dbPath, exportDir, String(options.dimensions)); + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, args, { + stdio: ['ignore', 'ignore', 'pipe'], + windowsHide: true, + }); + let stderr = ''; + let timedOut = false; + child.stderr.on('data', (chunk: Buffer) => { + if (stderr.length < 4096) stderr += chunk.toString().slice(0, 4096 - stderr.length); + }); + const timer = setTimeout(() => { + timedOut = true; + // A process boundary is safe to kill even while native code is executing. + child.kill('SIGKILL'); + }, options.timeoutMs ?? DEFAULT_EXTRACTION_TIMEOUT_MS); + child.once('error', (err) => { + clearTimeout(timer); + reject(err); + }); + child.once('close', (code, signal) => { + clearTimeout(timer); + if (code === 0 && !signal && !timedOut) resolve(); + else + reject( + new Error( + `staged embedding extraction failed${timedOut ? ' (timeout)' : ` (${signal ?? code})`}${stderr ? `: ${stderr.trim()}` : ''}`, + ), + ); + }); + }); +} diff --git a/gitnexus/src/core/git-staleness.ts b/gitnexus/src/core/git-staleness.ts index ece63a87c..e224b2319 100644 --- a/gitnexus/src/core/git-staleness.ts +++ b/gitnexus/src/core/git-staleness.ts @@ -17,8 +17,8 @@ export type { StalenessInfo, StalenessStatus } from './staleness-status.js'; const execFileAsync = promisify(execFile); /** - * Ceiling for one `git rev-list` staleness probe. Generous for the local - * history walk this is, and short enough that an unresponsive working tree + * Per-command ceiling for async `git rev-list` and both HEAD probes. + * Generous for local Git queries, and short enough that an unresponsive working tree * degrades to "not stale" quickly rather than holding a request open. */ const STALENESS_TIMEOUT_MS = 5_000; @@ -33,12 +33,23 @@ const behindHint = (n: number): string => const DIVERGED_HINT = "⚠️ Index is not at HEAD and the commit gap could not be counted — the recorded commit may no longer be in this clone's history. Run analyze tool to update."; +// `rev-list --count lastCommit..HEAD` answering 0 does NOT mean "HEAD is the +// indexed commit" — it means "HEAD has no commits lastCommit lacks", which is +// also true when HEAD is an *ancestor* of lastCommit (the working tree checked +// out an older commit than the one indexed, or switched to a line of history +// behind it). That read a rollback as `current` until this hint existed. +const REGRESSED_HINT = + '⚠️ Index is not at HEAD — the indexed commit is not reachable from the checked-out commit (the working tree may have checked out an older commit, or a different line of history). Run analyze tool to update.'; + const unknown = (): StalenessInfo => ({ isStale: false, commitsBehind: 0, status: 'unknown' }); -const fromCount = (commitsBehind: number): StalenessInfo => - commitsBehind > 0 - ? { isStale: true, commitsBehind, hint: behindHint(commitsBehind), status: 'behind' } - : { isStale: false, commitsBehind: 0, status: 'current' }; +// Called only once a positive HEAD-only count is in hand. +const behind = (commitsBehind: number): StalenessInfo => ({ + isStale: true, + commitsBehind, + hint: behindHint(commitsBehind), + status: 'behind', +}); /** * `rev-list` could not answer. Asking for HEAD alone needs no history walk and @@ -53,6 +64,26 @@ const fromHead = (head: string | null, lastCommit: string): StalenessInfo => { return { isStale: false, commitsBehind: 0, hint: DIVERGED_HINT, status: 'diverged' }; }; +/** + * `rev-list --left-right --count lastCommit...HEAD` measures both sides in + * one process, so a later HEAD change cannot mix two snapshots. The left + * count identifies a rollback even when the HEAD-only (right) count is 0. + * Positive right counts keep the existing `behind` behavior, including when + * both sides have commits (divergent branches or a re-shallowed clone). + */ +const fromCounts = (output: string): StalenessInfo => { + const counts = /^(\d+)\s+(\d+)$/.exec(output.trim()); + if (!counts) return unknown(); + const indexedOnly = Number(counts[1]); + const headOnly = Number(counts[2]); + if (!Number.isSafeInteger(indexedOnly) || !Number.isSafeInteger(headOnly)) return unknown(); + if (headOnly > 0) return behind(headOnly); + if (indexedOnly > 0) { + return { isStale: true, commitsBehind: 0, hint: REGRESSED_HINT, status: 'diverged' }; + } + return { isStale: false, commitsBehind: 0, status: 'current' }; +}; + const readHeadSync = (repoPath: string): string | null => { try { return ( @@ -61,6 +92,7 @@ const readHeadSync = (repoPath: string): string | null => { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true, + timeout: STALENESS_TIMEOUT_MS, }).trim() || null ); } catch { @@ -89,14 +121,18 @@ export function checkStaleness(repoPath: string, lastCommit: string): StalenessI // No recorded commit is not "at HEAD": there is nothing to measure against. if (!lastCommit) return unknown(); try { - const result = execFileSync('git', ['rev-list', '--count', `${lastCommit}..HEAD`], { - cwd: repoPath, - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - windowsHide: true, - }).trim(); + const result = execFileSync( + 'git', + ['rev-list', '--left-right', '--count', `${lastCommit}...HEAD`], + { + cwd: repoPath, + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true, + }, + ); - return fromCount(parseInt(result, 10) || 0); + return fromCounts(result); } catch { return fromHead(readHeadSync(repoPath), lastCommit); } @@ -115,21 +151,25 @@ export async function checkStalenessAsync( try { // Note: promisified execFile captures stdout/stderr by default (no stdio option needed, // unlike the sync variant which requires explicit stdio: ['pipe','pipe','pipe']). - const { stdout } = await execFileAsync('git', ['rev-list', '--count', `${lastCommit}..HEAD`], { - cwd: repoPath, - encoding: 'utf-8', - windowsHide: true, - // The catch below fails closed on every git ERROR, but a hang is not an - // error — it is silence, and without a bound this await never settles. - // A working tree on a disconnected network mount or behind a stuck lock - // does exactly that, and `/api/repos` fans this out once per registered - // repo, so one unreachable mount could hold the whole listing open - // (#3232 review). The timeout kills the child and rejects, and the catch - // below reports it as `unknown` — still the fail-closed `isStale: false`. - timeout: STALENESS_TIMEOUT_MS, - }); + const { stdout } = await execFileAsync( + 'git', + ['rev-list', '--left-right', '--count', `${lastCommit}...HEAD`], + { + cwd: repoPath, + encoding: 'utf-8', + windowsHide: true, + // The catch below fails closed on every git ERROR, but a hang is not an + // error — it is silence, and without a bound this await never settles. + // A working tree on a disconnected network mount or behind a stuck lock + // does exactly that, and `/api/repos` fans this out once per registered + // repo, so one unreachable mount could hold the whole listing open + // (#3232 review). The timeout kills the child and rejects, and the catch + // below reports it as `unknown` — still the fail-closed `isStale: false`. + timeout: STALENESS_TIMEOUT_MS, + }, + ); - return fromCount(parseInt(stdout.trim(), 10) || 0); + return fromCounts(stdout); } catch (err) { // A rev-list that timed out means the working tree is not answering. Asking // it again for HEAD would only double the bound #3232 put on a hung mount. diff --git a/gitnexus/src/core/group/extractors/http-patterns/go.ts b/gitnexus/src/core/group/extractors/http-patterns/go.ts index ebf87f0e1..6d29de5fe 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/go.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/go.ts @@ -1,27 +1,38 @@ +import type Parser from 'tree-sitter'; import Go from 'tree-sitter-go'; +import { goImportPackageName } from '../../../ingestion/languages/go/import-package-name.js'; +import { stringLiteral } from '../../../ingestion/route-extractors/go-shared.js'; import { compilePatterns, runCompiledPatterns, - unquoteLiteral, type LanguagePatterns, } from '../tree-sitter-scanner.js'; import type { HttpDetection, HttpLanguagePlugin } from './types.js'; /** * Go HTTP plugin. Handles: - * - gin / echo / chi framework routing — `r.GET("/path", handler)` + * - gin / echo framework routing — `r.GET("/path", handler)`, including + * prefixes from route groups bound in the same function (`r.Group("/api")`) * - net/http stdlib — `http.HandleFunc("/path", handler)` * - net/http consumer — `http.Get(...)`, `http.NewRequest("METHOD", ...)` * - resty consumer — `client.R().Delete("/path")` */ // ─── Provider: framework routing ────────────────────────────────────── -// Matches `\w+\.GET(...)` etc. (gin, echo, chi all share this shape). -// Captures the HTTP method (field name), path literal, and the handler — -// anchored to the LAST argument (`@handler .`) so a variadic middleware -// chain (`r.GET("/x", mw, handler)`, gin/echo/chi style) binds the real -// handler, not a middleware identifier (which would otherwise over-match -// and attach the route to the wrong symbol — see #2276 review). +// Matches `\w+\.GET(...)` etc. (gin and echo share this shape). +// Captures the receiver, the HTTP method (field name), and the path literal +// (either Go string form; stringLiteral decodes both, as ingestion does). The +// query does not anchor the path with `.`: tree-sitter counts comments as +// named children, so `GET(/* c */ "/p", h)` would fail the anchor. scan instead +// requires the path to be the first argument in code (comments skipped) and +// picks the handler out of the remaining code arguments. Which argument that is depends on the framework: +// gin is `GET(path, middleware..., handler)` (last), echo is +// `GET(path, handler, middleware...)` (first) — see readFrameworkImports and +// the per-call choice in scan below. +// The handler must be an identifier, an inline func literal, or a method +// value / package-qualified function (`h.ListUsers`, `handlers.ListUsers`); +// anything else there means the call cannot be attributed to a symbol, so it +// is dropped rather than guessed (variadic-middleware over-match, #2276). const FRAMEWORK_ROUTE_PATTERNS = compilePatterns({ name: 'go-framework-route', language: Go, @@ -31,16 +42,431 @@ const FRAMEWORK_ROUTE_PATTERNS = compilePatterns({ query: ` (call_expression function: (selector_expression + operand: (_) @receiver field: (field_identifier) @http_method (#match? @http_method "^(GET|POST|PUT|DELETE|PATCH)$")) arguments: (argument_list - (interpreted_string_literal) @path - [(identifier) (func_literal)] @handler - .)) + [(interpreted_string_literal) (raw_string_literal)] @path)) `, }, ], } satisfies LanguagePatterns>); +/** Named children that are code, not comments (tree-sitter names comments). */ +function codeChildren(node: Parser.SyntaxNode | null | undefined): Parser.SyntaxNode[] { + return node ? node.namedChildren.filter((c) => c.type !== 'comment') : []; +} + +/** Argument forms a route handler may take. */ +const HANDLER_ARG_TYPES: ReadonlySet = new Set([ + 'identifier', + 'func_literal', + 'selector_expression', +]); + +/** + * The file's framework import aliases: which local qualifiers resolve to + * echo and to gin. Matched on the import path rather than the local name, so + * an aliased import still counts; an unaliased import is keyed by its + * conventional package name (`goImportPackageName`). `_` and `.` imports bind + * no qualifier this file can route through. An empty set means the file + * proves nothing about that framework. Echo's verb calls take the handler as + * the FIRST argument after the path (`GET(path, handler, middleware...)`), + * gin's as the LAST (`GET(path, middleware..., handler)`) — `scan` picks the + * rule per call from these sets. + */ +function readFrameworkImports(root: Parser.SyntaxNode): { + echo: Set; + gin: Set; +} { + // Imports sit only at file scope; skip bodies. + const specs = root.namedChildren + .filter((node) => node.type === 'import_declaration') + .flatMap((decl) => decl.descendantsOfType('import_spec')); + const echo = new Set(); + const gin = new Set(); + for (const spec of specs) { + const importPath = stringLiteral(spec.childForFieldName('path')); + if (importPath === null) continue; + const local = spec.childForFieldName('name')?.text ?? goImportPackageName(importPath); + if (local === '_' || local === '.') continue; + if (importPath.includes('labstack/echo')) echo.add(local); + else if (importPath.includes('gin-gonic/gin')) gin.add(local); + } + return { echo, gin }; +} + +// ─── Route groups: `v1 := r.Group("/api/v1")` ───────────────────────── +// gin (`*gin.RouterGroup`) and echo (`*echo.Group`) routes registered on a +// group inherit every enclosing `Group(prefix)`. The prefix is recovered by +// walking the route's receiver back through its bindings, lexically, inside +// the enclosing function declaration only: a group handed to another +// function (`registerAdmin(v1)`) arrives as a parameter and contributes no +// prefix there, and a receiver bound to anything but a literal-prefix +// `Group(...)` call contributes none either — the route keeps its literal path. +// Statement-scoped bindings count too: an `if`/`switch` initializer, a `for` +// clause (including `range`), a type-switch guard, and declarations inside a +// switch case all scope over their statement the same way Go scopes them, so +// they shadow an outer group of the same name instead of being skipped. A +// select case's receive binding (`case g := <-ch:`) stops the walk entirely: +// what arrives from the channel is statically unknown, so the route keeps its +// literal path rather than inheriting an outer group. + +const MAX_GROUP_DEPTH = 32; + +function joinRoutePath(prefix: string, relative: string): string { + let joined = relative; + if (prefix && relative) { + joined = `${prefix.replace(/\/+$/, '')}/${relative.replace(/^\/+/, '')}`; + } else if (prefix) { + joined = prefix; + } + // Collapse duplicate slashes on the FINAL result — every return branch, not + // just the join — because ingestion's normalizeExtractedRoutePath collapses + // all "//" while the downstream contract-id normalizer does not: a path + // that keeps "//" would split into two contract ids across the strategies. + const collapsed = joined.replace(/\/+/g, '/'); + // Force a leading "/" for the same reason: ingestion's + // normalizeExtractedRoutePath always adds one, while normalizeHttpPath (the + // shared contract-id normalizer) does not — a literal "x" or a slashless + // Group("api") prefix would emit `...::x` here and `...::/x` there. Gin + // also refuses a registration path that does not start with "/". + return collapsed.startsWith('/') ? collapsed : `/${collapsed}`; +} + +/** `parent.Group("/p", mw...)` → its receiver and literal prefix; null otherwise. */ +function asGroupCall( + node: Parser.SyntaxNode, +): { parent: Parser.SyntaxNode; prefix: string } | null { + if (node.type !== 'call_expression') return null; + const fn = node.childForFieldName('function'); + if (fn?.type !== 'selector_expression' || fn.childForFieldName('field')?.text !== 'Group') { + return null; + } + const parent = fn.childForFieldName('operand'); + const first = codeChildren(node.childForFieldName('arguments'))[0]; + // Only a string literal carries a prefix, and it must decode to the text + // the runtime registers: stringLiteral applies Go unescaping (both `"…"` + // with escapes and raw `` `…` `` strings). A non-literal argument (a + // variable, concatenation) or an undecodable string contributes no prefix. + const prefix = first ? stringLiteral(first) : null; + if (!parent || prefix === null) return null; + return { parent, prefix }; +} + +/** The expression `name` is assigned by `stmt` (`:=`, `=`, or `var`), if any. */ +function boundValue(stmt: Parser.SyntaxNode, name: string): Parser.SyntaxNode | null | undefined { + const pick = ( + names: Parser.SyntaxNode[], + values: Parser.SyntaxNode | null, + ): Parser.SyntaxNode | null | undefined => { + const i = names.findIndex((n) => n.type === 'identifier' && n.text === name); + if (i < 0) return undefined; + return codeChildren(values)[i] ?? null; + }; + switch (stmt.type) { + case 'short_var_declaration': + case 'assignment_statement': + return pick(codeChildren(stmt.childForFieldName('left')), stmt.childForFieldName('right')); + case 'var_spec': + return pick(stmt.childrenForFieldName('name'), stmt.childForFieldName('value')); + case 'var_declaration': { + const specs = stmt.namedChildren.flatMap((c) => + c.type === 'var_spec_list' ? c.namedChildren : [c], + ); + for (const spec of specs) { + if (spec.type !== 'var_spec') continue; + const value = pick(spec.childrenForFieldName('name'), spec.childForFieldName('value')); + if (value !== undefined) return value; + } + return undefined; + } + default: + return undefined; + } +} + +/** + * A binding whose value cannot be established statically: an earlier + * statement writes the name inside a nested scope (`{ g = r.Group("/new") }`, + * a branch or loop body), so which value reaches the use depends on control + * flow. Callers decline the route instead of picking the older binding. + */ +const CONFLICT = Symbol('conflicting-binding'); +/** The name is a parameter (or method receiver): its declaration carries a static type. */ +interface ParamBinding { + param: Parser.SyntaxNode; +} +type Binding = Parser.SyntaxNode | null | undefined | typeof CONFLICT | ParamBinding; + +function isParamBinding(b: Binding): b is ParamBinding { + return typeof b === 'object' && b !== null && 'param' in b; +} + +/** The parameter_declaration of `fn` (parameters or method receiver) declaring `name`. */ +function paramDeclaring(fn: Parser.SyntaxNode, name: string): Parser.SyntaxNode | null { + for (const list of [fn.childForFieldName('parameters'), fn.childForFieldName('receiver')]) { + for (const decl of codeChildren(list)) { + if (decl.type !== 'parameter_declaration' && decl.type !== 'variadic_parameter_declaration') { + continue; + } + if (decl.childrenForFieldName('name').some((n) => n.text === name)) return decl; + } + } + return null; +} + +/** Whether `inner` lies within `outer`'s source span. */ +function within(outer: Parser.SyntaxNode | null, inner: Parser.SyntaxNode): boolean { + return !!outer && outer.startIndex <= inner.startIndex && inner.endIndex <= outer.endIndex; +} + +/** + * Whether `stmt` writes `name` with a plain assignment somewhere inside it + * (`g = …` in a nested block, branch, loop, or closure body). A nested `:=` + * declares a new variable and is not a write to the outer one. + */ +function writesNameInside(stmt: Parser.SyntaxNode, name: string): boolean { + return [stmt, ...stmt.descendantsOfType('assignment_statement')].some( + (a) => a.type === 'assignment_statement' && declaresName(a.childForFieldName('left'), name), + ); +} + +/** Whether an identifier or expression_list (e.g. a range left side) declares `name`. */ +function declaresName(node: Parser.SyntaxNode | null, name: string): boolean { + if (!node) return false; + if (node.type === 'identifier') return node.text === name; + return node.namedChildren.some((n) => n.type === 'identifier' && n.text === name); +} + +/** + * The value last bound to identifier `ident` before its use: the nearest + * binding site in the enclosing scopes, walking outward — preceding statements + * in blocks and switch/select cases (`expression_case`/`type_case`/ + * `communication_case`/`default_case` act as statement containers), then + * statement-scoped bindings (`if`/`switch` initializers, `for` clauses + * including `range`, type-switch guards), up to the enclosing function + * declaration. Returns null when the name is a parameter, is bound without a + * value, is received from a channel by a select case head (the received value + * is statically unknown, so the walk stops instead of escaping to an outer + * group), or is not bound in scope. + */ +function findBinding(ident: Parser.SyntaxNode): Parser.SyntaxNode | null | typeof CONFLICT { + const binding = lookupBinding(ident); + return isParamBinding(binding) ? null : (binding ?? null); +} + +/** + * findBinding's walk, keeping "declared without a traceable value" (null: a + * `func` literal parameter, `var x T`, a select receive) apart from "not + * declared before reaching the enclosing function declaration" (undefined). + * CONFLICT means a preceding statement writes the name in a nested scope, so + * the value at the use is control-flow dependent. A parameter or method + * receiver of the enclosing function returns its declaration (ParamBinding): + * no value, but a static type. + */ +function lookupBinding(ident: Parser.SyntaxNode): Binding { + const name = ident.text; + let child: Parser.SyntaxNode = ident; + for (let node = ident.parent; node; child = node, node = node.parent) { + if ( + node.type === 'function_declaration' || + node.type === 'method_declaration' || + node.type === 'func_literal' + ) { + const param = paramDeclaring(node, name); + if (param) return { param }; + if (node.type === 'func_literal') continue; + return undefined; + } + // Inside a grouped `var ( a = …; b = a.Group(…) )`, the specs before the + // one holding the use are already in scope; the current and later specs + // are not (`var g = g.Group(…)` reads the outer g). + if (node.type === 'var_spec_list') { + const specs = codeChildren(node); + const useIndex = specs.findIndex((s) => s.id === child.id); + for (const spec of specs.slice(0, useIndex).reverse()) { + const value = boundValue(spec, name); + if (value !== undefined) return value; + } + continue; + } + if ( + node.type === 'block' || + node.type === 'expression_case' || + node.type === 'type_case' || + node.type === 'communication_case' || + node.type === 'default_case' + ) { + // A select case head can rebind the name (`case g := <-ch:` or + // `case g = <-ch:`); the received value is statically unknown, so the + // walk must STOP with no traceable value — the same decline the + // ingestion-side route bindings record (value null) — instead of + // escaping to an outer group of the same name. + if (node.type === 'communication_case') { + for (const head of node.children) { + if ( + head.type === 'receive_statement' && + declaresName(head.childForFieldName('left'), name) + ) { + return null; + } + } + } + const stmts = codeChildren(node); + const useIndex = stmts.findIndex((s) => s.id === child.id); + for (const stmt of stmts.slice(0, useIndex).reverse()) { + const value = boundValue(stmt, name); + if (value !== undefined) return value; + // A write nested in an earlier statement (block, branch, loop) may or + // may not run before the use: the reaching value is unprovable. + if (writesNameInside(stmt, name)) return CONFLICT; + } + continue; + } + // Statement-scoped bindings enclose the use the same way Go scopes them. + if (node.type === 'if_statement' || node.type === 'expression_switch_statement') { + // A use inside the initializer itself (`if g := g.Group(…); …`) reads + // the OUTER binding: the new one only scopes over what follows it. + const init = node.childForFieldName('initializer'); + if (init && !within(init, ident)) { + const value = boundValue(init, name); + if (value !== undefined) return value; + } + continue; + } + if (node.type === 'for_statement') { + const clause = codeChildren(node)[0]; + // A write in the loop's post statement, condition, or body runs between + // iterations, so from the second pass on the body sees that value + // instead of the one it entered with (`for ; c; g = r.Group("/post")`): + // the prefix is control-flow dependent, so decline it. + if (within(node.childForFieldName('body'), ident)) { + const loopParts = [ + node.childForFieldName('body'), + clause?.type === 'for_clause' ? clause.childForFieldName('update') : null, + clause?.type === 'for_clause' ? clause.childForFieldName('condition') : null, + ]; + if (loopParts.some((part) => part && writesNameInside(part, name))) return CONFLICT; + } + if (clause?.type === 'for_clause') { + // Only the initializer binds before the body; an absent initializer + // (`for ; c; i++`) binds nothing. + const init = clause.childForFieldName('initializer'); + if (init && !within(init, ident)) { + const value = boundValue(init, name); + if (value !== undefined) return value; + } + } else if (clause?.type === 'range_clause') { + if ( + declaresName(clause.childForFieldName('left'), name) && + !within(clause.childForFieldName('right'), ident) + ) { + return clause.childForFieldName('right') ?? null; + } + } + continue; + } + if (node.type === 'type_switch_statement') { + // `switch g := x.(type)` — the guard list is the `alias` field, which + // only the `:=` form has (bare `switch x.(type)` parses with none), + // matching how the ingestion-side route-bindings read it. The switched + // value is the operand right after the guard list. + const guard = node.childForFieldName('alias'); + const switched = codeChildren(node)[1] ?? null; + if ( + guard?.type === 'expression_list' && + declaresName(guard, name) && + !within(switched, ident) + ) { + return switched; + } + continue; + } + } + return undefined; +} + +/** + * Whether a mixed-import file's route receiver traces back to echo's + * constructor — `e := echo.New()` / `echo.Default()` with `echo` resolving to + * one of the file's verified echo import aliases — either directly or through + * enclosing `Group(...)` calls (`users := api.Group(…)` ← `api := e.Group(…)` + * ← `echo.New()`), the normal shape of grouped routes (review #7, #10). + * A parameter or method receiver counts by its declared type instead: + * `e *echo.Echo` / `g *echo.Group` (with `echo` one of those aliases) proves + * echo; any other type — gin's, or one the file cannot tie to echo — does not. + * Unrelated packages' `New()`, a local that shadows the echo import name, and + * anything else return false so the caller keeps the conservative + * last-argument fallback instead of guessing. Returns null when the Group chain exceeds MAX_GROUP_DEPTH or a + * binding on it is control-flow dependent (CONFLICT): the framework is then + * unprovable either way, so the caller declines the route. + */ +function receiverBindsToEchoConstructor( + receiver: Parser.SyntaxNode, + echoAliases: ReadonlySet, + depth = 0, +): boolean | null { + if (depth > MAX_GROUP_DEPTH) return null; + // An identifier resolves through its binding; a chained `X.Group(…).Group(…)` + // operand is already a call and is inspected as-is. + const value = receiver.type === 'identifier' ? lookupBinding(receiver) : receiver; + if (value === CONFLICT) return null; + if (isParamBinding(value)) + return isEchoRouterType(value.param.childForFieldName('type'), echoAliases); + if (value?.type !== 'call_expression') return false; + const fn = value.childForFieldName('function'); + if (fn?.type !== 'selector_expression') return false; + const field = fn.childForFieldName('field')?.text; + const operand = fn.childForFieldName('operand'); + if (!operand) return false; + if (field === 'New' || field === 'Default') { + return operand.type === 'identifier' && echoAliases.has(operand.text) && !isLocalName(operand); + } + if (field === 'Group') { + return receiverBindsToEchoConstructor(operand, echoAliases, depth + 1); + } + return false; +} + +/** `*echo.Echo` / `echo.Echo` / `*echo.Group` with `echo` a verified echo import alias. */ +function isEchoRouterType( + type: Parser.SyntaxNode | null, + echoAliases: ReadonlySet, +): boolean { + const named = type?.type === 'pointer_type' ? codeChildren(type)[0] : type; + if (named?.type !== 'qualified_type') return false; + const pkg = named.childForFieldName('package')?.text; + const typeName = named.childForFieldName('name')?.text; + return !!pkg && echoAliases.has(pkg) && (typeName === 'Echo' || typeName === 'Group'); +} + +/** + * Whether `ident` names a local value rather than an imported package: a + * declaration in scope (with or without a value — `var echo Factory` shadows + * too) or a parameter/receiver of an enclosing function. Go lets either + * shadow a package qualifier (`func f(echo *Factory) { echo.New() }`). + */ +function isLocalName(ident: Parser.SyntaxNode): boolean { + // lookupBinding covers parameters and receivers too (ParamBinding). + return lookupBinding(ident) !== undefined; +} + +/** + * Joined `Group(...)` prefix of a route receiver; '' when it cannot be traced. + * Returns null when the chain exceeds MAX_GROUP_DEPTH, or when a binding on it + * is control-flow dependent (CONFLICT): a partial or stale prefix would emit + * a wrong path, so the caller declines the route instead. + */ +function groupPrefix(receiver: Parser.SyntaxNode, depth = 0): string | null { + if (depth > MAX_GROUP_DEPTH) return null; + const value = receiver.type === 'identifier' ? findBinding(receiver) : receiver; + if (value === CONFLICT) return null; + const group = value ? asGroupCall(value) : null; + if (!group) return ''; + const outer = groupPrefix(group.parent, depth + 1); + return outer === null ? null : joinRoutePath(outer, group.prefix); +} + // ─── Provider: net/http `http.HandleFunc("/p", handler)` ───────────── const HANDLE_FUNC_PATTERNS = compilePatterns({ name: 'go-handle-func', @@ -135,27 +561,87 @@ export const GO_HTTP_PLUGIN: HttpLanguagePlugin = { scan(tree) { const out: HttpDetection[] = []; - // Framework providers: r.GET/POST/... with handler identifier + // Framework providers: r.GET/POST/... on an engine or (nested) route group + const imports = readFrameworkImports(tree.rootNode); + const echoOnly = imports.echo.size > 0 && imports.gin.size === 0; + const mixed = imports.echo.size > 0 && imports.gin.size > 0; + // Handler names declared more than once in this file (`(h *A) List` and + // `(o *B) List`): the emitted name is field-only, so these must resolve + // only when unique in the file instead of taking the first same-named row. + const declaredNames = new Map(); + for (const decl of tree.rootNode.descendantsOfType([ + 'function_declaration', + 'method_declaration', + ])) { + const declName = decl.childForFieldName('name')?.text; + if (declName) declaredNames.set(declName, (declaredNames.get(declName) ?? 0) + 1); + } for (const match of runCompiledPatterns(FRAMEWORK_ROUTE_PATTERNS, tree)) { const methodNode = match.captures.http_method; const pathNode = match.captures.path; - const handlerNode = match.captures.handler; + const receiverNode = match.captures.receiver; if (!methodNode || !pathNode) continue; - const path = unquoteLiteral(pathNode.text); - if (path === null) continue; + const literalPath = stringLiteral(pathNode); + if (literalPath === null) continue; + const argList = pathNode.parent; + if (argList?.type !== 'argument_list') continue; + const args = codeChildren(argList); + if (args[0]?.id !== pathNode.id) continue; + // The path is the first code argument, so everything after it is a handler or + // middleware candidate: echo's verb calls take the FIRST of those, gin's + // the LAST (see FRAMEWORK_ROUTE_PATTERNS / readFrameworkImports). The + // rule is chosen per call: an echo-only file is unambiguous; a + // mixed-import file takes the first argument only when the receiver + // provably traces to echo's constructor (directly or through enclosing + // Group() calls) — everything else keeps the last-argument anchor, + // gin's order and the safer default when the file proves nothing. + const rest = args.slice(1); + if (rest.length === 0) continue; + const echoOrder = mixed + ? receiverNode + ? receiverBindsToEchoConstructor(receiverNode, imports.echo) + : false + : echoOnly; + // A Group chain deeper than MAX_GROUP_DEPTH proves neither the full + // prefix nor the framework order: decline rather than emit a guess. + if (echoOrder === null) continue; + const prefix = receiverNode ? groupPrefix(receiverNode) : ''; + if (prefix === null) continue; + const handlerNode = echoOrder ? rest[0] : rest[rest.length - 1]; + if (!HANDLER_ARG_TYPES.has(handlerNode.type)) continue; + const path = receiverNode ? joinRoutePath(prefix, literalPath) : literalPath; // An inline `func(){…}` handler has no name → emit `name: null` and a // `line` so it resolves to its containing/closure symbol by line-span - // containment (like a consumer). A named identifier handler keeps its - // name and resolves by name; `line` is harmless there. + // containment (like a consumer). A named handler keeps its name and + // resolves by name; `line` is harmless there. For a method value or a + // package-qualified function (`h.List`, `pkg.List`) that name is the + // field, and the operand (usually a local variable, not the receiver + // type) does not prove where `List` is declared — so the detection is + // marked qualifiedHandler: resolve only to a repo-wide unique `List`, + // never to a same-named local method that merely shares the name. const isInlineHandler = handlerNode?.type === 'func_literal'; + const isQualified = handlerNode?.type === 'selector_expression'; + const handlerName = isQualified + ? (handlerNode.childForFieldName('field')?.text ?? null) + : (handlerNode?.text ?? null); out.push({ role: 'provider', framework: 'go-framework', method: methodNode.text.toUpperCase(), path, - name: isInlineHandler ? null : (handlerNode?.text ?? null), + name: isInlineHandler ? null : handlerName, line: (handlerNode ?? pathNode).startPosition.row + 1, confidence: 0.8, + ...(isQualified ? { qualifiedHandler: true } : {}), + // A bare name declared more than once in this file (a function and a + // method of the same name) resolves only when the file holds exactly + // one match, rather than binding to whichever row the graph lists first. + ...(!isQualified && + !isInlineHandler && + handlerName && + (declaredNames.get(handlerName) ?? 0) > 1 + ? { strictHandlerResolution: true } + : {}), }); } @@ -164,7 +650,7 @@ export const GO_HTTP_PLUGIN: HttpLanguagePlugin = { const pathNode = match.captures.path; const handlerNode = match.captures.handler; if (!pathNode) continue; - const path = unquoteLiteral(pathNode.text); + const path = stringLiteral(pathNode); if (path === null) continue; // Inline `func(){…}` handler → resolve by containment (see go-framework // note above); a named handler resolves by name. @@ -187,7 +673,7 @@ export const GO_HTTP_PLUGIN: HttpLanguagePlugin = { if (!fnNode || !pathNode) continue; const httpMethod = HTTP_CLIENT_METHOD_TO_HTTP[fnNode.text]; if (!httpMethod) continue; - const path = unquoteLiteral(pathNode.text); + const path = stringLiteral(pathNode); if (path === null) continue; out.push({ role: 'consumer', @@ -205,8 +691,8 @@ export const GO_HTTP_PLUGIN: HttpLanguagePlugin = { const methodNode = match.captures.http_method; const pathNode = match.captures.path; if (!methodNode || !pathNode) continue; - const method = unquoteLiteral(methodNode.text); - const path = unquoteLiteral(pathNode.text); + const method = stringLiteral(methodNode); + const path = stringLiteral(pathNode); if (method === null || path === null) continue; out.push({ role: 'consumer', @@ -224,7 +710,7 @@ export const GO_HTTP_PLUGIN: HttpLanguagePlugin = { const methodNode = match.captures.http_method; const pathNode = match.captures.path; if (!methodNode || !pathNode) continue; - const path = unquoteLiteral(pathNode.text); + const path = stringLiteral(pathNode); if (path === null) continue; out.push({ role: 'consumer', diff --git a/gitnexus/src/core/group/extractors/http-patterns/types.ts b/gitnexus/src/core/group/extractors/http-patterns/types.ts index 38c197704..bbd3fe1a2 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/types.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/types.ts @@ -58,6 +58,14 @@ export interface HttpDetection { handlerImport?: { name: string; module: string }; /** Resolve only from the registration file or exact import target; never guess repo-wide. */ strictHandlerResolution?: boolean; + /** + * The handler was designated through a qualifier the plugin cannot tie to a + * declaration (`recv.name`, `pkg.name`): `name` alone does not prove the + * handler lives in the registration file. Skip the file-scoped name lookup + * — a same-named but unrelated local symbol would win it — and accept only + * a repo-wide unique match, else keep the file-level fallback. + */ + qualifiedHandler?: boolean; /** * The plugin saw a provider handler designator but could not prove its owner. * Prevents the orchestrator from treating it as an anonymous inline handler diff --git a/gitnexus/src/core/group/extractors/http-route-extractor.ts b/gitnexus/src/core/group/extractors/http-route-extractor.ts index d116bc046..707ccdff7 100644 --- a/gitnexus/src/core/group/extractors/http-route-extractor.ts +++ b/gitnexus/src/core/group/extractors/http-route-extractor.ts @@ -629,11 +629,17 @@ export class HttpRouteExtractor implements ContractExtractor { if (d.strictHandlerResolution) return null; return resolveSymbolByNameUnique(d.handlerImport.name); } - const byName = d.strictHandlerResolution - ? resolveFileSymbolByNameUnique(syms, d.name) - : resolveSymbolByName(syms, d.name); - if (byName) return byName; - if (d.strictHandlerResolution) return null; + // A qualified designator (`recv.name`) does not prove the handler is + // declared in this file, so the file-first rung could bind an + // unrelated same-named local symbol: go straight to the unique + // repo-wide match. + if (!d.qualifiedHandler) { + const byName = d.strictHandlerResolution + ? resolveFileSymbolByNameUnique(syms, d.name) + : resolveSymbolByName(syms, d.name); + if (byName) return byName; + if (d.strictHandlerResolution) return null; + } const byGlobal = await resolveSymbolByNameUnique(d.name); if (byGlobal) return byGlobal; // A NAMED handler we could not resolve by name (neither file-scoped nor diff --git a/gitnexus/src/core/incremental/write-reconciliation.ts b/gitnexus/src/core/incremental/write-reconciliation.ts new file mode 100644 index 000000000..bbd717e0f --- /dev/null +++ b/gitnexus/src/core/incremental/write-reconciliation.ts @@ -0,0 +1,107 @@ +import type { GraphNode } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../graph/types.js'; +import { NODE_TABLES, type NodeTableName } from '../lbug/schema.js'; +import { sanitizeUTF8 } from '../lbug/csv-generator.js'; + +// These layers need a different oracle: folders can outlive their last file, +// derived nodes may be preserved without recomputation, and PDG can be streamed. +const EXCLUDED = new Set(['Folder', 'Community', 'Process', 'BasicBlock']); +const WITHOUT_RANGE = new Set(['File', 'Route', 'Tool']); + +interface IdentityRow { + id: string; + name: string | null; + filePath: string | null; + startLine?: number | null; + endLine?: number | null; +} + +const storedString = (value: string | undefined): string => sanitizeUTF8(value || ''); + +const storedIdentityField = (node: GraphNode, field: string): unknown => { + const value = node.properties[field]; + if (field === 'name' || field === 'filePath') { + return storedString(value as string | undefined); + } + if (node.label === 'Destination') { + return typeof value === 'number' && Number.isFinite(value) ? value : null; + } + return value ?? -1; +}; + +/** + * Certify the identity fields used by context/impact/detect_changes, including + * retained rows. A write-set-only probe misses corruption of an unchanged row. + * Scan one label at a time, without path/range predicates: a corrupt field must + * not be able to hide its own row. No multi-label scans (#3139), N+1 ID lookups, + * source-content copies, or whole-graph row materialization. + * + * This certifies node identity, not relationships or native storage internals. + * Call after COPY/checkpoint and after FTS/embedding/checkpoint, before metadata. + */ +export async function reconcileGraphNodeIdentities( + graph: KnowledgeGraph, + query: (cypher: string) => Promise, + phase: string, +): Promise<{ nodes: number; tables: number }> { + const expected = new Map>(); + for (const table of NODE_TABLES) { + if (!EXCLUDED.has(table)) expected.set(table, new Map()); + } + for (const node of graph.iterNodes()) { + const table = expected.get(node.label as NodeTableName); + if (!table) continue; + const id = storedString(node.id); + if (!id || table.has(id)) { + throw new Error(`Graph identity reconciliation (${phase}): invalid or duplicate ID ${id}`); + } + table.set(id, node); + } + + let nodes = 0; + for (const [table, remaining] of expected) { + const fields = WITHOUT_RANGE.has(table) + ? ['id', 'name', 'filePath'] + : ['id', 'name', 'filePath', 'startLine', 'endLine']; + const identityFields = fields.slice(1); + const fail: (detail: string, cause?: unknown) => never = (detail, cause) => { + throw new Error( + `Graph identity reconciliation failed (${phase}, ${table}): ${detail}. ` + + 'Freshness was not advanced; run `gitnexus analyze --force` to rebuild the graph.', + { cause }, + ); + }; + let rows: IdentityRow[]; + try { + rows = await query( + `MATCH (n:\`${table}\`) RETURN ${fields.map((f) => `n.${f} AS ${f}`).join(', ')}`, + ); + } catch (error) { + fail( + `could not read identity fields: ${error instanceof Error ? error.message : String(error)}`, + error, + ); + } + for (const row of rows) { + const node = remaining.get(row.id); + if (!node) fail(`unexpected or duplicate ID ${JSON.stringify(row.id)}`); + for (const field of identityFields) { + const wanted = storedIdentityField(node, field); + const actual = field === 'name' || field === 'filePath' ? (row[field] ?? '') : row[field]; + if (actual !== wanted) { + fail( + `${row.id}.${field}: expected ${JSON.stringify(wanted)}, read ${JSON.stringify(actual)}`, + ); + } + } + remaining.delete(row.id); + nodes++; + } + if (remaining.size) { + fail( + `${remaining.size} missing ID(s), including ${JSON.stringify(remaining.keys().next().value)}`, + ); + } + } + return { nodes, tables: expected.size }; +} diff --git a/gitnexus/src/core/ingestion/cobol-processor.ts b/gitnexus/src/core/ingestion/cobol-processor.ts index 3fac5072d..6fae16805 100644 --- a/gitnexus/src/core/ingestion/cobol-processor.ts +++ b/gitnexus/src/core/ingestion/cobol-processor.ts @@ -26,6 +26,8 @@ import { import { expandCopies } from './cobol/cobol-copy-expander.js'; import { processJclFiles } from './cobol/jcl-processor.js'; import { resolveCobolCopyTarget } from './languages/cobol/copy-target.js'; +import { COBOL_EXTENSIONS, JCL_EXTENSIONS } from './cobol/file-types.js'; +export { isCobolFile, isJclFile } from './cobol/file-types.js'; import { logger } from '../logger.js'; @@ -33,10 +35,6 @@ import { logger } from '../logger.js'; // File detection // --------------------------------------------------------------------------- -const COBOL_EXTENSIONS = new Set(['.cob', '.cbl', '.cobol', '.cpy', '.copybook']); - -const JCL_EXTENSIONS = new Set(['.jcl', '.job', '.proc']); - const COPYBOOK_EXTENSIONS = new Set(['.cpy', '.copybook']); interface CobolFile { @@ -67,16 +65,6 @@ export interface CobolProcessResult { arithmeticOps: number; } -/** Returns true if the file is a COBOL or copybook file. */ -export function isCobolFile(filePath: string): boolean { - return COBOL_EXTENSIONS.has(path.extname(filePath).toLowerCase()); -} - -/** Returns true if the file is a JCL file. */ -export function isJclFile(filePath: string): boolean { - return JCL_EXTENSIONS.has(path.extname(filePath).toLowerCase()); -} - /** Returns true if the file is a COBOL copybook. */ function isCopybook(filePath: string): boolean { return COPYBOOK_EXTENSIONS.has(path.extname(filePath).toLowerCase()); diff --git a/gitnexus/src/core/ingestion/cobol/file-types.ts b/gitnexus/src/core/ingestion/cobol/file-types.ts new file mode 100644 index 000000000..a41d622ad --- /dev/null +++ b/gitnexus/src/core/ingestion/cobol/file-types.ts @@ -0,0 +1,15 @@ +import path from 'node:path'; + +// Keep detection shared without loading the analyze-only language providers. +export const COBOL_EXTENSIONS = new Set(['.cob', '.cbl', '.cobol', '.cpy', '.copybook']); +export const JCL_EXTENSIONS = new Set(['.jcl', '.job', '.proc']); + +/** Includes COBOL programs and copybooks accepted by ingestion. */ +export function isCobolFile(filePath: string): boolean { + return COBOL_EXTENSIONS.has(path.extname(filePath).toLowerCase()); +} + +/** Includes job and procedure aliases accepted by JCL ingestion. */ +export function isJclFile(filePath: string): boolean { + return JCL_EXTENSIONS.has(path.extname(filePath).toLowerCase()); +} diff --git a/gitnexus/src/core/ingestion/community-processor.ts b/gitnexus/src/core/ingestion/community-processor.ts index 77b66d73b..233326e03 100644 --- a/gitnexus/src/core/ingestion/community-processor.ts +++ b/gitnexus/src/core/ingestion/community-processor.ts @@ -176,7 +176,10 @@ export interface CommunityMembership { export interface CommunityDetectionResult { communities: CommunityNode[]; + /** Assignments to retained communities, safe to emit as MEMBER_OF edges. */ memberships: CommunityMembership[]; + /** All Leiden assignments, including filtered singletons. Optional for legacy producers. */ + rawMemberships?: CommunityMembership[]; stats: { totalCommunities: number; modularity: number; @@ -235,6 +238,7 @@ export const processCommunities = async ( return { communities: [], memberships: [], + rawMemberships: [], stats: { totalCommunities: 0, modularity: 0, @@ -267,13 +271,16 @@ export const processCommunities = async ( onProgress?.('Creating membership edges...', 80); - // Step 4: Create membership mappings + // Step 4: Preserve all assignments for skill generation, but only emit + // memberships for retained communities with a corresponding graph node. + const retainedCommunityIds = new Set(communityNodes.map((community) => community.id)); const memberships: CommunityMembership[] = []; + const rawMemberships: CommunityMembership[] = []; Object.entries(details.communities).forEach(([nodeId, communityNum]) => { - memberships.push({ - nodeId, - communityId: `comm_${communityNum}`, - }); + const communityId = `comm_${communityNum}`; + const membership = { nodeId, communityId }; + rawMemberships.push(membership); + if (retainedCommunityIds.has(communityId)) memberships.push(membership); }); onProgress?.('Community detection complete!', 100); @@ -281,6 +288,7 @@ export const processCommunities = async ( return { communities: communityNodes, memberships, + rawMemberships, stats: { totalCommunities: details.count, modularity: details.modularity, diff --git a/gitnexus/src/core/ingestion/language-provider.ts b/gitnexus/src/core/ingestion/language-provider.ts index 90122aacc..718d8693d 100644 --- a/gitnexus/src/core/ingestion/language-provider.ts +++ b/gitnexus/src/core/ingestion/language-provider.ts @@ -46,7 +46,7 @@ import type { RepoConstants, } from './route-extractors/constant-resolver.js'; import type Parser from 'tree-sitter'; -import type { ExtractedDecoratorRoute } from './workers/parse-worker.js'; +import type { ExtractedDecoratorRoute, ExtractedToolDef } from './workers/parse-worker.js'; import type { SemanticModel } from './model/semantic-model.js'; /** What a provider's {@link LanguageProviderConfig.resolveRouteHandler} can see. */ @@ -546,6 +546,15 @@ interface LanguageProviderConfig { */ readonly extractTextRoutes?: (filePath: string, content: string) => ExtractedRoute[]; + /** Extract tool registrations after captures, using only emitted callable identities. + * The map keys are declaration-name AST node IDs, local to this parsed tree. */ + readonly extractToolDefinitions?: ( + tree: Parser.Tree, + filePath: string, + lineOffset: number, + callableBindings: ReadonlyMap, + ) => ExtractedToolDef[]; + /** * Extract routes that a parsed file declares in its own AST. * diff --git a/gitnexus/src/core/ingestion/languages/swift/callable-visibility.ts b/gitnexus/src/core/ingestion/languages/swift/callable-visibility.ts new file mode 100644 index 000000000..f89b2b639 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/swift/callable-visibility.ts @@ -0,0 +1,78 @@ +import type { ParsedFile, ScopeId, SymbolDefinition, TypeRef } from 'gitnexus-shared'; +import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import { findClassBindingInScope } from '../../scope-resolution/scope/walkers.js'; + +export function swiftIsCallableVisibleFromCaller(ctx: { + readonly candidate: SymbolDefinition; + readonly callerParsed?: ParsedFile; + readonly callArity?: number; + readonly callerScope?: ScopeId; + readonly scopes?: ScopeResolutionIndexes; +}): boolean { + const indexes = ctx.scopes; + if (ctx.callerScope === undefined || indexes === undefined || ctx.callArity !== 0) return true; + + const name = ctx.candidate.qualifiedName?.split('.').at(-1); + if (name === undefined) return true; + + let scopeId: ScopeId | null = ctx.callerScope; + let selfType: TypeRef | undefined; + while (scopeId !== null) { + const scope = indexes.scopeTree.getScope(scopeId); + if (scope === undefined) break; + // A local function selected inside the caller wins before member lookup. + if ( + scope.kind !== 'Class' && + scope.ownedDefs.some((def) => def.nodeId === ctx.candidate.nodeId) + ) + return true; + selfType ??= scope.typeBindings.get('self'); + if (scope.kind === 'Class') { + const classDef = + scope.ownedDefs.find((def) => def.type === 'Class') ?? + (selfType === undefined + ? undefined + : findClassBindingInScope(ctx.callerScope, selfType.rawName, indexes, undefined, { + uniqueQualifiedNameFallback: false, + })); + const propertyOnOwner = (ownerId: string, ownerName: string): boolean => + indexes.qualifiedNames.get(`${ownerName}.${name}`).some((defId) => { + const def = indexes.defs.get(defId); + return def?.type === 'Property' && def.ownerId === ownerId; + }); + const ownProperty = + scope.ownedDefs.some( + (def) => def.type === 'Property' && def.qualifiedName?.split('.').at(-1) === name, + ) || + (classDef !== undefined && + classDef.qualifiedName !== undefined && + propertyOnOwner(classDef.nodeId, classDef.qualifiedName)); + const inheritedProperty = + classDef !== undefined && + indexes.methodDispatch.mroFor(classDef.nodeId).some((ownerId) => { + const ownerName = indexes.defs.get(ownerId)?.qualifiedName; + return ownerName !== undefined && propertyOnOwner(ownerId, ownerName); + }); + if (!ownProperty && !inheritedProperty) return true; + + // Swift's nested function is owned by its own Function scope. A + // same-file sibling must not count as a nearer lexical binding. + const declarationScope = ctx.callerParsed?.scopes.find((candidateScope) => + candidateScope.ownedDefs.some((def) => def.nodeId === ctx.candidate.nodeId), + ); + if ( + declarationScope !== undefined && + declarationScope.kind === 'Function' && + (declarationScope.id === ctx.callerScope || + indexes.scopeTree.getAncestors(declarationScope.id).includes(ctx.callerScope)) + ) + return true; + + // Scope defs do not carry Swift access modifiers. A method positively + // selected on this type must not be vetoed by an uncertain ancestor. + return classDef !== undefined && ctx.candidate.ownerId === classDef.nodeId; + } + scopeId = scope.parent; + } + return true; +} diff --git a/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts index 0822b8cf8..7fe075abf 100644 --- a/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts @@ -70,6 +70,7 @@ import { import { stripSwiftTypePreservingDecoration } from './interpret.js'; import { groupSwiftFilesByModule } from './target-grouping.js'; import { swiftIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; +import { swiftIsCallableVisibleFromCaller } from './callable-visibility.js'; const ZERO_RANGE = { startLine: 0, startCol: 0, endLine: 0, endCol: 0 } as const; @@ -155,6 +156,7 @@ const swiftScopeResolver: ScopeResolver = { // no-`new` constructor + cross-file free-call shape). allowGlobalFreeCallFallback: true, isGlobalNameFallbackPlausible: swiftIsGlobalNameFallbackPlausible, + isCallableVisibleFromCaller: swiftIsCallableVisibleFromCaller, // Swift's call graph models `Type(...)` as a reference to the type // itself, not its `init` — both the legacy DAG and this test suite link diff --git a/gitnexus/src/core/ingestion/languages/typescript.ts b/gitnexus/src/core/ingestion/languages/typescript.ts index 5251910cb..c3509ea5e 100644 --- a/gitnexus/src/core/ingestion/languages/typescript.ts +++ b/gitnexus/src/core/ingestion/languages/typescript.ts @@ -130,6 +130,7 @@ import { extractDataRouteTableRoutes } from '../route-extractors/data-route-tabl import { extractNestRoutes } from '../route-extractors/nest.js'; import { extractTrpcRoutes, shouldScanForTrpcRoutes } from '../route-extractors/trpc.js'; import { extractConvexEndpointProperties } from './typescript/convex-endpoint-metadata.js'; +import { extractToolDefinitions } from './typescript/tool-definitions.js'; const extractJsTsRoutes = (...args: Parameters) => [ ...extractDispatchGuardRoutes(...args), @@ -494,6 +495,7 @@ export const typescriptProvider = defineLanguage({ // Content-based (not AST): tRPC procedure routers are scanned from source text. // Path-gate lives here (language provider), not in the shared parse worker. extractTextRoutes: extractJsTsTextRoutes, + extractToolDefinitions, }); export const javascriptProvider = defineLanguage({ @@ -577,4 +579,5 @@ export const javascriptProvider = defineLanguage({ extractDecoratorRoutes: extractJsTsRoutes, // Content-based (not AST): tRPC procedure routers are scanned from source text. extractTextRoutes: extractJsTsTextRoutes, + extractToolDefinitions, }); diff --git a/gitnexus/src/core/ingestion/languages/typescript/tool-definitions.ts b/gitnexus/src/core/ingestion/languages/typescript/tool-definitions.ts new file mode 100644 index 000000000..307998c56 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/typescript/tool-definitions.ts @@ -0,0 +1,365 @@ +import type Parser from 'tree-sitter'; +import type { SyntaxNode } from 'tree-sitter'; +import type { ExtractedToolDef } from '../../workers/parse-worker.js'; +import { plainString, propertyName } from '../../route-extractors/data-route-table.js'; + +interface Scope { + parent?: Scope; + functionScope: boolean; + bindings: Map; +} + +interface Binding { + name: SyntaxNode; + scope: Scope; + kind: 'unknown' | 'sdk' | 'sdk-namespace' | 'variable' | 'parameter' | 'function'; + value?: SyntaxNode; + type?: SyntaxNode; + typeOnly?: boolean; + immutable?: boolean; + invalid?: boolean; +} + +const FUNCTIONS = new Set([ + 'function_declaration', + 'generator_function_declaration', + 'function_expression', + 'generator_function', + 'arrow_function', + 'method_definition', +]); +const BLOCKS = new Set([ + 'statement_block', + 'for_statement', + 'for_in_statement', + 'switch_body', + 'catch_clause', + 'class_body', +]); +const SDK_MODULES = new Set([ + '@modelcontextprotocol/sdk/server/mcp.js', + '@modelcontextprotocol/sdk/server/mcp', +]); + +function lookup(scope: Scope, name: string): Binding | undefined { + for (let current: Scope | undefined = scope; current; current = current.parent) { + const binding = current.bindings.get(name); + if (binding) return binding; + } +} + +/** Only binding/assignment patterns: never descend into keys, types or defaults. + * Member assignment targets are reported separately from binding names. */ +function patternNames(pattern: SyntaxNode, onMember?: (member: SyntaxNode) => void): SyntaxNode[] { + const names: SyntaxNode[] = []; + const pending = [pattern]; + while (pending.length) { + const node = pending.pop()!; + if (node.type === 'identifier' || node.type === 'shorthand_property_identifier_pattern') { + names.push(node); + } else if (node.type === 'member_expression' || node.type === 'subscript_expression') { + onMember?.(node); + } else if (node.type === 'pair_pattern') { + const value = node.childForFieldName('value'); + if (value) pending.push(value); + } else if (node.type === 'assignment_pattern' || node.type === 'object_assignment_pattern') { + const left = node.childForFieldName('left'); + if (left) pending.push(left); + } else if ( + node.type === 'array_pattern' || + node.type === 'object_pattern' || + node.type === 'rest_pattern' + ) { + pending.push(...node.namedChildren); + } + } + return names; +} + +function declare(scope: Scope, name: SyntaxNode, details: Partial = {}): void { + const previous = scope.bindings.get(name.text); + if (previous) { + previous.invalid = true; + } else { + scope.bindings.set(name.text, { name, scope, kind: 'unknown', ...details }); + } +} + +function variableScope(scope: Scope): Scope { + while (!scope.functionScope && scope.parent) scope = scope.parent; + return scope; +} + +function collectBindings(root: SyntaxNode) { + const moduleScope: Scope = { functionScope: true, bindings: new Map() }; + const scopes = new Map(); + const calls: SyntaxNode[] = []; + const writes: SyntaxNode[] = []; + const pending = [{ node: root, scope: moduleScope }]; + while (pending.length) { + const entry = pending.pop()!; + const node = entry.node; + let scope = entry.scope; + const isFunction = FUNCTIONS.has(node.type); + const name = node.childForFieldName('name'); + if (node.type === 'function_declaration' || node.type === 'generator_function_declaration') { + if (name) declare(scope, name, { kind: 'function', value: node, immutable: true }); + } else if ( + [ + 'class_declaration', + 'interface_declaration', + 'type_alias_declaration', + 'enum_declaration', + ].includes(node.type) + ) { + if (name) declare(scope, name); + } + if ( + isFunction || + BLOCKS.has(node.type) || + node.type === 'class' || + node.type === 'class_declaration' + ) { + scope = { parent: scope, functionScope: isFunction, bindings: new Map() }; + } + scopes.set(node.id, scope); + + if (node.type === 'class' && name) declare(scope, name); + + if (isFunction) { + if (name && (node.type === 'function_expression' || node.type === 'generator_function')) { + declare(scope, name, { kind: 'function', value: node, immutable: true }); + } + const parameters = node.childForFieldName('parameters'); + const single = node.childForFieldName('parameter'); + for (const parameter of parameters?.namedChildren ?? (single ? [single] : [])) { + const pattern = parameter.childForFieldName('pattern') ?? parameter; + const type = parameter.childForFieldName('type')?.namedChildren[0]; + for (const bindingName of patternNames(pattern)) { + declare(scope, bindingName, { + kind: 'parameter', + ...(pattern.type === 'identifier' && type ? { type } : {}), + }); + } + } + } else if (node.type === 'import_statement') { + const source = node.childForFieldName('source'); + const sdk = source !== null && SDK_MODULES.has(plainString(source) ?? ''); + const typeOnly = node.children.some((child) => child.type === 'type'); + const clause = node.namedChildren.find((child) => child.type === 'import_clause'); + for (const child of clause?.namedChildren ?? []) { + if (child.type === 'identifier') declare(scope, child); + else if (child.type === 'namespace_import') { + const local = child.namedChildren[0]; + if (local) declare(scope, local, { kind: sdk ? 'sdk-namespace' : 'unknown', typeOnly }); + } else if (child.type === 'named_imports') { + for (const specifier of child.namedChildren) { + const imported = specifier.childForFieldName('name'); + const local = specifier.childForFieldName('alias') ?? imported; + if (local) + declare(scope, local, { + kind: sdk && imported?.text === 'McpServer' ? 'sdk' : 'unknown', + typeOnly: typeOnly || specifier.children.some((part) => part.type === 'type'), + }); + } + } + } + } else if (node.type === 'variable_declarator') { + if (name) { + const target = node.parent?.type === 'variable_declaration' ? variableScope(scope) : scope; + for (const bindingName of patternNames(name)) { + declare(target, bindingName, { + kind: 'variable', + immutable: node.parent?.childForFieldName('kind')?.type === 'const', + ...(name.type === 'identifier' + ? { value: node.childForFieldName('value') ?? undefined } + : {}), + }); + } + } + } else if (node.type === 'catch_clause') { + const parameter = node.childForFieldName('parameter'); + if (parameter) for (const bindingName of patternNames(parameter)) declare(scope, bindingName); + } else if (node.type === 'for_in_statement') { + const left = node.childForFieldName('left'); + const kind = node.childForFieldName('kind'); + if (left && kind) { + const target = kind.type === 'var' ? variableScope(scope) : scope; + for (const bindingName of patternNames(left)) declare(target, bindingName); + } else if (left) writes.push(left); + } else if (node.type === 'type_parameter') { + if (name) declare(scope, name); + } + if (node.type === 'call_expression') calls.push(node); + if (node.type === 'assignment_expression' || node.type === 'augmented_assignment_expression') { + const left = node.childForFieldName('left'); + if (left) writes.push(left); + } else if ( + node.type === 'update_expression' || + (node.type === 'unary_expression' && node.children.some((child) => child.type === 'delete')) + ) { + const argument = node.childForFieldName('argument'); + if (argument) writes.push(argument); + } + for (let index = node.namedChildCount - 1; index >= 0; index--) { + pending.push({ node: node.namedChild(index)!, scope }); + } + } + // Resolve writes after declarations so later declarations also shadow outer names. + while (writes.length) { + let target = writes.pop()!; + const scope = scopes.get(target.id)!; + const members: Array = []; + while (target.type === 'member_expression' || target.type === 'subscript_expression') { + const object = target.childForFieldName('object'); + if (!object) break; + const property = target.childForFieldName('property'); + const index = target.childForFieldName('index'); + members.push(property ? propertyName(property) : index ? plainString(index) : null); + target = object; + } + // Nested member targets must pass the same guard as direct property writes. + for (const name of patternNames(target, (member) => writes.push(member))) { + const binding = lookup(scope, name.text); + // Lifecycle callbacks and other known properties do not replace the receiver + // or its registration methods. Unknown keys and constructor mutations remain unsafe. + if ( + binding?.kind !== 'sdk' && + binding?.kind !== 'sdk-namespace' && + members.length > 0 && + members.every((member) => member !== null) && + !['tool', 'registerTool', '__proto__'].includes(members[members.length - 1]!) + ) + continue; + if (binding) binding.invalid = true; + } + } + return { scopes, calls }; +} + +function sdkBinding(node: SyntaxNode, scope: Scope, forType = false): boolean { + let kind: Binding['kind'] = 'sdk'; + if (node.type === (forType ? 'nested_type_identifier' : 'member_expression')) { + const namespace = node.childForFieldName(forType ? 'module' : 'object'); + const member = node.childForFieldName(forType ? 'name' : 'property'); + if (namespace?.type !== 'identifier' || member?.text !== 'McpServer') return false; + node = namespace; + kind = 'sdk-namespace'; + } + if (node.type !== 'identifier' && node.type !== 'type_identifier') return false; + const binding = lookup(scope, node.text); + return binding?.kind === kind && !binding.invalid && (forType || !binding.typeOnly); +} + +function sdkReceiver(node: SyntaxNode, scope: Scope, scopes: ReadonlyMap): boolean { + if (node.type !== 'identifier') return false; + const binding = lookup(scope, node.text); + if (!binding || binding.invalid) return false; + if (binding.kind === 'parameter' && binding.type) { + return sdkBinding(binding.type, binding.scope, true); + } + const value = binding.value; + if (binding.kind !== 'variable' || value?.type !== 'new_expression') return false; + if (value.endIndex > node.startIndex && variableScope(binding.scope) === variableScope(scope)) + return false; + const constructor = value.childForFieldName('constructor'); + return constructor !== null && sdkBinding(constructor, scopes.get(value.id)!); +} + +/** An unknown later property can replace description; a later explicit property restores proof. */ +function descriptionFromConfig(config: SyntaxNode): string { + let description = ''; + if (config.type !== 'object') return description; + for (const child of config.namedChildren) { + if (child.type === 'comment') continue; + const key = child.childForFieldName('key') ?? child.childForFieldName('name'); + const name = + child.type === 'shorthand_property_identifier' ? child.text : key && propertyName(key); + if (child.type === 'pair' && name === 'description') { + const value = child.childForFieldName('value'); + description = value ? (plainString(value) ?? '') : ''; + } else if (!name || name === 'description') { + description = ''; + } + } + return description; +} + +function handlerNodeId( + node: SyntaxNode, + scope: Scope, + callableBindings: ReadonlyMap | undefined, +): string | undefined { + if (node.type !== 'identifier') return undefined; + const binding = lookup(scope, node.text); + if (!binding || binding.invalid) return undefined; + if (binding.kind === 'variable') { + const value = binding.value; + if ( + !binding.immutable || + !value || + (value.type !== 'arrow_function' && value.type !== 'function_expression') + ) + return undefined; + if (value.endIndex > node.startIndex && variableScope(binding.scope) === variableScope(scope)) + return undefined; + } else if (binding.kind !== 'function') return undefined; + return callableBindings?.get(binding.name.id); +} + +/** Direct SDK registrations only; no wrapper, alias-chain or runtime-value inference. */ +export function extractToolDefinitions( + tree: Parser.Tree, + filePath: string, + lineOffset = 0, + callableBindings?: ReadonlyMap, +): ExtractedToolDef[] { + // Ordinary files need no lexical walk; every supported receiver originates here. + const importsSdk = tree.rootNode.namedChildren.some((node) => { + if (node.type !== 'import_statement') return false; + const source = node.childForFieldName('source'); + return source !== null && SDK_MODULES.has(plainString(source) ?? ''); + }); + if (!importsSdk) return []; + + const { scopes, calls } = collectBindings(tree.rootNode); + const definitions: ExtractedToolDef[] = []; + for (const call of calls) { + const callee = call.childForFieldName('function'); + if (call.hasError || callee?.type !== 'member_expression') continue; + const receiver = callee.childForFieldName('object'); + const method = callee.childForFieldName('property'); + if ( + !receiver || + method?.type !== 'property_identifier' || + (method.text !== 'registerTool' && method.text !== 'tool') || + !sdkReceiver(receiver, scopes.get(call.id)!, scopes) + ) + continue; + const args = + call + .childForFieldName('arguments') + ?.namedChildren.filter((child) => child.type !== 'comment') ?? []; + if (args.some((arg) => arg.type === 'spread_element')) continue; + if (method.text === 'registerTool' ? args.length !== 3 : args.length < 2 || args.length > 5) + continue; + const toolName = plainString(args[0]); + if (toolName === null) continue; + const description = + method.text === 'registerTool' + ? descriptionFromConfig(args[1]) + : args.length > 2 + ? (plainString(args[1]) ?? '') + : ''; + const handler = handlerNodeId(args[args.length - 1], scopes.get(call.id)!, callableBindings); + definitions.push({ + filePath, + toolName, + description, + lineNumber: call.startPosition.row + 1 + lineOffset, + ...(handler !== undefined ? { handlerNodeId: handler } : {}), + allowFileFallback: false, + }); + } + return definitions; +} diff --git a/gitnexus/src/core/ingestion/pipeline-phases/processes.ts b/gitnexus/src/core/ingestion/pipeline-phases/processes.ts index 6be8839c3..75f8a0b48 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/processes.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/processes.ts @@ -323,6 +323,7 @@ export const processesPhase: PipelinePhase = { const toolsByHandlerId = new Map(); const toolsWithoutHandlerByFile = new Map(); for (const td of toolDefs) { + if (!td.handlerNodeId && td.allowFileFallback === false) continue; const key = td.handlerNodeId ?? td.filePath; const targetMap = td.handlerNodeId ? toolsByHandlerId : toolsWithoutHandlerByFile; let list = targetMap.get(key); diff --git a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts index 42bf639d2..cc1c48488 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts @@ -14,7 +14,7 @@ import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js'; import { getPhaseOutput } from './types.js'; import type { ParseOutput } from './parse.js'; -import { isBladeTemplateFilename } from 'gitnexus-shared'; +import { isTemplateRouteCandidate } from '../utils/template-file.js'; import { nextjsFileToRouteURL, normalizeFetchURL } from '../route-extractors/nextjs.js'; import { expoFileToRouteURL } from '../route-extractors/expo.js'; import { phpFileToRouteURL } from '../route-extractors/php.js'; @@ -41,6 +41,8 @@ import { readFileContents } from '../filesystem-walker.js'; import { isDev } from '../utils/env.js'; import { logger } from '../../logger.js'; +export { isTemplateRouteCandidate } from '../utils/template-file.js'; + const EXPO_NAV_PATTERNS = [ /router\.(push|replace|navigate)\(\s*['"`]([^'"`]+)['"`]/g, /]*href=\s*['"`]([^'"`]+)['"`]/g, @@ -121,17 +123,6 @@ function hasRouteParameters(routeUrl: string): boolean { return /\{[^}]+\}/.test(routeUrl); } -export const isTemplateRouteCandidate = (filePath: string): boolean => { - const normalized = filePath.replace(/\\/g, '/').toLowerCase(); - return ( - normalized.endsWith('.html') || - normalized.endsWith('.htm') || - normalized.endsWith('.ejs') || - normalized.endsWith('.hbs') || - isBladeTemplateFilename(normalized) - ); -}; - export function extractTemplateStaticFetchCalls( filePath: string, content: string, diff --git a/gitnexus/src/core/ingestion/pipeline-phases/tools.ts b/gitnexus/src/core/ingestion/pipeline-phases/tools.ts index 32a0ae71f..296d0a42a 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/tools.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/tools.ts @@ -22,6 +22,7 @@ export interface ToolDef { filePath: string; description: string; handlerNodeId?: string; + allowFileFallback?: false; } export interface ToolsOutput { @@ -51,6 +52,7 @@ export const toolsPhase: PipelinePhase = { filePath: td.filePath, description: td.description, ...(handlerNodeId !== undefined ? { handlerNodeId } : {}), + ...(td.allowFileFallback === false ? { allowFileFallback: false as const } : {}), }); } diff --git a/gitnexus/src/core/ingestion/route-extractors/go-gin-echo.ts b/gitnexus/src/core/ingestion/route-extractors/go-gin-echo.ts index f7cdd1c64..e77b1b82b 100644 --- a/gitnexus/src/core/ingestion/route-extractors/go-gin-echo.ts +++ b/gitnexus/src/core/ingestion/route-extractors/go-gin-echo.ts @@ -26,6 +26,7 @@ import type Parser from 'tree-sitter'; import { goImportPackageName } from '../languages/go/import-package-name.js'; import { GoRouteBindings, type GoRouteBinding } from '../languages/go/route-bindings.js'; +import { stringLiteral } from './go-shared.js'; import { normalizeExtractedRoutePath } from './route-path.js'; import type { SyntaxNode } from 'tree-sitter'; import type { ExtractedDecoratorRoute, RouteHandlerReceiver } from '../workers/parse-worker.js'; @@ -58,59 +59,6 @@ interface Framework { const FUNCTION_TYPE_LIST = ['function_declaration', 'method_declaration', 'func_literal']; const FUNCTION_TYPES: ReadonlySet = new Set(FUNCTION_TYPE_LIST); -function stringLiteral(node: SyntaxNode | null | undefined): string | null { - if (!node || node.hasError) return null; - const body = node.text.slice(1, -1); - // Go discards carriage returns in raw strings, including CRLF source files. - if (node.type === 'raw_string_literal') return body.replace(/\r/g, ''); - if (node.type !== 'interpreted_string_literal') return null; - if (!body.includes('\\')) return body; - - const simple: Readonly> = { - a: '\x07', - b: '\b', - f: '\f', - n: '\n', - r: '\r', - t: '\t', - v: '\v', - '\\': '\\', - '"': '"', - }; - const chunks: Buffer[] = []; - const tokens = - /\\(?:[abfnrtv\\"]|[0-7]{3}|x[\da-fA-F]{2}|u[\da-fA-F]{4}|U[\da-fA-F]{8})|[^\\"\n]+/g; - let consumed = 0; - for (const match of body.matchAll(tokens)) { - if (match.index !== consumed) return null; - const token = match[0]; - consumed += token.length; - if (!token.startsWith('\\')) { - chunks.push(Buffer.from(token)); - } else if (simple[token[1]] !== undefined) { - chunks.push(Buffer.from(simple[token[1]])); - } else { - const octal = /[0-7]/.test(token[1]); - const value = Number.parseInt(token.slice(octal ? 1 : 2), octal ? 8 : 16); - if (octal || token[1] === 'x') { - // Octal and hex escapes encode bytes, not Unicode code points. - if (value > 255) return null; - chunks.push(Buffer.from([value])); - } else { - if (value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff)) return null; - chunks.push(Buffer.from(String.fromCodePoint(value))); - } - } - } - if (consumed !== body.length) return null; - try { - // Arbitrary non-UTF-8 Go byte strings cannot be represented losslessly in a URL. - return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(Buffer.concat(chunks)); - } catch { - return null; - } -} - /** The framework this file routes with, when exactly one is imported. */ function readImports(root: SyntaxNode): { readonly framework: Framework | null; diff --git a/gitnexus/src/core/ingestion/route-extractors/go-shared.ts b/gitnexus/src/core/ingestion/route-extractors/go-shared.ts new file mode 100644 index 000000000..a2d13faef --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/go-shared.ts @@ -0,0 +1,75 @@ +import type { SyntaxNode } from 'tree-sitter'; + +/** + * Go string-literal decoding shared by the Go route extractors on both + * layers: the ingestion extractor (`go-gin-echo.ts`, Strategy A) and the + * group-mode HTTP plugin (`group/extractors/http-patterns/go.ts`, + * Strategy B). Both sides must produce the SAME text for a route literal, + * or the same route lands under two different contract ids that never + * collide in the merge — a wrong-path duplicate that survives alongside + * the graph's correct entry. + * + * Go's semantics, per `strconv.Unquote`: + * - interpreted (`"…"`) strings decode escapes (`\n`, `\x2f`, `ሴ`, + * `\101`, …); hex and octal escapes encode BYTES, not code points; + * - raw (`` `…` ``) strings have no escapes (a backslash is literal), + * and Go discards carriage returns in them, including CRLF source files; + * - a string that cannot be decoded to valid UTF-8 text (invalid escape, + * non-UTF-8 bytes) returns null — callers decline instead of guessing, + * since a URL cannot carry those bytes losslessly. + * + * Nodes that are not string literals (an identifier prefix, a rune + * literal, an errored subtree) also return null. + */ +export function stringLiteral(node: SyntaxNode | null | undefined): string | null { + if (!node || node.hasError) return null; + const body = node.text.slice(1, -1); + // Go discards carriage returns in raw strings, including CRLF source files. + if (node.type === 'raw_string_literal') return body.replace(/\r/g, ''); + if (node.type !== 'interpreted_string_literal') return null; + if (!body.includes('\\')) return body; + + const simple: Readonly> = { + a: '\x07', + b: '\b', + f: '\f', + n: '\n', + r: '\r', + t: '\t', + v: '\v', + '\\': '\\', + '"': '"', + }; + const chunks: Buffer[] = []; + const tokens = + /\\(?:[abfnrtv\\"]|[0-7]{3}|x[\da-fA-F]{2}|u[\da-fA-F]{4}|U[\da-fA-F]{8})|[^\\"\n]+/g; + let consumed = 0; + for (const match of body.matchAll(tokens)) { + if (match.index !== consumed) return null; + const token = match[0]; + consumed += token.length; + if (!token.startsWith('\\')) { + chunks.push(Buffer.from(token)); + } else if (simple[token[1]] !== undefined) { + chunks.push(Buffer.from(simple[token[1]])); + } else { + const octal = /[0-7]/.test(token[1]); + const value = Number.parseInt(token.slice(octal ? 1 : 2), octal ? 8 : 16); + if (octal || token[1] === 'x') { + // Octal and hex escapes encode bytes, not Unicode code points. + if (value > 255) return null; + chunks.push(Buffer.from([value])); + } else { + if (value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff)) return null; + chunks.push(Buffer.from(String.fromCodePoint(value))); + } + } + } + if (consumed !== body.length) return null; + try { + // Arbitrary non-UTF-8 Go byte strings cannot be represented losslessly in a URL. + return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(Buffer.concat(chunks)); + } catch { + return null; + } +} diff --git a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts index 46a80060a..22a7a2d41 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts @@ -1235,6 +1235,9 @@ export interface ScopeResolver { readonly isCallableVisibleFromCaller?: (ctx: { readonly callerParsed: ParsedFile; readonly candidate: SymbolDefinition; + /** Arity of the actual call, when known. A visibility veto must not + * infer applicability from a name match alone. */ + readonly callArity?: number; /** Caller's enclosing scope id. Languages that gate visibility on * caller scope (e.g. C++ two-phase template lookup) consult it; * others ignore. Optional so existing implementations stay valid. */ diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts index 6693a2bc6..af556d0d3 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts @@ -84,12 +84,7 @@ export function emitFreeCallFallback( * appended to its reason. See `ScopeResolver.markConstructionSites`. */ readonly markConstructionSites?: boolean; readonly isFileLocalDef?: (def: SymbolDefinition) => boolean; - readonly isCallableVisibleFromCaller?: (ctx: { - readonly callerParsed: ParsedFile; - readonly candidate: SymbolDefinition; - readonly callerScope?: ScopeId; - readonly scopes?: ScopeResolutionIndexes; - }) => boolean; + readonly isCallableVisibleFromCaller?: ScopeResolver['isCallableVisibleFromCaller']; readonly resolveAdlCandidates?: ( site: { readonly name: string; @@ -620,6 +615,7 @@ export function emitFreeCallFallback( options.isCallableVisibleFromCaller!({ callerParsed: parsed, candidate, + callArity: site.arity, callerScope: site.inScope, scopes, }) @@ -698,6 +694,7 @@ export function emitFreeCallFallback( !options.isCallableVisibleFromCaller({ callerParsed: parsed, candidate: fnDef, + callArity: site.arity, callerScope: site.inScope, scopes, }) diff --git a/gitnexus/src/core/ingestion/utils/template-file.ts b/gitnexus/src/core/ingestion/utils/template-file.ts new file mode 100644 index 000000000..1024de74e --- /dev/null +++ b/gitnexus/src/core/ingestion/utils/template-file.ts @@ -0,0 +1,13 @@ +import { isBladeTemplateFilename } from 'gitnexus-shared'; + +/** Templates whose URLs and fetches can contribute route relationships to the graph. */ +export const isTemplateRouteCandidate = (filePath: string): boolean => { + const normalized = filePath.replace(/\\/g, '/').toLowerCase(); + return ( + normalized.endsWith('.html') || + normalized.endsWith('.htm') || + normalized.endsWith('.ejs') || + normalized.endsWith('.hbs') || + isBladeTemplateFilename(normalized) + ); +}; diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index e6333c65d..18b65eba1 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -439,6 +439,8 @@ export interface ExtractedToolDef { description: string; lineNumber: number; handlerNodeId?: string; + /** Unresolved registrations must not inherit unrelated same-file flows. */ + allowFileFallback?: false; } export interface ExtractedORMQuery { @@ -1685,6 +1687,7 @@ const processFileGroup = ( // node id → graph node id for classes THIS file's capture loop materialized. // Keyed by in-memory AST identity (never persisted); filled below. const classOwnersByNodeId = new Map(); + const callableBindings = new Map(); // #2687: ONE pass over `matches` yields both suppression sets — the // definition-name claims by rank (callable > Property > value), so the dedup @@ -3123,6 +3126,11 @@ const processFileGroup = ( }), }); + // Keep actual emitted identities; providers must not reconstruct graph IDs. + if (nameNode && (nodeLabel === 'Function' || nodeLabel === 'Method')) { + callableBindings.set(nameNode.id, nodeId); + } + // enclosingClassId already computed above (before nodeId generation) const ownerId = enclosingClassId ?? objectLiteralOwnerInfo?.ownerId; @@ -3225,6 +3233,23 @@ const processFileGroup = ( } } + if (provider.extractToolDefinitions) { + // Distinct lexical declarations can share a graph ID (for example, sibling + // block-scoped functions). Such IDs cannot prove which handler owns a tool. + const seenCallableIds = new Set(); + const ambiguousCallableIds = new Set(); + for (const nodeId of callableBindings.values()) { + if (seenCallableIds.has(nodeId)) ambiguousCallableIds.add(nodeId); + seenCallableIds.add(nodeId); + } + for (const [bindingId, nodeId] of callableBindings) { + if (ambiguousCallableIds.has(nodeId)) callableBindings.delete(bindingId); + } + result.toolDefs.push( + ...provider.extractToolDefinitions(tree, file.path, lineOffset, callableBindings), + ); + } + // Extract framework routes via provider detection (e.g., Laravel routes.php) if (provider.isRouteFile?.(file.path)) { const extractedRoutes = extractLaravelRoutes(tree, file.path); diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index aafb556f6..5230ec713 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -110,9 +110,13 @@ export const sanitizeUTF8 = (str: string): string => { }; export const escapeCSVField = (value: string | number | undefined | null): string => { - if (value === undefined || value === null) return '""'; + if (value === undefined || value === null) return ''; let str = String(value); str = sanitizeUTF8(str); + // Preserve the NULL meaning of absent strings from the 0.18.3 writer. + // Newer LadybugDB readers retain a quoted empty field as a STRING value; + // it must stay unquoted or unresolved destination addresses can join. + if (str.length === 0) return ''; return `"${str.replace(/"/g, '""')}"`; }; diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 970832995..92adff18a 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -1101,28 +1101,78 @@ export type LbugProgressCallback = (message: string) => void; /** * Run a COPY, retrying once with IGNORE_ERRORS=true (which skips row-level - * errors) on first failure. On a second failure, hand the RAW retry error to - * `onError` — each call site formats + slices its own message (#2226 F5: node - * COPY slices to 200 chars and throws; relationship COPY slices to 80 and warns, - * so the helper must not pre-format and lose that distinction). `onError` may - * throw to propagate the failure. + * errors) on first failure. Log the original failure and native COPY/warning + * receipts even when the retry succeeds. Node COPY requires every row; a + * skipped-node receipt is a failure. Call sites retain their own message + * limits and relationship fallback policy. */ const copyCsvWithRetry = async ( targetConn: lbug.Connection, copyQuery: string, onError: (retryErr: unknown) => void, + expectedRows?: number, ): Promise => { try { await queryAndDrain(targetConn, copyQuery); - } catch { + } catch (firstError) { + logger.warn( + { err: firstError, copyQuery }, + 'First COPY failure; retrying with IGNORE_ERRORS=true', + ); try { const retryQuery = copyQuery.replace( 'auto_detect=false)', 'auto_detect=false, IGNORE_ERRORS=true)', ); - await queryAndDrain(targetConn, retryQuery); + // Keep COPY and its connection-local warning receipt in one critical + // section. Only project diagnostics, never skipped_line_or_record: that + // column contains source text. Warnings are retained at a bounded native + // limit, so their count is a lower bound, not the exact skipped total. + const retry = async () => { + await drainQueryResult(await targetConn.query('CALL CLEAR_WARNINGS()')); + const result = await readQueryRows(await targetConn.query(retryQuery)); + const warnings = await readQueryRows( + await targetConn.query('CALL SHOW_WARNINGS() RETURN message, file_path, line_number'), + ); + // The native warning limit can be zero; warning rows alone cannot + // prove that every CSV row landed. Node COPY exposes its copied count + // in the result receipt even when warning retention is disabled. + const countReceipt = result + .map((row) => String(row.result ?? '')) + .map((message) => /^(\d+) tuples have been copied/.exec(message)) + .find(Boolean); + const copiedRows = countReceipt ? Number(countReceipt[1]) : undefined; + logger.warn( + { + copyQuery, + copyResult: result, + expectedRows, + copiedRows, + skippedRows: + expectedRows !== undefined && copiedRows !== undefined + ? Math.max(0, expectedRows - copiedRows) + : undefined, + retainedWarnings: warnings.length, + warningSamples: warnings.slice(0, 5), + }, + 'COPY retry completed; retained warnings describe skipped rows (a lower bound)', + ); + if (expectedRows !== undefined && (copiedRows !== expectedRows || warnings.length > 0)) { + throw new Error( + `COPY retry skipped rows or could not verify a complete node load ` + + `(copied ${copiedRows ?? 'unknown'} of ${expectedRows}; ${warnings.length} retained warning(s))`, + ); + } + }; + await (isSharedSingletonConn(targetConn) ? withConnLock(retry) : retry()); } catch (retryErr) { - onError(retryErr); + const firstMessage = firstError instanceof Error ? firstError.message : String(firstError); + const retryMessage = retryErr instanceof Error ? retryErr.message : String(retryErr); + onError( + new Error(`COPY retry failed: ${retryMessage}; first failure: ${firstMessage}`, { + cause: retryErr, + }), + ); } } }; @@ -1192,17 +1242,22 @@ const copyNodeCSVs = async ( if (!(await stagingCsvExists(csvPath))) throw missingStagingCsvError(table, csvPath, rows); const copyQuery = getCopyQuery(table, normalizeCopyPath(csvPath)); - await copyCsvWithRetry(targetConn, copyQuery, (retryErr) => { - const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr); - // Pool exhaustion gets a remedy (#2631): the raw binder text gives the - // operator nothing to act on, and on non-4K-page hosts (Ascend aarch64, - // Apple Silicon) the pool bills up to pageSize/4KiB x faster than the - // sizing was calibrated for — name the knob and the mechanism. - const remedy = bufferPoolExhaustionRemedy(retryMsg); - throw new Error( - `COPY failed for ${table}: ${retryMsg.slice(0, 200)}${remedy ? ` ${remedy}` : ''}`, - ); - }); + await copyCsvWithRetry( + targetConn, + copyQuery, + (retryErr) => { + const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr); + // Pool exhaustion gets a remedy (#2631): the raw binder text gives the + // operator nothing to act on, and on non-4K-page hosts (Ascend aarch64, + // Apple Silicon) the pool bills up to pageSize/4KiB x faster than the + // sizing was calibrated for — name the knob and the mechanism. + const remedy = bufferPoolExhaustionRemedy(retryMsg); + throw new Error( + `COPY failed for ${table}: ${retryMsg.slice(0, 200)}${remedy ? ` ${remedy}` : ''}`, + ); + }, + rows, + ); } }; @@ -3989,6 +4044,7 @@ export const buildFtsQueryCypher = ( * @param query - Search query string * @param limit - Maximum results * @param conjunctive - If true, all terms must match (AND); if false, any term matches (OR) + * @param missingIndex - Preserve the empty-result default, or propagate missing indexes for diagnostics * @returns Array of { node properties, score } */ export const queryFTS = async ( @@ -3997,6 +4053,7 @@ export const queryFTS = async ( query: string, limit: number = 20, conjunctive: boolean = false, + missingIndex: 'empty' | 'throw' = 'empty', ): Promise< Array<{ nodeId: string; name: string; filePath: string; score: number; [key: string]: any }> > => { @@ -4027,7 +4084,7 @@ export const queryFTS = async ( // NEW-6 — this used to be a bare `.includes('does not exist')` check // that could not tell the two apart). const message = e instanceof Error ? e.message : String(e); - if (classifyFtsQueryError(message) === 'missing-index') { + if (missingIndex === 'empty' && classifyFtsQueryError(message) === 'missing-index') { return []; } throw e; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 8d302c913..5f43df3e2 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -36,7 +36,12 @@ import fs from 'fs/promises'; import { constants as fsConstants, existsSync } from 'node:fs'; import { randomUUID } from 'node:crypto'; import { retryRename } from '../storage/fs-atomic.js'; -import { acquireIndexLock, requireExclusiveIndexLock } from '../storage/index-lock.js'; +import { + acquireIndexLock, + requireExclusiveIndexLock, + sweepStagingArtifacts, +} from '../storage/index-lock.js'; +import { resolveEmbeddingRecovery } from '../storage/embedding-recovery.js'; import { invalidateNodeWorkspacePackages } from './ingestion/import-resolvers/node-workspace-packages.js'; import { logNameFallbackSummary, @@ -51,6 +56,7 @@ import { import { summarizeUndecidedSatisfaction } from './ingestion/scope-resolution/undecided-satisfaction.js'; import { summarizeScopeExtractionFailures } from './ingestion/scope-resolution/scope-extraction-failures.js'; import type { KnowledgeGraph } from './graph/types.js'; +import { reconcileGraphNodeIdentities } from './incremental/write-reconciliation.js'; import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js'; import { initLbug, @@ -128,6 +134,7 @@ import { resolveFtsVersionPair } from './lbug/vendored-extension-path.js'; import { startWalCheckpointDriver, checkpointOnce, + isManualCheckpointEnabled, type WalCheckpointDriver, } from './lbug/wal-checkpoint-driver.js'; import { @@ -1285,6 +1292,8 @@ export async function runFullAnalysis( const log = (msg: string) => callbacks.onLog?.(stripControlCharacters(msg)); const acquireOpts = { log, + // Resolve and validate the canonical slot under the lock before cleanup. + sweep: false, onWaitStart: () => callbacks.onProgress('lock', 0, 'Waiting for another analyze to finish on this index…'), }; @@ -1343,6 +1352,7 @@ export async function runFullAnalysis( } const flatShared = writeTarget.placement.branch ? undefined : writeTarget.sharedStore; if (flatShared) await seedSharedSlot(flatShared, repoPath, log); + sweepStagingArtifacts(writeTarget.metaDir, log); const slotToLeave = options.noShare ? await optedInSlotToLeave(repoPath) : undefined; const result = await runFullAnalysisInner( repoPath, @@ -1840,7 +1850,13 @@ async function runFullAnalysisInner( decision = decideEmbeddingResume(checkpoint, embeddingIdentityForRun, resumeOptions); } if (decision.action === 'abort') throw new Error(decision.error); - log(decision.log); + log( + decision.action === 'resume' && checkpoint.recovery + ? `Previous analyze recorded an embedding checkpoint (${checkpoint.nodesProcessed}/` + + `${checkpoint.totalNodes} nodes); validating staged vectors before retrying ` + + `${decision.pendingNodeIds.size} pending node(s).` + : decision.log, + ); if (options.dropEmbeddings) { // --drop-embeddings has always implied a rebuild here; the decision only // covers the marker. @@ -2669,6 +2685,74 @@ async function runFullAnalysisInner( } } + // A checkpoint's pending decision and its paid, complete vectors are + // independent: --force discards the former but can still reuse the latter. + // Select only the explicitly referenced generation, never an orphan by age. + const stagedRecovery = resolveEmbeddingRecovery(metaDir, existingMeta?.embeddingCheckpoint); + const stagedCheckpoint = existingMeta?.embeddingCheckpoint; + const inheritedUnsafeNodeIds = new Set([ + ...pendingEmbeddingNodeIds, + ...(stagedRecovery?.unsafeNodeIds ?? []), + ]); + if (shouldLoadCache && !options.dropEmbeddings && stagedRecovery && stagedCheckpoint) { + if (!embeddingIdentityForRun) { + const { resolveEmbeddingIdentity } = await import('./embeddings/embedding-identity.js'); + embeddingIdentityForRun = resolveEmbeddingIdentity(); + } + const marker = stagedCheckpoint; + const matchesIdentity = + marker.model === embeddingIdentityForRun.model && + marker.dimensions === embeddingIdentityForRun.dimensions && + marker.provider === embeddingIdentityForRun.provider; + if (matchesIdentity && stagedRecovery.schemaFingerprint === SCHEMA_FINGERPRINT) { + // A force-discarded pending decision must not turn a known incomplete + // inherited group into a reusable cache merely because its hash matches. + if (inheritedUnsafeNodeIds.size > 0) { + const rows = cachedSnapshot.rows.filter((row) => !inheritedUnsafeNodeIds.has(row.nodeId)); + cachedSnapshot = { + ...cachedSnapshot, + rows, + embeddingNodeIds: new Set(rows.map((row) => row.nodeId)), + }; + } + let recovered: CachedEmbeddingsSnapshot | undefined; + try { + const { recoverStagedEmbeddings, mergeRecoveredEmbeddings } = + await import('./embeddings/staged-embedding-recovery.js'); + recovered = await recoverStagedEmbeddings(stagedRecovery.dbPath, { + dimensions: embeddingIdentityForRun.dimensions, + excludedNodeIds: stagedRecovery.unsafeNodeIds, + }); + const liveCacheDims = snapshotEmbeddingDims(cachedSnapshot); + if (liveCacheDims !== undefined && liveCacheDims !== embeddingIdentityForRun.dimensions) { + log( + `Embedding dimensions changed (${liveCacheDims}d -> ` + + `${embeddingIdentityForRun.dimensions}d), discarding published cache`, + ); + discardCachedEmbeddings(); + } + if (recovered.rows.length > 0) { + const merged = mergeRecoveredEmbeddings(cachedSnapshot, recovered); + disposeEmbeddingSpill(cachedSnapshot.spill); + adoptCachedEmbeddings(merged); + } + log( + `Recovered ${recovered.rows.length} complete staged embedding chunk(s) ` + + `for ${recovered.embeddingNodeIds.size} node(s); unchanged content can reuse them.`, + ); + } catch (err) { + log( + `Warning: could not recover staged embeddings (${(err as Error).message}); ` + + 'the retry will regenerate missing chunks.', + ); + } finally { + disposeEmbeddingSpill(recovered?.spill); + } + } else { + log('Staged embedding identity or schema changed; its vectors will not be reused.'); + } + } + // ── Load incremental parse cache ────────────────────────────────── // Content-addressed: `--force` reuses parser shards; `useParseCache: false` // stages a new generation under a run-unique parse-rebuild.* dir and publishes @@ -2854,7 +2938,7 @@ async function runFullAnalysisInner( // (Bugbot review on PR #1479: a prediction that flipped post-pipeline // could skip the embedding cache load and then take the full-rebuild // path, silently losing embeddings). - const isIncremental = + const incrementalEligible = !options.force && !!existingMeta && // Belt and braces, not a second gate: the guard above already set `force` @@ -2867,6 +2951,13 @@ async function runFullAnalysisInner( repoHasGit && allFilePaths.length > 0; + // Select a full build before any selective mutation when the operator has + // disabled the checkpoint needed to certify an incremental publication. + const isIncremental = incrementalEligible && isManualCheckpointEnabled(); + if (incrementalEligible && !isIncremental) { + log('Manual WAL checkpoints are disabled; switching to a full DB write before mutation.'); + } + const hashDiff = isIncremental ? diffFileHashes(newFileHashes, existingMeta!.fileHashes) : undefined; @@ -3112,6 +3203,24 @@ async function runFullAnalysisInner( // collapse check compares the whole in-memory graph against the whole DB, // which is only a like-for-like comparison on a full rebuild. let wroteChangedSubgraphOnly = false; + const markIncrementalGraphVerification = async (): Promise => { + if (buildPath !== lbugPath) return; + const latest = await loadMeta(metaDir); + if (!latest?.incrementalInProgress) { + throw new Error('Cannot certify incremental graph without its dirty metadata marker.'); + } + // A failed or aborted identity scan is a graph failure. FTS-only repair + // and FTS crash recovery must not clear it while retaining these rows. + await saveMeta(metaDir, { + ...latest, + incrementalInProgress: { + ...latest.incrementalInProgress, + phase: 'graph-reconciliation', + updatedAt: Date.now(), + checkpointSucceeded: false, + }, + }); + }; let incrementalFtsRebuildTables: Set | undefined; if (isIncremental && hashDiff) { // ── Incremental DB writeback ─────────────────────────────────── @@ -3163,6 +3272,8 @@ async function runFullAnalysisInner( phase: string, extra: Partial> = {}, ): Promise => { + // Do not stamp live metadata while writing a staging database. + if (buildPath !== lbugPath) return; await saveMeta(metaDir, { ...existingMeta!, incrementalInProgress: { @@ -3906,6 +4017,14 @@ async function runFullAnalysisInner( 'Continuing; recovery will treat the graph-boundary checkpoint as unsuccessful.', ); } + if (wroteChangedSubgraphOnly) { + await markIncrementalGraphVerification(); + await reconcileGraphNodeIdentities( + pipelineResult.graph, + executeQuery, + 'post-COPY/checkpoint', + ); + } if (shouldStampFtsDirtyPhase(ftsWritePlan)) { // Lift the prior-meta precondition: a first-ever in-place run (Windows // full rebuild, or any in-place incremental) must stamp too. Staging @@ -4002,6 +4121,7 @@ async function runFullAnalysisInner( ? (table, indexName) => log(`FTS: ready ${table}.${indexName}`) : undefined, }); + if (wroteChangedSubgraphOnly) await markIncrementalGraphVerification(); if (ftsResult.ok) { progress('fts', 90, 'Search indexes ready'); } else if (ftsFailureIsFatal(ftsResult.failureClass, useAtomicSwap)) { @@ -4054,6 +4174,12 @@ async function runFullAnalysisInner( progress('fts', 90, 'Search indexes skipped (FTS unavailable)'); } + if (wroteChangedSubgraphOnly) { + // FTS has returned. Later embedding/finalization failures must require + // graph recovery, since post-FTS node identities are not yet certified. + await markIncrementalGraphVerification(); + } + // ── Phase 3.5: Re-insert cached embeddings ──────────────────────── // Runs on BOTH the full-rebuild path and the incremental path: // - Full rebuild / escalated write: DB was wiped, every cached row @@ -4428,6 +4554,7 @@ async function runFullAnalysisInner( semanticMode = vectorIndexReady ? 'vector-index' : 'exact-scan'; } + let stagedCheckpointEmbeddingCount: number | undefined; if (!embeddingSkipped) { const { isHttpMode } = await import('./embeddings/http-client.js'); const httpMode = isHttpMode(); @@ -4442,6 +4569,16 @@ async function runFullAnalysisInner( embeddingIdentityForRun = resolveEmbeddingIdentity(); } const embeddingIdentity = embeddingIdentityForRun; + const stagedRecoveryEnabled = useAtomicSwap && isManualCheckpointEnabled(); + if (useAtomicSwap && !stagedRecoveryEnabled) { + log( + 'Manual WAL checkpoints are disabled; new staged work cannot be recovered after interruption. ' + + 'Any previous durable recovery source is retained until publication.', + ); + } + const unsafeRecoveryNodeIds = new Set([...inheritedUnsafeNodeIds, ...restoreFailedNodeIds]); + let activeWindowNodeIds: string[] = []; + let recoveryGenerationDurable = false; // Build a Map from cached embeddings for incremental mode let existingEmbeddings: Map | undefined; if (cachedSnapshot.embeddingNodeIds.size > 0) { @@ -4478,11 +4615,10 @@ async function runFullAnalysisInner( // /api/embed checkpoint writer in server/api.ts already uses, which also // keeps a concurrent writer's update from being reverted by a stale // snapshot) and replace ONLY `embeddingCheckpoint` — plus - // `stats.embeddings` when the caller actually MEASURED the live count - // (the post-window `onCheckpoint`). The window-start callback passes - // nothing: restating the previous run's count there both re-published a - // stale number and clobbered the live count a preceding `onCheckpoint` - // had just written. + // `stats.embeddings` when the caller actually MEASURED the published + // index (the post-window `onCheckpoint` on an in-place build). A staging + // build's count is not published until the atomic swap succeeds. The + // window-start callback passes nothing, preserving the latest count. const saveEmbeddingCheckpoint = async ( checkpoint: { nodesProcessed: number; @@ -4492,7 +4628,18 @@ async function runFullAnalysisInner( pendingNodeIds: string[], embeddings?: number, ): Promise => { + if (embeddings !== undefined && buildPath !== lbugPath) { + stagedCheckpointEmbeddingCount = embeddings; + } const latestMeta = (await loadMeta(metaDir)) ?? existingMeta; + // An in-place write or manual-checkpoint opt-out cannot create a new + // recoverable staged generation. Keep the complete previous receipt: + // updated progress or unsafe nodes would describe different source bytes. + const preservedRecoveryCheckpoint = + !stagedRecoveryEnabled && + resolveEmbeddingRecovery(metaDir, latestMeta?.embeddingCheckpoint) + ? latestMeta?.embeddingCheckpoint + : undefined; // First-ever analyze of this repo: no meta exists on disk yet (the // pre-wipe dirty stamp only fires when one does). Mint the minimum // RepoMeta requires, with `lastCommit: ''` — never `currentCommit` — @@ -4503,16 +4650,30 @@ async function runFullAnalysisInner( lastCommit: '', indexedAt: new Date().toISOString(), }; + const interrupted = mintInterruptedCheckpoint( + embeddingIdentity, + checkpoint, + pendingNodeIds, + ); await saveMeta(metaDir, { ...base, - ...(embeddings === undefined ? {} : { stats: { ...base.stats, embeddings } }), + ...(embeddings === undefined || buildPath !== lbugPath + ? {} + : { stats: { ...base.stats, embeddings } }), // Written by a run that is still IN FLIGHT — see the `kind` doc in // repo-manager.ts. - embeddingCheckpoint: mintInterruptedCheckpoint( - embeddingIdentity, - checkpoint, - pendingNodeIds, - ), + embeddingCheckpoint: preservedRecoveryCheckpoint ?? { + ...interrupted, + ...(stagedRecoveryEnabled + ? { + recovery: { + stagingFile: path.basename(buildPath), + schemaFingerprint: SCHEMA_FINGERPRINT, + unsafeNodeIds: [...unsafeRecoveryNodeIds], + }, + } + : {}), + }, }); }; @@ -4535,7 +4696,21 @@ async function runFullAnalysisInner( { forceReembedNodeIds: pendingEmbeddingNodeIds, onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => { + const handoff = stagedRecoveryEnabled && !recoveryGenerationDurable; + if (handoff) { + if (!(await checkpointOnce())) { + throw new Error( + 'Could not checkpoint restored embeddings before recovery handoff.', + ); + } + recoveryGenerationDurable = true; + } + activeWindowNodeIds = nodeIds; + for (const id of nodeIds) unsafeRecoveryNodeIds.add(id); await saveEmbeddingCheckpoint(checkpoint, nodeIds); + // Reclaim the old source only after the new durable generation's + // reference is saved. Later windows retain this same generation. + if (handoff) sweepStagingArtifacts(metaDir, log); }, // ── The mid-run count is a DIAGNOSTIC, not a gate (#2790) ────── // This used to run the count query bare. THIS callback's rejection @@ -4549,7 +4724,12 @@ async function runFullAnalysisInner( // touch stats.embeddings" signal — so the checkpoint still lands, // with whatever count is already on disk left alone. onCheckpoint: async (checkpoint) => { - await checkpointOnce(); + const durable = await checkpointOnce(); + if (stagedRecoveryEnabled && !durable) { + throw new Error('Could not checkpoint the completed embedding window for recovery.'); + } + for (const id of activeWindowNodeIds) unsafeRecoveryNodeIds.delete(id); + activeWindowNodeIds = []; const measured = await measurePersistedEmbeddingCount(executeQuery); if (measured.kind === 'unknown') { log( @@ -4708,14 +4888,13 @@ async function runFullAnalysisInner( // already written to disk: prior meta says 0, a clean run inserts // embeddings and checkpoints the real count, the final probe is // unavailable, and finalization carries the stale 0 forward while reporting - // success. `loadMeta` never throws (it returns null), and the checkpoint - // writer already re-reads the same way, so this is the same freshness - // discipline applied to the same field. + // success. For a staged build, use its last measured count only in the + // final meta, written after the swap; never publish it at a checkpoint. const latestMetaForCount = embeddingCount === undefined ? ((await loadMeta(metaDir)) ?? existingMeta) : undefined; const persistedEmbeddingCount = resolvePersistedEmbeddingCount( measuredEmbeddingCount, - latestMetaForCount?.stats?.embeddings, + stagedCheckpointEmbeddingCount ?? latestMetaForCount?.stats?.embeddings, ); const { getRuntimeCapabilities } = await import('./platform/capabilities.js'); @@ -4956,96 +5135,25 @@ async function runFullAnalysisInner( // Parse-cache publish waits until after that swap + saveMeta so a failed // registerRepo / close / swap cannot replace live shards (#3153). - // Forward the --name alias and the registry-collision bypass bit. - // `allowDuplicateName` is its own concern — independent from the - // pipeline `force` above. The CLI maps it from - // `--allow-duplicate-name` only; `--force` and `--skills` both - // trigger pipeline re-run but never bypass the registry guard. - // The returned name is the one actually written to the registry - // (after applying the precedence chain in registerRepo) — reuse it - // so AGENTS.md / skill files reference the same name MCP clients - // will look up (#979). - const projectName = await registerRepo(repoPath, meta, { - name: options.registryName, - onRename: (previousName, nextName) => - log(`Registry name changed: "${previousName}" -> "${nextName}".`), - allowDuplicateName: options.allowDuplicateName, - // Non-primary branch runs upsert into the entry's branches[]; the - // primary/flat run (placement.branch === undefined) refreshes the - // top-level fields (#2106). - branch: placement.branch, - storagePath, - }); - - // ── #2354: the flat workspace slot has adopted this run's branch ────── - // Drop a now-shadowed `branches//` sub-index for the same label - // (unreachable once the flat slot serves it) and align the registry's - // top-level branch label. Best-effort (#2364 review F5): the index is - // complete and registered, and a failure - // here leaves only a stale registry label / undeleted shadowed dir — - // never wrong routing, because the flat meta this run already stamped is - // what applyBranchScope trusts. Retried by the next content-changing run - // (same-commit fast-path runs skip it: their guard compares the - // already-stamped meta label). - if (!placement.branch && branchLabel) { - try { - await adoptFlatBranchLabel(repoPath, branchLabel, storagePath); - } catch (e) { - log( - `Warning: could not sync the workspace branch label (${(e as Error).message}); continuing.`, + if (wroteChangedSubgraphOnly) { + // Registry freshness must not advance either. Include FTS, embedding + // restoration and the final WAL drain in the certified boundary. + await markIncrementalGraphVerification(); + await walCheckpointDriver.stop(); + if (!(await checkpointOnce())) { + throw new Error( + 'Graph identity reconciliation failed: final checkpoint could not be verified; run `gitnexus analyze --force`.', ); } + await reconcileGraphNodeIdentities( + pipelineResult.graph, + executeQuery, + 'pre-publish/checkpoint', + ); } - // Keep generated .gitnexus contents ignored without editing the user's root .gitignore. await ensureGitNexusIgnored(repoPath, storagePath); - // ── Generate AI context files (best-effort) ─────────────────────── - let aggregatedClusterCount = 0; - if (pipelineResult.communityResult?.communities) { - const groups = new Map(); - for (const c of pipelineResult.communityResult.communities) { - const label = c.heuristicLabel || c.label || 'Unknown'; - groups.set(label, (groups.get(label) || 0) + c.symbolCount); - } - aggregatedClusterCount = Array.from(groups.values()).filter((count) => count >= 5).length; - } - - // Only (re)generate the repo-root AI context files (AGENTS.md / CLAUDE.md / - // skills) for the primary/flat index (#2106). A non-primary branch analyze - // must not churn the repo's committed AGENTS.md with branch-specific stats. - if (!placement.branch) { - try { - await generateAIContextFiles( - repoPath, - storagePath, - projectName, - { - files: pipelineResult.totalFileCount, - nodes: stats.nodes, - edges: stats.edges, - communities: - pipelineResult.communityResult?.stats.totalCommunities ?? - existingMeta?.stats?.communities, - clusters: aggregatedClusterCount, - processes: - pipelineResult.processResult?.stats.totalProcesses ?? existingMeta?.stats?.processes, - }, - undefined, - { - skipAgentsMd: options.skipAgentsMd, - skipSkills: options.skipSkills, - noStats: options.noStats, - defaultBranch: options.defaultBranch, - hasPdg: options.pdg === true, - hasSpringActuator: options.springActuatorPath !== undefined, - }, - ); - } catch { - // Best-effort — don't fail the entire analysis for context file issues - } - } - // ── Close LadybugDB ────────────────────────────────────────────── // Stop the manual checkpoint driver before closeLbug so its // in-flight CHECKPOINT cannot race the `safeClose` CHECKPOINT. @@ -5108,6 +5216,97 @@ async function runFullAnalysisInner( // is a crash-safety improvement: a failed swap leaves the previous index // live and the next run recovers via the full-rebuild path. await saveMeta(metaDir, meta); + sweepStagingArtifacts(metaDir, log); + + // Registry freshness is published only after the graph and its metadata. + // A failed close, swap, or metadata save must leave the previous registry + // receipt intact, just as it leaves the cache unpublished. + // Forward the --name alias and the registry-collision bypass bit. + // `allowDuplicateName` is its own concern — independent from the + // pipeline `force` above. The CLI maps it from + // `--allow-duplicate-name` only; `--force` and `--skills` both + // trigger pipeline re-run but never bypass the registry guard. + // The returned name is the one actually written to the registry + // (after applying the precedence chain in registerRepo) — reuse it + // so AGENTS.md / skill files reference the same name MCP clients + // will look up (#979). + const projectName = await registerRepo(repoPath, meta, { + name: options.registryName, + onRename: (previousName, nextName) => + log(`Registry name changed: "${previousName}" -> "${nextName}".`), + allowDuplicateName: options.allowDuplicateName, + // Non-primary branch runs upsert into the entry's branches[]; the + // primary/flat run (placement.branch === undefined) refreshes the + // top-level fields (#2106). + branch: placement.branch, + storagePath, + }); + + // ── #2354: the flat workspace slot has adopted this run's branch ────── + // Drop a now-shadowed `branches//` sub-index for the same label + // (unreachable once the flat slot serves it) and align the registry's + // top-level branch label. Best-effort (#2364 review F5): the index is + // complete and registered, and a failure + // here leaves only a stale registry label / undeleted shadowed dir — + // never wrong routing, because the flat meta this run already stamped is + // what applyBranchScope trusts. Retried by the next content-changing run + // (same-commit fast-path runs skip it: their guard compares the + // already-stamped meta label). + if (!placement.branch && branchLabel) { + try { + await adoptFlatBranchLabel(repoPath, branchLabel, storagePath); + } catch (e) { + log( + `Warning: could not sync the workspace branch label (${(e as Error).message}); continuing.`, + ); + } + } + + // ── Generate AI context files (best-effort) ─────────────────────── + let aggregatedClusterCount = 0; + if (pipelineResult.communityResult?.communities) { + const groups = new Map(); + for (const c of pipelineResult.communityResult.communities) { + const label = c.heuristicLabel || c.label || 'Unknown'; + groups.set(label, (groups.get(label) || 0) + c.symbolCount); + } + aggregatedClusterCount = Array.from(groups.values()).filter((count) => count >= 5).length; + } + + // Only (re)generate the repo-root AI context files (AGENTS.md / CLAUDE.md / + // skills) for the primary/flat index (#2106). A non-primary branch analyze + // must not churn the repo's committed AGENTS.md with branch-specific stats. + if (!placement.branch) { + try { + await generateAIContextFiles( + repoPath, + storagePath, + projectName, + { + files: pipelineResult.totalFileCount, + nodes: stats.nodes, + edges: stats.edges, + communities: + pipelineResult.communityResult?.stats.totalCommunities ?? + existingMeta?.stats?.communities, + clusters: aggregatedClusterCount, + processes: + pipelineResult.processResult?.stats.totalProcesses ?? existingMeta?.stats?.processes, + }, + undefined, + { + skipAgentsMd: options.skipAgentsMd, + skipSkills: options.skipSkills, + noStats: options.noStats, + defaultBranch: options.defaultBranch, + hasPdg: options.pdg === true, + hasSpringActuator: options.springActuatorPath !== undefined, + }, + ); + } catch { + // Best-effort — don't fail the entire analysis for context file issues + } + } // Persist the incremental parse cache only after a successful graph // publish (#3153). try/catch so a cache-write failure never breaks an @@ -5227,7 +5426,13 @@ async function runFullAnalysisInner( // rethrow below is the surface, and the lock's sweep remains the backstop. if (useAtomicSwap && buildPath !== lbugPath) { try { - await wipeLbugDbFiles(buildPath); + const recovery = resolveEmbeddingRecovery( + metaDir, + (await loadMeta(metaDir))?.embeddingCheckpoint, + ); + // Both paths belong to this locked slot. The validated generation + // basename identifies the same file even through a directory alias. + if (recovery?.stagingFile !== path.basename(buildPath)) await wipeLbugDbFiles(buildPath); } catch { /* swallow — orphan reclamation must never mask the real failure */ } @@ -5239,6 +5444,12 @@ async function runFullAnalysisInner( // IndexLockTimeoutError and other domain failures with `instanceof`. recordLiveIndexMutationRisk(err); } + if (/max(?:imum)?(?: database| db)? size|database size limit|maxDBSize/i.test(String(err))) { + log( + 'The database size limit was reached. Set GITNEXUS_LBUG_MAX_DB_SIZE to a larger ' + + 'byte limit before retrying analyze; retained complete embeddings can be reused.', + ); + } throw err; } } diff --git a/gitnexus/src/core/search/bm25-index.ts b/gitnexus/src/core/search/bm25-index.ts index 8aaa279ef..c105a62a3 100644 --- a/gitnexus/src/core/search/bm25-index.ts +++ b/gitnexus/src/core/search/bm25-index.ts @@ -47,6 +47,8 @@ export interface FTSSearchResponse { * which only does so when every table failed). */ nonBenignErrors?: string[]; + /** Configured table.index names that could not be queried because their index is missing. */ + missingIndexes?: string[]; } /** @@ -142,6 +144,7 @@ export const searchFTSFromLbug = async ( const resultsByIndex: any[][] = []; let queriesSucceeded = 0; const nonBenignErrors: string[] = []; + const missingIndexes: string[] = []; const ftsExtension = getExtensionCapabilities().find((c) => c.name === 'fts'); if (ftsExtension && !ftsExtension.loaded) { @@ -171,24 +174,28 @@ export const searchFTSFromLbug = async ( if (outcome.rows) { queriesSucceeded++; resultsByIndex.push(outcome.rows); - } else if (!outcome.benign) { + } else if (outcome.benign) { + missingIndexes.push(`${table}.${indexName}`); + } else { nonBenignErrors.push(redactPaths(outcome.message ?? 'Unknown FTS query error')); } } } else { // Use core lbug adapter (CLI / pipeline context) — also sequential for safety. - // tri-review Residual-1: `queryFTS` itself only swallows a genuinely-missing - // index (via the SAME classifyFtsQueryError this module re-exports); a - // missing-table or real query error rethrows here — track it the same way - // the MCP pool path does instead of a bare `catch {}` that dropped it. + // Opt into missing-index propagation so absent indexes cannot masquerade + // as successful zero-match queries. Keep core/pool classification identical. for (const { table, indexName } of FTS_INDEXES) { try { - const result = await queryFTS(table, indexName, searchQuery, limit, false); + const result = await queryFTS(table, indexName, searchQuery, limit, false, 'throw'); queriesSucceeded++; resultsByIndex.push(result); } catch (e) { const message = e instanceof Error ? e.message : String(e); - nonBenignErrors.push(redactPaths(message)); + if (classifyFtsQueryError(message) === 'missing-index') { + missingIndexes.push(`${table}.${indexName}`); + } else { + nonBenignErrors.push(redactPaths(message)); + } } } } @@ -234,5 +241,6 @@ export const searchFTSFromLbug = async ( })), ftsAvailable, ...(nonBenignErrors.length > 0 && { nonBenignErrors }), + ...(missingIndexes.length > 0 && { missingIndexes }), }; }; diff --git a/gitnexus/src/core/search/fts-indexes.ts b/gitnexus/src/core/search/fts-indexes.ts index 2070fadf3..5f8b59f13 100644 --- a/gitnexus/src/core/search/fts-indexes.ts +++ b/gitnexus/src/core/search/fts-indexes.ts @@ -110,19 +110,21 @@ export const ftsDegradedWarning = ( }; /** - * Warning for when the FTS extension is loaded and indexes exist, but every - * configured table's query failed for a real, non-benign reason (timeout, - * connection reset, native fault) — as opposed to `ftsDegradedWarning`'s - * missing-index case. `--repair-fts` will not fix a query/connection error, - * so this deliberately does NOT suggest it: reusing the missing-index - * message here would reproduce, for this cause, the exact misleading - * "run --repair-fts" guidance #2767 itself was about (tri-review NEW-1). + * Warning when no FTS query succeeded and at least one failed for a real, + * non-benign reason (timeout, connection reset, native fault). `--repair-fts` + * will not fix those errors. If indexes are also missing, the caller composes + * their repair guidance separately; do not deny that additional failure cause. */ -export const ftsQueryFailedWarning = (context: FtsWarningContext): string => +export const ftsQueryFailedWarning = ( + context: FtsWarningContext, + hasMissingIndexes = false, +): string => 'FTS keyword search failed — every configured index query returned an error' + (context.lastErrorRedacted ? ` (${context.lastErrorRedacted})` : '') + - '; results do not include keyword matches. This is not a missing-index ' + - 'condition — see server logs for details.' + + '; results do not include keyword matches. ' + + (hasMissingIndexes + ? 'See server logs for query error details.' + : 'This is not a missing-index condition — see server logs for details.') + ` (resolved: ${formatResolvedSuffix(context)})`; // Stemmers shipped by the LadybugDB FTS extension. Mirrors the lowercase token diff --git a/gitnexus/src/core/staleness-status.ts b/gitnexus/src/core/staleness-status.ts index 97a8cce94..40f9d3b51 100644 --- a/gitnexus/src/core/staleness-status.ts +++ b/gitnexus/src/core/staleness-status.ts @@ -18,21 +18,35 @@ * provably stale index indistinguishable from a fresh one. `status` is the * additive channel that separates them for a caller that wants to act on it: * - * - `current` — the index is at HEAD: `rev-list` answered 0, or it could not - * answer but HEAD alone resolved to the indexed commit. - * - `behind` — `rev-list` answered N > 0; `commitsBehind` is N. - * - `diverged` — `rev-list` could not answer, but HEAD resolved and is not the - * indexed commit. The index is provably not at HEAD; only the count is - * unknown. A branch-pinned `serve` clone reaches this once git prunes the - * commit a failed re-index left behind — the pinned update is a - * `fetch --depth 1`, which orphans it — and a rewritten history reaches it - * directly. It is the rule the Claude hook already applies: - * HEAD !== lastCommit. - * - `unknown` — HEAD could not be resolved at all: not a git repository, git - * timed out, or no commit was recorded. + * The successful probe is `rev-list --left-right --count lastCommit...HEAD`: + * the left count is indexed-only commits, and the right is HEAD-only commits. + * Both counts come from one HEAD snapshot, without a follow-up process. * - * `isStale` and `commitsBehind` keep their historical values in every case, so - * no existing consumer changes behaviour unless it reads `status`. + * - `current` — both counts are 0, or `rev-list` could not answer but a + * fallback `rev-parse HEAD` resolved to the indexed commit. + * - `behind` — the right count is N > 0; `commitsBehind` is N, including + * when the left count is positive too (divergent or shallow history). + * - `diverged` — the index is provably not at HEAD, reached two different ways: + * - The left count is positive and the right is 0: HEAD is an ancestor + * of the indexed commit (#3127). The working tree checked out an older + * commit than the one indexed, or a release branch behind the indexed + * tip. The mismatch is established: `isStale` is `true` and + * `commitsBehind` stays 0 (there is no forward count to report). + * - `rev-list` could not answer at all, but HEAD resolved and is not the + * indexed commit: only the count is unknown. A branch-pinned `serve` + * clone reaches this once git prunes the commit a failed re-index left + * behind — the pinned update is a `fetch --depth 1`, which orphans it — + * and a rewritten history reaches it directly. This arm keeps the + * historical fail-open `isStale: false` (see below). + * - `unknown` — the probe could not establish the relationship: no readable + * HEAD, a timeout, no recorded commit, or malformed count output. + * + * `isStale` and `commitsBehind` keep their historical fail-open values + * (`false` / `0`) whenever the check could not fully answer — every `unknown`, + * and the `rev-list`-failure arm of `diverged` — so no existing consumer + * changes behaviour there unless it reads `status`. The other arm of + * `diverged` (the confirmed rollback) is a successful, computed + * answer rather than a failure, so `isStale` reflects it (`true`) instead. */ export type StalenessStatus = 'current' | 'behind' | 'diverged' | 'unknown'; diff --git a/gitnexus/src/mcp/local/aop-metadata.ts b/gitnexus/src/mcp/local/aop-metadata.ts index a3506d9c5..d5d41ab02 100644 --- a/gitnexus/src/mcp/local/aop-metadata.ts +++ b/gitnexus/src/mcp/local/aop-metadata.ts @@ -1,4 +1,9 @@ import { executeParameterized } from '../../core/lbug/pool-adapter.js'; +import { + assertSymbolIdentity, + assertIdentityFields, + rethrowSymbolIdentityError, +} from './query-result-integrity.js'; import { decodeSpringAopReason, type SpringAopReason, @@ -151,6 +156,7 @@ const DETERMINISTIC_RELATIONSHIP_ORDER = 'ORDER BY sourceId, targetId, reason, s * shared decoder. Other DECLARES edges (for example Spring Bean factories) * and malformed/forward-version evidence are ignored. Query failures are * fail-soft because older or partially upgraded indexes must remain readable. + * Corrupt identities propagate to the context/impact integrity error boundary. */ export async function querySpringAopMetadata( lbugPath: string, @@ -204,6 +210,24 @@ export async function querySpringAopMetadata( ), ]); + for (const rows of [ + outgoingAdviceRows, + incomingAdviceRows, + outgoingPointcutRows, + incomingPointcutRows, + ]) { + for (const row of rows) { + assertSymbolIdentity(readRowValue(row, 'sourceId', 0)); + assertSymbolIdentity(readRowValue(row, 'targetId', 3)); + assertIdentityFields( + readRowValue(row, 'sourceName', 1), + readRowValue(row, 'sourceFilePath', 2), + readRowValue(row, 'targetName', 4), + readRowValue(row, 'targetFilePath', 5), + ); + } + } + const behaviors: SpringAopBehaviorMetadata[] = []; const advices: SpringAopAdviceMetadata[] = []; const resolvedPointcuts: SpringAopResolvedPointcutMetadata[] = []; @@ -346,7 +370,8 @@ export async function querySpringAopMetadata( resolvedPointcuts: dedupedResolvedPointcuts, unresolvedPointcuts: dedupedPointcuts, }; - } catch { + } catch (error) { + rethrowSymbolIdentityError(error); return undefined; } } diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 71864e674..e5dc607f1 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -10,7 +10,12 @@ import { resolveGraphPath } from '../../storage/shared-store.js'; import fs from 'fs/promises'; import path from 'path'; import { createHash } from 'crypto'; -import { scoreImpactRisk, unusedAxesForImpactWalk, type ImpactRiskResult } from 'gitnexus-shared'; +import { + scoreImpactRisk, + unusedAxesForImpactWalk, + getLanguageFromFilename, + type ImpactRiskResult, +} from 'gitnexus-shared'; import { initLbug, executeQuery, @@ -23,6 +28,15 @@ import { } from '../../core/lbug/pool-adapter.js'; import { queryClassBeanMetadata } from './bean-metadata.js'; import { querySpringAopMetadata } from './aop-metadata.js'; +import { + SYMBOL_IDENTITY_RECOVERY_SUGGESTION, + SymbolIdentityError, + assertSymbolIdentity, + queryRowValue, + assertIdentityFields, + assertQueryIdentity, + rethrowSymbolIdentityError, +} from './query-result-integrity.js'; import { queryConvexDispatchMetadata } from './convex-metadata.js'; import { isValidQueryParams } from '../../core/lbug/query-params.js'; import { toDisplayLine } from './line-display.js'; @@ -30,8 +44,10 @@ import { shapeQueryProcessAttaches } from './query-process-attaches.js'; import { LBUG_ID_PROBE_BATCH_SIZE, LBUG_QUERY_BATCH_SIZE } from '../../core/lbug/query-batch.js'; import { chunk, mapConcurrent } from '../../lib/utils.js'; import { pathSuffixOf } from './path-predicate.js'; +import { isCobolFile, isJclFile } from '../../core/ingestion/cobol/file-types.js'; import { toOneBasedLine } from '../../core/ingestion/utils/line-base.js'; import { isTestFilePath } from '../../core/ingestion/utils/test-file-path.js'; +import { isTemplateRouteCandidate } from '../../core/ingestion/utils/template-file.js'; import { isWalCorruptionError, WAL_RECOVERY_SUGGESTION } from '../../core/lbug/lbug-config.js'; // Embedding imports are lazy (dynamic import) to avoid loading onnxruntime-node // at MCP server startup — crashes on unsupported Node ABI versions (#89) @@ -317,6 +333,70 @@ function nonBlankUid(value: unknown): string | undefined { return typeof value === 'string' ? value.trim() || undefined : undefined; } +function assertSymbolRowIdentity(row: unknown): void { + assertQueryIdentity(row, 'id', 0, [ + ['name', 1], + ['type', 2], + ['filePath', 3], + ]); +} + +function assertContextRefs(rows: unknown[]): void { + for (const row of rows) { + assertQueryIdentity(row, 'uid', 1, [ + ['name', 2], + ['filePath', 3], + ['kind', 4], + ]); + assertSymbolIdentity(queryRowValue(row, 'relType', 0)); + } +} + +const RESPONSE_IDENTITY_FIELDS = new Set([ + 'id', + 'uid', + 'name', + 'filePath', + 'label', + 'kind', + 'type', + 'relationType', + 'processType', + 'url', + 'method', + 'sourceId', + 'targetId', + 'symbolId', + 'symbolName', + 'symbolFilePath', + 'adviceId', + 'adviceName', + 'adviceFilePath', + 'advisedId', + 'advisedName', + 'advisedFilePath', + 'evidenceId', +]); + +/** Cover nested additive identity fields while leaving source/metadata text alone. */ +function assertResponseIdentities(value: unknown): void { + if (Array.isArray(value)) { + for (const item of value) assertResponseIdentities(item); + } else if (value !== null && typeof value === 'object') { + for (const [key, field] of Object.entries(value)) { + if (key === 'seedBlocks' || key === 'reachableBlocks' || key === 'intraReachableBlocks') { + if (!Array.isArray(field)) throw new SymbolIdentityError(); + for (const id of field) assertSymbolIdentity(id); + continue; + } + if (RESPONSE_IDENTITY_FIELDS.has(key)) assertIdentityFields(field); + if (key !== 'content' && key !== 'methodMetadata' && key !== 'bean') { + assertResponseIdentities(field); + } + } + } +} + interface StringAliasDefinition { canonical: string; aliases: readonly string[]; @@ -3143,6 +3223,7 @@ export class LocalBackend { // regardless of whether OTHER tables succeeded — previously a real error // on N-1 of N tables while one succeeded left zero diagnostic trail. const ftsQueryErrors = bm25SearchResult?.nonBenignErrors; + const ftsMissingIndexes = bm25SearchResult?.missingIndexes; if (ftsQueryErrors) { // tri-review NEW-5: these strings are already classified non-benign by // classifyFtsQueryError — do NOT route them through logQueryError, @@ -3632,13 +3713,15 @@ export class LocalBackend { branch: repo.branch, indexedAt: this.lastObservedPoolState.get(repo.lbugPath)?.indexedAt ?? repo.indexedAt, }; - // tri-review NEW-1: every table failing for a REAL error (timeout, - // connection reset) is not a missing-index condition — `ftsDegradedWarning`'s - // "run --repair-fts" headline won't fix it. Route to a dedicated message - // instead of burying the real cause as a trailing suffix on bad advice. + // Real errors (timeout, connection reset) need their own diagnosis. + // When some indexes are also missing, preserve both causes and append + // their repair guidance below even though no FTS query succeeded. warnings.push( ftsQueryErrors - ? ftsQueryFailedWarning({ ...warningContext, lastErrorRedacted: ftsQueryErrors[0] }) + ? ftsQueryFailedWarning( + { ...warningContext, lastErrorRedacted: ftsQueryErrors[0] }, + !!ftsMissingIndexes?.length, + ) : ftsDegradedWarning(warningContext, ftsDisabledReason), ); } else if (ftsQueryErrors) { @@ -3651,6 +3734,12 @@ export class LocalBackend { `FTS keyword search partially failed — ${ftsQueryErrors.length} of the configured indexes hit a query error and were skipped; results may be missing matches from those node types (see server logs).`, ); } + if (ftsMissingIndexes?.length && (ftsUsed || ftsQueryErrors)) { + warnings.push( + `FTS keyword search is incomplete: missing configured indexes (${ftsMissingIndexes.join(', ')}). ` + + 'Results may be missing matches from those node types. Run `gitnexus analyze --repair-fts`.', + ); + } // #2331: a CJK query against a server process resolving // GITNEXUS_FTS_CJK_SEGMENTATION to 'none' silently misses sub-phrase // matches with no other signal — this is the only place an agent driving @@ -3782,7 +3871,7 @@ export class LocalBackend { // #2767: a partial FTS failure (some tables ok, one or more real errors) // is as much a "results may be incomplete" signal as enrichmentDegraded — // flag it the same way rather than only via the warning string. - const ftsPartial = ftsUsed && !!ftsQueryErrors; + const ftsPartial = ftsUsed && (!!ftsQueryErrors || !!ftsMissingIndexes?.length); return { processes, @@ -3803,7 +3892,12 @@ export class LocalBackend { query: string, limit: number, disabledReason?: FtsDisabledReason, - ): Promise<{ results: any[]; ftsUsed: boolean; nonBenignErrors?: string[] }> { + ): Promise<{ + results: any[]; + ftsUsed: boolean; + nonBenignErrors?: string[]; + missingIndexes?: string[]; + }> { if (disabledReason) return { results: [], ftsUsed: false }; let searchFTSFromLbug; try { @@ -3837,6 +3931,7 @@ export class LocalBackend { const bm25Results = ftsResponse?.results ?? []; const ftsUsed = ftsResponse?.ftsAvailable ?? false; const nonBenignErrors = ftsResponse?.nonBenignErrors; + const missingIndexes = ftsResponse?.missingIndexes; const results: any[] = []; @@ -3910,7 +4005,12 @@ export class LocalBackend { } } - return { results, ftsUsed, ...(nonBenignErrors && { nonBenignErrors }) }; + return { + results, + ftsUsed, + ...(nonBenignErrors && { nonBenignErrors }), + ...(missingIndexes && { missingIndexes }), + }; } /** @@ -4344,9 +4444,10 @@ export class LocalBackend { * "unknown kind" and, worse, makes the `kind` disambiguation hint unable to * filter it out (#2687). * - * Failures are swallowed: label enrichment is an optimisation for + * Ordinary query failures are swallowed: label enrichment is an optimisation for * downstream scoring and #480 Class/Interface BFS seeding; if it fails * the symbol still resolves, just without the kind-priority bonus. + * Corrupt identities propagate to the context/impact error envelope. */ private async enrichCandidateLabels( repo: RepoHandle, @@ -4380,6 +4481,7 @@ export class LocalBackend { ); const labelById = new Map(); for (const r of rows as any[]) { + assertQueryIdentity(r, 'id', 0, [['label', 1]]); const id = (r.id ?? r[0]) as string; const label = (r.label ?? r[1]) as string; if (id && label && !labelById.has(id)) labelById.set(id, label); @@ -4387,7 +4489,8 @@ export class LocalBackend { for (const c of candidates) { if (c.type === '' && labelById.has(c.id)) c.type = labelById.get(c.id) as string; } - } catch { + } catch (error) { + rethrowSymbolIdentityError(error); /* best-effort — downstream resolvers still work without the label */ } } @@ -4512,6 +4615,7 @@ export class LocalBackend { { uid }, ); if (rows.length === 0) return { kind: 'not_found' }; + assertSymbolRowIdentity(rows[0]); const r = rows[0] as any; const symbol = { id: (r.id ?? r[0]) as string, @@ -4522,6 +4626,7 @@ export class LocalBackend { endLine: (r.endLine ?? r[5]) as number, ...(include_content ? { content: (r.content ?? r[6]) as string | undefined } : {}), }; + assertSymbolIdentity(symbol.id, uid); // Same LadybugDB label-enrichment as the name-based path: a UID // pointing at a Class must still surface `type: 'Class'` so impact's // Class/Interface BFS seed fires. No-op when type is already set. @@ -4645,6 +4750,10 @@ export class LocalBackend { if (rows.length === 0) return { kind: 'not_found' }; + // Reject every raw candidate before narrowing/scoring can hide a corrupt row. + for (const row of rows) { + assertSymbolRowIdentity(row); + } // Normalise row shape across object / tuple returns from LadybugDB. let normalized = rows.map((r: any) => ({ id: (r.id ?? r[0]) as string, @@ -4655,7 +4764,6 @@ export class LocalBackend { endLine: (r.endLine ?? r[5]) as number, ...(include_content ? { content: (r.content ?? r[6]) as string | undefined } : {}), })); - // An exact File path wins over anchored suffix candidates. Without this, // `lib/a.ts` and `src/lib/a.ts` both score as File candidates and turn an // otherwise unambiguous exact target into `ambiguous` (#3084 review P2). @@ -4805,9 +4913,14 @@ export class LocalBackend { }, ): Promise { try { - return await this._contextImpl(repo, params); + const result = await this._contextImpl(repo, params); + if (!result.error) assertResponseIdentities(result); + return result; } catch (err: any) { const msg = (err instanceof Error ? err.message : String(err)) || 'Context query failed'; + if (err instanceof SymbolIdentityError) { + return { error: msg, recoverySuggestion: SYMBOL_IDENTITY_RECOVERY_SUGGESTION }; + } if (isWalCorruptionError(err)) { return { error: msg, @@ -4922,6 +5035,8 @@ export class LocalBackend { { symId }, ), ]); + assertContextRefs(incomingRows); + assertContextRefs(incomingAdvisedRows); incomingRows.push(...incomingAdvisedRows); let typedPropertyRows: any[] = []; @@ -5026,6 +5141,16 @@ export class LocalBackend { }, ), ]); + assertContextRefs(ctorIncoming); + assertContextRefs(fileIncoming); + assertContextRefs(typedPropertyIncoming); + for (const row of typedProperties) { + assertQueryIdentity(row, 'uid', 0, [ + ['name', 1], + ['filePath', 2], + ['kind', 3], + ]); + } typedPropertyRows = typedProperties; // Deduplicate by (relType, uid) — a caller can have multiple relation @@ -5042,6 +5167,7 @@ export class LocalBackend { } } } catch (e) { + rethrowSymbolIdentityError(e); logQueryError('context:class-incoming-expansion', e); } } @@ -5072,6 +5198,8 @@ export class LocalBackend { { symId }, ), ]); + assertContextRefs(outgoingRows); + assertContextRefs(outgoingAdvisedRows); outgoingRows.push(...outgoingAdvisedRows); // Process participation. @@ -5095,7 +5223,14 @@ export class LocalBackend { `, { symId }, ); + for (const row of processRows) { + assertQueryIdentity(row, 'pid', 0, [ + ['label', 1], + ['entryPointId', 4], + ]); + } } catch (e) { + rethrowSymbolIdentityError(e); logQueryError('context:process-participation', e); } @@ -5132,6 +5267,7 @@ export class LocalBackend { // (GET/POST pair). URL-only dedup would drop the second endpoint. const seenRoutes = new Set(); for (const r of routeRows) { + assertIdentityFields(queryRowValue(r, 'url', 0), queryRowValue(r, 'method', 1)); const url = r.url ?? r[0]; const method = r.method ?? r[1]; const dedupKey = routeEnrichmentKey(method ? String(method) : undefined, url); @@ -5141,7 +5277,8 @@ export class LocalBackend { } } } catch (e) { - // Best-effort enrichment — never fail the context call. + rethrowSymbolIdentityError(e); + // Ordinary query failures leave this best-effort enrichment unavailable. logQueryError('context:route-lookup', e); } @@ -5191,6 +5328,7 @@ export class LocalBackend { ); const beanMetadataPromise = queryClassBeanMetadata(repo.lbugPath, symId, epistemicSymType); const aopMetadataPromise = querySpringAopMetadata(repo.lbugPath, symId, epistemicSymType); + void aopMetadataPromise.catch(() => undefined); // R3-1. A `Property` whose name the analyzer declined to link — because // every definition of it lives in another language — otherwise returns an @@ -5285,6 +5423,7 @@ export class LocalBackend { try { chain = await this._computeContextChain(repo, symId, requestedDepth); } catch (e) { + rethrowSymbolIdentityError(e); logQueryError('context:chain-bfs', e); } } @@ -5325,8 +5464,8 @@ export class LocalBackend { processes: processRows.map((r: any) => ({ id: r.pid || r[0], name: r.label || r[1], - step_index: r.step || r[2], - step_count: r.stepCount || r[3], + step_index: r.step ?? r[2], + step_count: r.stepCount ?? r[3], })), }; } @@ -5407,6 +5546,13 @@ export class LocalBackend { visited: Array.from(visited), }); if (rows.length === 0) return { nextFrontier: [] }; + for (const row of rows) { + assertQueryIdentity(row, 'uid', 0, [ + ['name', 1], + ['filePath', 2], + ['kind', 3], + ]); + } // MATCH is one row per CALLS edge. Cypher `WITH DISTINCT` applies // LIMIT 50 to unique neighbors; this second pass still collapses // twins if a driver/engine ever returns duplicate rows. @@ -5426,6 +5572,7 @@ export class LocalBackend { for (const r of fresh) visited.add(r.uid); return { nodes, nextFrontier: fresh.map((r: any) => r.uid) }; } catch (e) { + rethrowSymbolIdentityError(e); logQueryError(logLabel, e); return { nextFrontier: [] }; } @@ -6445,6 +6592,7 @@ export class LocalBackend { // throwaway arrays the size of the row set (40k rows 11.4ms → 4.5ms, 200k // rows 71.3ms → 26.6ms). const exactlyMatchedPaths = new Set(); + const mappedPaths = new Set(); for (const row of symbolRows) { if (row.filePath === row.diffPath) exactlyMatchedPaths.add(row.diffPath); } @@ -6454,6 +6602,8 @@ export class LocalBackend { if (sym.filePath !== sym.diffPath && exactlyMatchedPaths.has(diffPath)) continue; const hunks = hunksByPath.get(diffPath) ?? []; if (!hunksOverlapRange(hunks, sym.startLine, sym.endLine)) continue; + // A suffix fallback is a hint, not proof that this is the changed file. + if (sym.filePath === diffPath) mappedPaths.add(diffPath); if (changedSymbols.has(sym.id)) continue; changedSymbols.set(sym.id, { @@ -6465,6 +6615,27 @@ export class LocalBackend { }); } + // An empty successful query cannot prove a source diff is safe: its rows + // may be missing, outside indexed spans, or not yet indexed. Keep ordinary + // docs/config diffs measurable, but withhold a ranked source-risk verdict. + const isSourceFile = (file: string): boolean => + getLanguageFromFilename(file) !== null || + isCobolFile(file) || + isJclFile(file) || + isTemplateRouteCandidate(file); + const unmappedFiles = [ + ...new Set( + fileDiffs + .filter( + ({ filePath, oldFilePath }) => + !mappedPaths.has(filePath) && + (isSourceFile(filePath) || (oldFilePath !== undefined && isSourceFile(oldFilePath))), + ) + .map(({ filePath }) => filePath), + ), + ]; + if (unmappedFiles.length > 0) queryDegraded = true; + // Find affected processes -- batched queries instead of N+1 const affectedProcesses = new Map(); if (changedSymbols.size > 0) { @@ -6565,8 +6736,9 @@ export class LocalBackend { }, changed_symbols: listedSymbols, affected_processes: Array.from(affectedProcesses.values()), - // A swallowed query failure makes the counts/risk above incomplete — tell - // the caller so the safety gate isn't trusted as a clean result (#2283). + ...(unmappedFiles.length > 0 && { unmapped_files: unmappedFiles }), + // Failed queries or unmapped source files leave counts/risk incomplete; + // the safety gate must not treat that as a clean result (#2283). ...(queryDegraded && { partial: true }), ...(listedSymbols.length < changedSymbols.size && { truncated: true }), }; @@ -7082,13 +7254,23 @@ export class LocalBackend { private async impact(repo: RepoHandle, params: ImpactParams): Promise { try { - return await this._impactImpl(repo, params); + const result = await this._impactImpl(repo, params); + if (!result.error) assertResponseIdentities(result); + return result; } catch (err: any) { // Return structured error instead of crashing (#321) const message = (err instanceof Error ? err.message : String(err)) || 'Impact analysis failed'; - const suggestion = 'The graph query failed — try gitnexus context as a fallback'; - const recoverySuggestion = isWalCorruptionError(err) ? WAL_RECOVERY_SUGGESTION : undefined; + const recoverySuggestion = + err instanceof SymbolIdentityError + ? SYMBOL_IDENTITY_RECOVERY_SUGGESTION + : isWalCorruptionError(err) + ? WAL_RECOVERY_SUGGESTION + : undefined; + const suggestion = + err instanceof SymbolIdentityError + ? SYMBOL_IDENTITY_RECOVERY_SUGGESTION + : 'The graph query failed — try gitnexus context as a fallback'; if (params.mode === 'pdg') { // Symbol resolution never reached the catch with a resolved symbol (the // throw can originate before/within resolution), so the envelope carries @@ -7392,6 +7574,7 @@ export class LocalBackend { } catch (e) { probeFailed = true; candidateProbeFailed = true; + rethrowSymbolIdentityError(e); logQueryError('impact:ambiguous-candidate', e); } return { @@ -7649,6 +7832,7 @@ export class LocalBackend { }); return composeUnifiedPdgImpactResult(pdgResult, interproceduralResult); } catch (e) { + rethrowSymbolIdentityError(e); logQueryError('impact:pdg-interprocedural-reach', e); return composeUnifiedPdgImpactResult(pdgResult, null, e); } @@ -7686,10 +7870,12 @@ export class LocalBackend { { ids: blockIds }, ); for (const r of rows as any[]) { + assertIdentityFields(r.callees ?? r[0]); const raw = String(r.callees ?? r[0] ?? ''); for (const n of raw.split(' ')) if (n) names.add(n); } } catch (e) { + rethrowSymbolIdentityError(e); logQueryError('impact:pdg-slice-callees', e); } return names; @@ -7725,6 +7911,7 @@ export class LocalBackend { for (const id of splitCalleeIds(r.calleeIds ?? r[0])) ids.add(id); } } catch (e) { + rethrowSymbolIdentityError(e); logQueryError('impact:pdg-slice-callee-ids', e); } return ids; @@ -7878,7 +8065,10 @@ export class LocalBackend { ORDER BY id LIMIT 25`, { symId, heritage: HERITAGE_TYPES }, - ).catch(() => []); + ).catch((error) => { + rethrowSymbolIdentityError(error); + return []; + }); const undecidedSummary = meta?.undecidedInterfaceSatisfaction; const undecidedDrops = undecidedSummary === undefined @@ -7917,6 +8107,10 @@ export class LocalBackend { } const ifaceRows = await interfaceRowsPromise; for (const r of ifaceRows) { + assertQueryIdentity(r, 'id', 0, [ + ['name', 1], + ['label', 2], + ]); const id = (r.id ?? r[0]) as string; if (id && !boundary.has(id)) { boundary.set(id, { @@ -7942,7 +8136,10 @@ export class LocalBackend { WHERE iface.id = $ifaceId AND r.type IN $types RETURN COUNT(DISTINCT other.id) AS cnt`, { ifaceId, types }, - ).catch(() => []); + ).catch((error) => { + rethrowSymbolIdentityError(error); + return []; + }); const cnt = rows.length > 0 ? Number((rows[0] as any).cnt ?? (rows[0] as any)[0] ?? 0) : 0; m.set(ifaceId, cnt); @@ -8001,7 +8198,8 @@ export class LocalBackend { callableValueReferences: droppedBoundaries.callableValueReferences, }, }; - } catch { + } catch (error) { + rethrowSymbolIdentityError(error); // Never let the heritage probe's failure suppress a drop we already know // about — the whole point is that silence must not read as certainty. return epistemicFrom(droppedBoundaries); @@ -8093,6 +8291,7 @@ export class LocalBackend { `Impact target '${sym.name || sym[1] || '?'}' resolved without a node id; refusing to report a blast radius`, ); } + assertSymbolRowIdentity(sym); // #1858 — kick off the epistemic boundary probe concurrently with the BFS. // It depends only on symId/symType/symName (all known now) and touches no @@ -8128,6 +8327,7 @@ export class LocalBackend { opts.skipEpistemic || summaryOnly ? Promise.resolve(undefined) : querySpringAopMetadata(repo.lbugPath, symId, symType); + void aopMetadataPromise.catch(() => undefined); const impacted: any[] = []; const visited = new Set([symId]); const pdgBridgeEvidenceById = new Map(); @@ -8172,6 +8372,7 @@ export class LocalBackend { ]); for (const r of ctorRows) { + assertSymbolRowIdentity(r); const rid = r.id || r[0]; if (rid && !visited.has(rid)) { visited.add(rid); @@ -8179,6 +8380,7 @@ export class LocalBackend { } } for (const r of fileRows) { + assertSymbolRowIdentity(r); const rid = r.id || r[0]; if (rid && !visited.has(rid)) { visited.add(rid); @@ -8203,6 +8405,7 @@ export class LocalBackend { ); for (const r of typedPropertyRows) { + assertSymbolRowIdentity(r); const rid = r.id || r[0]; if (rid && !visited.has(rid)) { visited.add(rid); @@ -8210,6 +8413,7 @@ export class LocalBackend { } } } catch (e) { + rethrowSymbolIdentityError(e); logQueryError('impact:class-node-expansion', e); traversalComplete = false; } @@ -8247,6 +8451,9 @@ export class LocalBackend { `, { symId }, ); + for (const row of memberRows) { + assertSymbolRowIdentity(row); + } memberRows.sort((a, b) => compareCodeUnits(String(a.id ?? a[0]), String(b.id ?? b[0]))); if (memberRows.length > OBJECT_CALLABLE_MEMBER_CAP) traversalComplete = false; for (const row of memberRows.slice(0, OBJECT_CALLABLE_MEMBER_CAP)) { @@ -8266,6 +8473,7 @@ export class LocalBackend { } } } catch (e) { + rethrowSymbolIdentityError(e); logQueryError('impact:object-callable-expansion', e); traversalComplete = false; } @@ -8322,6 +8530,17 @@ export class LocalBackend { relTypes: relationTypes, ...(safeMinConfidence > 0 ? { minConfidence: safeMinConfidence } : {}), }); + // Validate before filtering/deduplication; a discarded corrupt edge is + // still evidence that this result's counts cannot be trusted. + for (const row of related) { + assertQueryIdentity(row, 'id', 1, [ + ['sourceId', 0], + ['name', 2], + ['type', 3], + ['filePath', 4], + ]); + assertSymbolIdentity(queryRowValue(row, 'relType', 5)); + } const edges: ImpactFrontierEdge[] = related.map((rel) => ({ id: rel.id || rel[1], @@ -8421,6 +8640,7 @@ export class LocalBackend { }); } } catch (e) { + rethrowSymbolIdentityError(e); logQueryError('impact:depth-traversal', e); // Break out of depth loop on query failure but return partial results // collected so far, rather than silently swallowing the error (#321) @@ -8546,6 +8766,7 @@ export class LocalBackend { `, { ids }, ).catch((err) => { + rethrowSymbolIdentityError(err); processQueryFailed = true; enrichmentDegraded = true; logQueryError('impact:process-chunk', err); @@ -8553,6 +8774,14 @@ export class LocalBackend { }); for (const row of rows) { + assertQueryIdentity(row, 'pId', 0, [ + ['name', 1], + ['processType', 2], + ['entryPointId', 3], + ['epName', 7], + ['epType', 8], + ['epFilePath', 9], + ]); const pId = row.pId ?? row[0]; const epId = row.entryPointId ?? row[3] ?? row.pId ?? row[0]; // Track mapping from process -> entryPoint so we can backfill missing minStep @@ -8601,6 +8830,7 @@ export class LocalBackend { ep.earliest_broken_step = Math.min(ep.earliest_broken_step, minStep ?? Infinity); } } catch (e) { + rethrowSymbolIdentityError(e); processQueryFailed = true; enrichmentDegraded = true; logQueryError('impact:process-chunk', e); @@ -8623,12 +8853,14 @@ export class LocalBackend { `, { pIds, ids: allImpactedIds }, ).catch((err) => { + rethrowSymbolIdentityError(err); enrichmentDegraded = true; logQueryError('impact:process-chunk-backfill', err); return []; }); for (const mr of missingRows) { + assertQueryIdentity(mr, 'pid', 0, []); const pid = mr.pid ?? mr[0]; const minStep = mr.minStep ?? mr[1]; const epId = processToEntryPoint.get(String(pid)); @@ -8640,6 +8872,7 @@ export class LocalBackend { } } } catch (e) { + rethrowSymbolIdentityError(e); enrichmentDegraded = true; logQueryError('impact:process-chunk-backfill', e); } @@ -8702,6 +8935,7 @@ export class LocalBackend { `, { ids: idsChunk }, ).catch((err) => { + rethrowSymbolIdentityError(err); moduleQueryFailed = true; enrichmentDegraded = true; logQueryError('impact:module-chunk', err); @@ -8709,12 +8943,14 @@ export class LocalBackend { }); for (const r of rows) { + assertIdentityFields(queryRowValue(r, 'name', 0)); const name = r.name ?? r[0] ?? null; const hits = (r.hits ?? r[1]) || 0; if (!name) continue; moduleHitsMap.set(name, (moduleHitsMap.get(name) || 0) + hits); } } catch (e) { + rethrowSymbolIdentityError(e); moduleQueryFailed = true; enrichmentDegraded = true; logQueryError('impact:module-chunk', e); @@ -8743,16 +8979,19 @@ export class LocalBackend { `, { ids: idsChunk }, ).catch((err) => { + rethrowSymbolIdentityError(err); enrichmentDegraded = true; moduleClassificationFailed = true; logQueryError('impact:direct-module-chunk', err); return []; }); for (const r of rows) { + assertIdentityFields(queryRowValue(r, 'name', 0)); const name = r.name ?? r[0] ?? null; if (name) directModuleSet.add(name); } } catch (e) { + rethrowSymbolIdentityError(e); enrichmentDegraded = true; moduleClassificationFailed = true; logQueryError('impact:direct-module-chunk', e); @@ -8814,11 +9053,14 @@ export class LocalBackend { `, { ids: chunkIds }, ).catch((err) => { + rethrowSymbolIdentityError(err); enrichmentDegraded = true; logQueryError('impact:route-chunk', err); return []; }); for (const row of rows) { + assertSymbolIdentity(queryRowValue(row, 'hid', 0)); + assertIdentityFields(queryRowValue(row, 'url', 1), queryRowValue(row, 'method', 2)); const hid = String(row.hid ?? row[0] ?? ''); const url = row.url ?? row[1]; if (!hid || typeof url !== 'string') continue; @@ -8975,8 +9217,16 @@ export class LocalBackend { p.processType AS pType, MIN(r.step) AS step `, { ids: chunkIds }, - ).catch(() => []); + ).catch((err) => { + rethrowSymbolIdentityError(err); + return []; + }); for (const row of rows) { + assertQueryIdentity(row, 'sid', 0, []); + assertQueryIdentity(row, 'pid', 1, [ + ['pName', 2], + ['pType', 3], + ]); const sid = row.sid ?? row[0]; if (!sid) continue; const procEntry = { @@ -8990,6 +9240,7 @@ export class LocalBackend { else perSymbolProcesses.set(String(sid), [procEntry]); } } catch (e) { + rethrowSymbolIdentityError(e); logQueryError('impact:per-symbol-process-chunk', e); } } @@ -9106,6 +9357,7 @@ export class LocalBackend { ]; try { + assertSymbolIdentity(sym.id ?? sym[0], uid); // skipPerSymbolEnrichment suppresses ONLY the per-symbol STEP_IN_PROCESS // enrichment pass while preserving byDepth. Group-mode cross-repo fan-out // may fan across many repos; the per-symbol pass adds up to MAX_CHUNKS diff --git a/gitnexus/src/mcp/local/pdg-impact.ts b/gitnexus/src/mcp/local/pdg-impact.ts index ae4e9836f..4b7321925 100644 --- a/gitnexus/src/mcp/local/pdg-impact.ts +++ b/gitnexus/src/mcp/local/pdg-impact.ts @@ -27,6 +27,11 @@ import { toDisplayLine } from './line-display.js'; import { toOneBasedLine } from '../../core/ingestion/utils/line-base.js'; import { decodeCallSummary } from '../../core/ingestion/taint/call-summary-codec.js'; import { decodeReachingDefReason } from '../../core/ingestion/cfg/reaching-def-reason-codec.js'; +import { + assertSymbolIdentity, + assertIdentityFields, + assertQueryIdentity, +} from './query-result-integrity.js'; /** * Parse the `` segment out of a `BasicBlock` id (1-based function start @@ -90,14 +95,19 @@ const INTERPROC_NODE_BUDGET = 5000; * `classifyPdgBridgeEvidence`); this is the same fact, read at the descent side. */ function parseCalleeIdsCell(raw: unknown): { ids: string[]; truncated: boolean } { + assertIdentityFields(raw); const ids: string[] = []; let truncated = false; + if (!String(raw ?? '').trim()) return { ids, truncated }; // Split on the SHARED CALLEE_ID_SEP (tab) — ids embed file paths / multi-word // C++ type tokens that can contain a space, so a space split would fragment // them. Producer (calleeIdsOfBlock) joins with the same constant. for (const id of String(raw ?? '').split(CALLEE_ID_SEP)) { if (id === CALLEES_TRUNCATED_SENTINEL) truncated = true; - else if (id) ids.push(id); + else { + assertSymbolIdentity(id); + ids.push(id); + } } return { ids, truncated }; } @@ -218,6 +228,7 @@ async function selfReachingDefEdgesByBlock( { ids: blockIds }, ); for (const r of rows as Array>) { + assertQueryIdentity(r, 'id', 0); const id = String(r['id'] ?? ''); if (!id) continue; const decoded = decodeReachingDefReason(r['reason']); @@ -297,6 +308,7 @@ async function pdgStatementsForBlocks( // Narrow the awaited rows ONCE at the boundary to a typed record shape; read // the aliased cells via bracket access with String()/Number() coercion. for (const r of rows as Array>) { + assertQueryIdentity(r, 'id', 0); const id = String(r['id'] ?? ''); const line = Number(r['line'] ?? 0); if (!id || !Number.isFinite(line) || line <= 0) continue; @@ -501,6 +513,10 @@ async function projectBlocksToSymbols(deps: { // non-aliased row shape) — no per-field `as any`, matching the typed-row // pattern used elsewhere in this file (e.g. lines ~264, ~1309, ~1386). for (const r of rows as Array>) { + assertQueryIdentity(r, 'id', 0, [ + ['name', 1], + ['label', 2], + ]); resolved.push({ id: String(r['id'] ?? r['0'] ?? ''), name: String(r['name'] ?? r['1'] ?? ''), @@ -1718,6 +1734,7 @@ async function bfsReachableBlocks(input: { // Narrow the awaited rows ONCE at the boundary (executeParameterized returns // any[]) to a typed record shape, then read the aliased `id` via bracket // access — no `as any` sprayed per field. + for (const row of rawRows) assertQueryIdentity(row, 'id', 0); const rows = rawRows.slice(0, stepLimit) as Array>; depthReached = depth + 1; if (rawRows.length > stepLimit) truncatedByLimit = true; @@ -1796,6 +1813,10 @@ async function calleeIdsByBlock( // Narrow the awaited rows ONCE at the boundary to a typed record shape; read // the aliased cells via bracket access — no per-field `as any`. for (const r of rows as Array>) { + assertQueryIdentity(r, 'id', 0, [ + ['calleeIds', 1], + ['callees', 2], + ]); const blockId = String(r['id'] ?? ''); if (!blockId) continue; // ONE pass over the cell classifies BOTH facts — a second full split just to @@ -1877,6 +1898,7 @@ async function calleesWithReturnFlow( { ids: calleeIds }, ); for (const r of rows as Array>) { + assertQueryIdentity(r, 'id', 0); const id = String(r['id'] ?? ''); if (!id) continue; const decoded = decodeCallSummary(r['reason']); @@ -1931,6 +1953,7 @@ async function resolveCalleeSpans( // the aliased columns via bracket access with Number()/String() coercion — // no per-field `as any` (the same boundary-narrowing the typed helpers use). for (const r of rows as Array>) { + assertQueryIdentity(r, 'id', 0, [['filePath', 1]]); const id = String(r['id'] ?? ''); const filePath = String(r['filePath'] ?? ''); const startLine = Number(r['startLine']); @@ -2168,6 +2191,7 @@ async function interproceduralDescent(input: { seedBlockQuery(anchorClause, probeLimit), queryParams, ); + for (const row of rawSeedRows) assertQueryIdentity(row, 'id', 0); const exceeded = rawSeedRows.length > stepLimit; const seeds = rawSeedRows .slice(0, stepLimit) @@ -2353,6 +2377,7 @@ export async function runImpactPDG(deps: RunPdgImpactDeps): Promise>; let seedBlocks: string[] = seedRows .map((r) => String(r['id'] ?? '')) diff --git a/gitnexus/src/mcp/local/query-result-integrity.ts b/gitnexus/src/mcp/local/query-result-integrity.ts new file mode 100644 index 000000000..da92cca55 --- /dev/null +++ b/gitnexus/src/mcp/local/query-result-integrity.ts @@ -0,0 +1,57 @@ +/** Shared integrity boundary for identities returned by impact/context queries. */ +export const SYMBOL_IDENTITY_RECOVERY_SUGGESTION = + 'Run gitnexus analyze --force from the affected repository root to rebuild the index.'; + +export class SymbolIdentityError extends Error { + constructor() { + super('The index returned an invalid symbol identity. ' + SYMBOL_IDENTITY_RECOVERY_SUGGESTION); + this.name = 'SymbolIdentityError'; + } +} + +/** Validate database identities before using them as graph traversal anchors. */ +export function assertSymbolIdentity(id: unknown, expectedUid?: string): asserts id is string { + if ( + typeof id !== 'string' || + !id.trim() || + id.includes('\0') || + (expectedUid !== undefined && id !== expectedUid) + ) { + throw new SymbolIdentityError(); + } +} + +/** Read either native row shape without turning an absent row into a TypeError. */ +export function queryRowValue(row: unknown, key: string, index: number): unknown { + if (typeof row !== 'object' || row === null) return undefined; + const value = row as Record; + return value[key] ?? value[index]; +} + +/** Optional labels/paths may be empty or NULL; NUL is never a usable identity. */ +export function assertIdentityFields(...values: unknown[]): void { + for (const value of values) { + if ( + value !== null && + value !== undefined && + (typeof value !== 'string' || value.includes('\0')) + ) { + throw new SymbolIdentityError(); + } + } +} + +export function assertQueryIdentity( + row: unknown, + idKey: string, + idIndex: number, + fields: ReadonlyArray = [], +): void { + assertSymbolIdentity(queryRowValue(row, idKey, idIndex)); + for (const [key, index] of fields) assertIdentityFields(queryRowValue(row, key, index)); +} + +/** Ordinary query failures may degrade; corrupt identities must reach the outer error envelope. */ +export function rethrowSymbolIdentityError(error: unknown): void { + if (error instanceof SymbolIdentityError) throw error; +} diff --git a/gitnexus/src/mcp/resources.ts b/gitnexus/src/mcp/resources.ts index 8436629e8..49687fa9b 100644 --- a/gitnexus/src/mcp/resources.ts +++ b/gitnexus/src/mcp/resources.ts @@ -351,7 +351,7 @@ async function getContextResource(backend: LocalBackend, repoName?: string): Pro // Check staleness using the current on-disk lastCommit (not the cached handle) const repoPath = repo.repoPath; - const lastCommit = freshMeta?.lastCommit ?? repo.lastCommit ?? 'HEAD'; + const lastCommit = freshMeta?.lastCommit ?? repo.lastCommit ?? ''; const staleness = repoPath ? checkStaleness(repoPath, lastCommit) : { isStale: false, commitsBehind: 0 }; diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index 2965fcd6f..77b527e00 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -400,7 +400,7 @@ AFTER THIS: Review affected processes. Use context() on high-risk symbols. READ GIT WORKTREE SUPPORT: GitNexus automatically detects when the MCP server was launched from inside a linked git worktree and runs git diff against that worktree — no extra parameters needed in the common case. Pass "worktree" explicitly only when the server was started from a different directory than the worktree you are editing (e.g., the server runs from the canonical root but your changes are in a linked worktree at a different path). Returns: changed symbols, affected processes, and a risk summary. -- partial: true — a step failed and was swallowed, so the result is incomplete and risk_level is "unknown" instead of a ranked level. Two causes, with different blast radii: the symbol query (or an unparseable diff) degrades everything — changed_symbols, both counts, and the processes derived from them — while a failed process lookup degrades only affected_processes and the risk read off it, leaving the changed-symbol counts sound. changed_count:0 with partial:true is NOT a clean pre-commit check; re-run before treating the diff as safe. +- partial: true — mapping is incomplete, so risk_level is "unknown" instead of a ranked level. A failed symbol query (or an unparseable diff) degrades changed_symbols, both counts, and derived processes; a failed process lookup degrades only affected_processes and risk, leaving changed-symbol counts sound. unmapped_files lists changed supported source files with no mapped symbols, including source renames without hunks: their symbols may be missing, unindexed, or outside indexed ranges even when every query succeeded. Rebuild the index and inspect those diffs; retry alone may not resolve this state. changed_count:0 with partial:true is NOT a clean pre-commit check. - truncated: true — the changed_symbols LISTING was capped for this response. summary.changed_count counts every symbol the run observed: the true total normally, a LOWER BOUND when partial:true. Compare it with the array length rather than trusting the array.`, annotations: READ_ONLY_TOOL_ANNOTATIONS, inputSchema: { diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 0d554125a..f4cb39c66 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -12,6 +12,7 @@ import { acquireIndexLock, IndexLockTimeoutError, requireExclusiveIndexLock, + sweepStagingArtifacts, type IndexLockHandle, } from '../storage/index-lock.js'; import { ensurePrivateSharedGraph } from '../core/shared-store-analyze.js'; @@ -550,7 +551,7 @@ const mapGraphRelationshipRow = (row: any): GraphRelationship => ({ sourceId: row.sourceId, targetId: row.targetId, confidence: row.confidence, - reason: row.reason, + reason: row.reason ?? '', step: row.step, }); @@ -2152,11 +2153,21 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // for the whole embedding write, released in the finally below. let slotLock: IndexLockHandle | undefined; try { - slotLock = await acquireIndexLock(storagePath); + slotLock = await acquireIndexLock(storagePath, { sweep: false }); requireExclusiveIndexLock( slotLock, `Cannot acquire the index lock at ${storagePath}; refusing an unlocked embedding run.`, ); + // This writer cannot recover staged generations. Preserve their + // receipts, including malformed ones, before sweeping or writing. + const recoveryCheckpoint = (await loadMeta(storagePath))?.embeddingCheckpoint; + if (recoveryCheckpoint && Object.hasOwn(recoveryCheckpoint, 'recovery')) { + throw new Error( + 'Cannot generate embeddings: the index checkpoint references staged embeddings. ' + + 'Run `gitnexus analyze` to recover them first.', + ); + } + sweepStagingArtifacts(storagePath); // Writes go to the slot's own graph; a shared-store checkout // reading an immutable commit graph (#3352) takes a private copy. if (!(await ensurePrivateSharedGraph(storagePath, () => {}))) { diff --git a/gitnexus/src/storage/embedding-recovery.ts b/gitnexus/src/storage/embedding-recovery.ts new file mode 100644 index 000000000..54821bcc7 --- /dev/null +++ b/gitnexus/src/storage/embedding-recovery.ts @@ -0,0 +1,167 @@ +/** + * Filesystem-only staged embedding provenance. Keep this independent of native + * and model imports: every index-lock caller needs the retention decision. + */ +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readFileSync, + realpathSync, +} from 'node:fs'; +import path from 'node:path'; +import type { EmbeddingRecoveryReference } from './repo-meta.js'; +import { INDEX_METADATA_FILE, LEGACY_METADATA_FILE } from './storage-constants.js'; + +const STAGING_FILENAME = + /^lbug\.staging\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +export const FAMILY_SUFFIXES = [ + '', + '.wal', + '.shadow', + '.wal.checkpoint', + '.lock', + '.checkpoint.intent.lock', + '.checkpoint.apply.lock', +] as const; + +export interface ResolvedEmbeddingRecovery extends EmbeddingRecoveryReference { + dbPath: string; + /** Exact basenames, never a prefix match that can preserve another generation. */ + familyFiles: string[]; +} + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === 'object' && !Array.isArray(value); +const isNonemptyString = (value: unknown): value is string => + typeof value === 'string' && value.length > 0; +const isNodeIds = (value: unknown): value is string[] => + Array.isArray(value) && value.every(isNonemptyString); +const isCount = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; + +/** + * Resolve only an explicit interrupted-generation receipt in this canonical + * slot. This validates provenance, not native contents or current identity; + * those checks must pass separately before any rows can be reused. + */ +export const resolveEmbeddingRecovery = ( + lockDir: string, + checkpoint: unknown, +): ResolvedEmbeddingRecovery | undefined => { + if (!isRecord(checkpoint) || !isRecord(checkpoint.recovery)) return undefined; + if (checkpoint.kind !== undefined && checkpoint.kind !== 'interrupted') return undefined; + if ( + !isNonemptyString(checkpoint.at) || + !Number.isFinite(Date.parse(checkpoint.at)) || + !isCount(checkpoint.nodesProcessed) || + !isCount(checkpoint.totalNodes) || + checkpoint.nodesProcessed > checkpoint.totalNodes || + !isCount(checkpoint.chunksProcessed) || + !isNonemptyString(checkpoint.model) || + !isCount(checkpoint.dimensions) || + checkpoint.dimensions === 0 || + !isNonemptyString(checkpoint.provider) || + (checkpoint.pendingNodeIds !== undefined && !isNodeIds(checkpoint.pendingNodeIds)) + ) { + return undefined; + } + const recovery = checkpoint.recovery; + if ( + !isNonemptyString(recovery.stagingFile) || + !STAGING_FILENAME.test(recovery.stagingFile) || + !isNonemptyString(recovery.schemaFingerprint) || + !isNodeIds(recovery.unsafeNodeIds) + ) { + return undefined; + } + const unsafeNodeIds = new Set(recovery.unsafeNodeIds); + if ( + isNodeIds(checkpoint.pendingNodeIds) && + checkpoint.pendingNodeIds.some((nodeId) => !unsafeNodeIds.has(nodeId)) + ) { + return undefined; + } + + try { + const canonicalDir = realpathSync(lockDir); + if (!lstatSync(canonicalDir).isDirectory()) return undefined; + const familyFiles = FAMILY_SUFFIXES.map((suffix) => recovery.stagingFile + suffix); + for (const [index, filename] of familyFiles.entries()) { + try { + // lstat refuses both live and dangling symlinks, without following one + // to a database outside the slot. The base file must exist. + if (!lstatSync(path.join(canonicalDir, filename)).isFile()) return undefined; + } catch (error) { + if (index > 0 && (error as NodeJS.ErrnoException).code === 'ENOENT') continue; + return undefined; + } + } + return { + dbPath: path.join(canonicalDir, recovery.stagingFile), + stagingFile: recovery.stagingFile, + schemaFingerprint: recovery.schemaFingerprint, + unsafeNodeIds: [...unsafeNodeIds], + familyFiles, + }; + } catch { + return undefined; + } +}; + +/** Synchronous mirror of loadMeta's primary-first, absent-only fallback rule. */ +export const readEmbeddingRecovery = (lockDir: string): ResolvedEmbeddingRecovery | undefined => { + let metadataPath = path.join(lockDir, INDEX_METADATA_FILE); + let descriptor: number | undefined; + const noFollow = constants.O_NOFOLLOW ?? 0; + const flags = constants.O_RDONLY | noFollow | (constants.O_NONBLOCK ?? 0); + try { + try { + descriptor = openSync(metadataPath, flags); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== 'ENOENT' && code !== 'ENOTDIR') return undefined; + // Windows cannot open with O_NOFOLLOW: an open of a dangling symlink + // reports ENOENT, but that existing primary entry must prevent fallback. + try { + lstatSync(metadataPath); + return undefined; + } catch (statError) { + const statCode = (statError as NodeJS.ErrnoException).code; + if (statCode !== 'ENOENT' && statCode !== 'ENOTDIR') return undefined; + } + metadataPath = path.join(lockDir, LEGACY_METADATA_FILE); + descriptor = openSync(metadataPath, flags); + } + const opened = fstatSync(descriptor, { bigint: true }); + const entry = lstatSync(metadataPath, { bigint: true }); + // Check the opened file itself and match the current non-symlink entry. + // This also refuses replacement on platforms without O_NOFOLLOW. + if ( + !opened.isFile() || + !entry.isFile() || + (noFollow === 0 && opened.ino === 0n) || + opened.dev !== entry.dev || + opened.ino !== entry.ino + ) { + return undefined; + } + const meta: unknown = JSON.parse(readFileSync(descriptor, 'utf8')); + if (!isRecord(meta)) return undefined; + // storagePath describes the flat/cache root, including in branch-slot + // metadata. The current locked directory is the generation boundary. + return resolveEmbeddingRecovery(lockDir, meta.embeddingCheckpoint); + } catch { + return undefined; + } finally { + if (descriptor !== undefined) { + try { + closeSync(descriptor); + } catch { + /* best-effort */ + } + } + } +}; diff --git a/gitnexus/src/storage/git.ts b/gitnexus/src/storage/git.ts index 0d0e138e4..f4e26b99c 100644 --- a/gitnexus/src/storage/git.ts +++ b/gitnexus/src/storage/git.ts @@ -827,6 +827,8 @@ export interface DiffHunk { export interface FileDiff { filePath: string; + /** Decoded pre-rename path; either side can identify a source-file change. */ + oldFilePath?: string; hunks: DiffHunk[]; } @@ -1061,7 +1063,10 @@ export function parseDiffHunksResult(diffOutput: string): DiffHunkParseResult { } else { unparsedGitHeaders++; } - } else if (line.startsWith('rename to ')) { + } else if (!inHunk && line.startsWith('rename from ')) { + const oldFilePath = decodeGitPathToken(line.slice('rename from '.length)); + if (current && oldFilePath) current.oldFilePath = oldFilePath; + } else if (!inHunk && line.startsWith('rename to ')) { const filePath = pathFromRenameTo(line); if (!filePath) continue; if (current) current.filePath = filePath; diff --git a/gitnexus/src/storage/index-lock.ts b/gitnexus/src/storage/index-lock.ts index 5588127e6..5d3b6cc47 100644 --- a/gitnexus/src/storage/index-lock.ts +++ b/gitnexus/src/storage/index-lock.ts @@ -62,6 +62,7 @@ import path from 'node:path'; import os from 'node:os'; import { randomBytes, randomUUID, createHash } from 'node:crypto'; import { isProcessAlive } from '../utils/process-identity.js'; +import { readEmbeddingRecovery } from './embedding-recovery.js'; const LOCK_FILENAME = 'analyze.lock'; const LOCK_RECORD_VERSION = 1 as const; @@ -433,8 +434,9 @@ const deniedCreateHandle = ( /** * Delete orphaned build/staging artifacts left in the lock directory by a * crashed prior writer. Safe precisely because we hold the exclusive lock: no - * other writer can be creating these here right now, so anything present is a - * crash orphan. Matches this slot's staging files ONLY — never `lbug` itself, + * other writer can be creating these here right now. A checkpoint-referenced + * generation is retained for embedding recovery; all other stages are orphans. + * Matches this slot's staging files ONLY — never `lbug` itself, * never `lbug.wal`/`lbug.shadow` (the LIVE index's own sidecars), and never a * `branches//` sub-slot (which owns its own lock + sweep). Non-recursive. */ @@ -442,6 +444,7 @@ export const sweepStagingArtifacts = (lockDir: string, log?: (msg: string) => vo // Matches `lbug.new`, `lbug.new.wal`, `lbug.staging.`, `lbug.staging..wal`, … // Does NOT match `lbug`, `lbug.wal`, `lbug.shadow`. const stagingRe = /^lbug\.(staging\..+|new(\..+)?)$/; + const retained = new Set(readEmbeddingRecovery(lockDir)?.familyFiles ?? []); let removed = 0; let entries: string[]; try { @@ -450,7 +453,7 @@ export const sweepStagingArtifacts = (lockDir: string, log?: (msg: string) => vo return; } for (const name of entries) { - if (!stagingRe.test(name)) continue; + if (!stagingRe.test(name) || retained.has(name)) continue; try { unlinkSync(path.join(lockDir, name)); removed++; diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index 37eb0baf7..77e37a250 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -822,7 +822,14 @@ import { copyV8CacheIfPresent, tryLoadV8Cache, writeV8CacheFile } from './v8-sid // `handlerReceiver` hint. Warm v123 Go worker results carry no routes. // v125 (#3402): Go route hints now honor lexical declarations and captured writes; // namespace imports retain whether their local name comes from the package clause. -const SCHEMA_BUMP = 125; +// v126 (#3446): SDK positional tool registrations now emit tool definitions, +// exact handler identities, and an opt-out from unrelated file-level flows. +// Warm v125 worker results omit these definitions and must be re-extracted. +// v127 (#3450): Destructured member writes invalidate SDK registration evidence. +// Warm v126 worker results can retain false tools after a method replacement. +// v128 (#3450): SDK namespace imports now prove positional tool receivers. +// Warm v127 worker results omit these definitions and must be re-extracted. +const SCHEMA_BUMP = 128; const GITNEXUS_PKG_VERSION = (() => { try { // package.json sits at gitnexus/package.json — two levels up from diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 0c412dd7c..1562ee8c7 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -43,6 +43,15 @@ export type ContentRetention = 'full' | 'symbol' | 'none'; export type FtsProfile = 'full' | 'symbol-no-file-content' | 'name-only'; export const CONTENT_RETENTION_SCHEMA_VERSION = 1; +/** Exact staged generation whose completed embedding groups can survive a retry. */ +export interface EmbeddingRecoveryReference { + /** A run-minted basename within this metadata file's index slot. */ + stagingFile: string; + schemaFingerprint: string; + /** Active-window and inherited incomplete groups: never reusable until completed. */ + unsafeNodeIds: string[]; +} + /** * Versioned receipt for the analyzer process that produced an index. * @@ -521,6 +530,11 @@ export interface RepoMeta { * subset of their chunks; for `'partial'` they hold none. */ pendingNodeIds?: string[]; + /** + * Interrupted atomic builds only. This does not publish the staged graph + * or advance live embedding statistics; it identifies a recovery source. + */ + recovery?: EmbeddingRecoveryReference; }; /** * Name of the git branch this index represents (#2106). Absent for the diff --git a/gitnexus/test/fixtures/lang-resolution/swift-injected-closure-call/Caller.swift b/gitnexus/test/fixtures/lang-resolution/swift-injected-closure-call/Caller.swift new file mode 100644 index 000000000..dfea6fe9a --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/swift-injected-closure-call/Caller.swift @@ -0,0 +1,54 @@ +import Foundation + +enum Example { + static func runScenario() -> Int { + makeValue(input: 1) + } +} + +final class Service { + private let clock: () -> Date + + init(clock: @escaping () -> Date) { + self.clock = clock + } + + func refreshValue() -> Date { + clock() + } + + func refreshWithLocalClock() -> Int { + func clock() -> Int { 2 } + return clock() + } +} + +class BaseService { + let clock: () -> Date + + init(clock: @escaping () -> Date) { + self.clock = clock + } +} + +final class DerivedService: BaseService { + func refreshInheritedValue() -> Date { + clock() + } +} + +final class LabeledService { + let first: Int = 1 + + func first(where value: Bool) -> Int { + value ? 2 : 0 + } + + func refreshLabeled() -> Int { + first(where: true) + } +} + +class PrivateBase { + private let clock: () -> Int = { 1 } +} diff --git a/gitnexus/test/fixtures/lang-resolution/swift-injected-closure-call/Helpers.swift b/gitnexus/test/fixtures/lang-resolution/swift-injected-closure-call/Helpers.swift new file mode 100644 index 000000000..64c6b6a10 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/swift-injected-closure-call/Helpers.swift @@ -0,0 +1,39 @@ +import Foundation + +extension Example { + static func makeValue(input: Int) -> Int { + input + 1 + } +} + +enum Other { + private static func makeValue(other: String) -> Int { + -1 + } + + static func clock() -> Date { + Date.distantPast + } +} + +extension DerivedService { + func refreshInheritedFromExtension() -> Date { + clock() + } +} + +extension BaseService { + func refreshOwnFromExtension() -> Date { + clock() + } +} + +final class PrivateDerived: PrivateBase { + func clock() -> Int { + 2 + } + + func refreshPrivateAncestor() -> Int { + clock() + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/handlers.ts b/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/handlers.ts new file mode 100644 index 000000000..e30cb154d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/handlers.ts @@ -0,0 +1 @@ +export function importedHandler() { return 'imported'; } diff --git a/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/server.js b/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/server.js new file mode 100644 index 000000000..eedf3488f --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/server.js @@ -0,0 +1,5 @@ +import { McpServer as Server } from '@modelcontextprotocol/sdk/server/mcp.js'; + +const server = new Server({ name: 'javascript', version: '1' }); +function jsPing() { return 'pong'; } +server.registerTool('js_ping', { description: 'Ping JavaScript' }, jsPing); diff --git a/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/server.ts b/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/server.ts new file mode 100644 index 000000000..4a9c954f8 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/server.ts @@ -0,0 +1,52 @@ +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { importedHandler } from './handlers.js'; + +const server = new McpServer({ name: 'fixture', version: '1' }); + +function formatSearch(query: string) { return query; } +function lookupSearch(query: string) { return formatSearch(query); } +export function searchFiles(query: string) { return lookupSearch(query); } + +function formatFile(file: string) { return file; } +function lookupFile(file: string) { return formatFile(file); } +export const readFile = (file: string) => lookupFile(file); + +function formatOther() { return 'other'; } +function lookupOther() { return formatOther(); } +export function unrelatedEntry() { return lookupOther(); } + +server.registerTool('search-files', { description: 'Search files' }, searchFiles); +server.tool('read_file', 'Read a file', {}, readFile); +server.registerTool('inline_callback', {}, async () => 'inline'); +server.tool('imported_callback', importedHandler); + +function install(server: McpServer, searchFiles: () => string) { + server.registerTool('parameter_callback', {}, searchFiles); +} + +let mutable = () => 'mutable'; +server.tool('mutable_callback', mutable); + +function replaced() { return 'before'; } +replaced = () => 'after'; +server.tool('reassigned_callback', replaced); + +{ + const searchFiles = 'not callable'; + server.tool('shadowed_callback', searchFiles); +} + +const alias = readFile; +server.tool('alias_callback', alias); + +const expressionHandler = function () { return 'expression'; }; +server.registerTool('function_expression_tool', {}, expressionHandler); + +{ + const handler = () => lookupSearch('first'); + server.tool('first_block_callback', handler); +} +{ + const handler = () => lookupFile('second'); + server.tool('second_block_callback', handler); +} diff --git a/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/tools.ts b/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/tools.ts new file mode 100644 index 000000000..0013ccfab --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/typescript-mcp-tools/src/tools.ts @@ -0,0 +1,4 @@ +export const tools = [ + { name: 'manifest_tool', description: 'Existing object manifest', inputSchema: {} }, + { name: 'read_file', description: 'Duplicate manifest entry', inputSchema: {} }, +]; diff --git a/gitnexus/test/fixtures/local-backend-seed.ts b/gitnexus/test/fixtures/local-backend-seed.ts index 4878348b2..1f7fd02c9 100644 --- a/gitnexus/test/fixtures/local-backend-seed.ts +++ b/gitnexus/test/fixtures/local-backend-seed.ts @@ -1,4 +1,5 @@ import type { FTSIndexDef } from '../helpers/test-indexed-db.js'; +import { FTS_INDEXES } from '../../src/core/search/fts-schema.js'; export const LOCAL_BACKEND_SEED_DATA = [ // Files @@ -60,9 +61,8 @@ export const LOCAL_BACKEND_SEED_DATA = [ CREATE (c)-[:CodeRelation {type: 'HAS_METHOD', confidence: 1.0, reason: 'class-method', step: 0}]->(m)`, ]; -export const LOCAL_BACKEND_FTS_INDEXES: FTSIndexDef[] = [ - { table: 'Function', indexName: 'function_fts', columns: ['name', 'content', 'description'] }, - { table: 'Class', indexName: 'class_fts', columns: ['name', 'content', 'description'] }, - { table: 'Method', indexName: 'method_fts', columns: ['name', 'content', 'description'] }, - { table: 'File', indexName: 'file_fts', columns: ['name', 'content'] }, -]; +// Healthy-backend fixtures must mirror the complete configured search index set. +// The old four-table subset is now correctly reported as partial FTS coverage. +export const LOCAL_BACKEND_FTS_INDEXES: FTSIndexDef[] = FTS_INDEXES.map( + ({ table, indexName, properties }) => ({ table, indexName, columns: [...properties] }), +); diff --git a/gitnexus/test/fixtures/staged-embedding-recovery/seed.mjs b/gitnexus/test/fixtures/staged-embedding-recovery/seed.mjs new file mode 100644 index 000000000..68401a4b8 --- /dev/null +++ b/gitnexus/test/fixtures/staged-embedding-recovery/seed.mjs @@ -0,0 +1,52 @@ +import lbug from '@ladybugdb/core'; +import fs from 'node:fs'; +import { createLbugDatabase } from '../../../src/core/lbug/lbug-config.ts'; + +const [dbPath, mode] = process.argv.slice(2); +const db = createLbugDatabase(lbug, dbPath); +const conn = new lbug.Connection(db); +async function query(cypher) { + const queried = await conn.query(cypher); + for (const result of Array.isArray(queried) ? queried : [queried]) { + await result.getAll(); + await result.close(); + } +} +await query('CREATE NODE TABLE CodeEmbedding (id STRING, nodeId STRING, chunkIndex INT32, startLine INT64, endLine INT64, embedding FLOAT[2], contentHash STRING, PRIMARY KEY(id))'); +async function row(id, nodeId, chunkIndex, hash = 'same', startLine = 1, endLine = 3) { + await query(`CREATE (:CodeEmbedding {id: '${id}', nodeId: '${nodeId}', chunkIndex: ${chunkIndex}, startLine: ${startLine}, endLine: ${endLine}, embedding: [1.0, 2.0], contentHash: ${hash === null ? 'NULL' : `'${hash}'`}})`); +} +await row('complete-0', 'complete', 0); +await row('complete-1', 'complete', 1); +await row('other', 'other', 0); +await query('CHECKPOINT'); +if (mode === 'interrupted-checkpoint') { + await row('checkpoint-only', 'checkpoint-only', 0); + const main = fs.readFileSync(dbPath); + const wal = fs.readFileSync(`${dbPath}.wal`); + await conn.close(); + await db.close(); + // Restore the pre-close bytes: writable close has already checkpointed them. + fs.writeFileSync(dbPath, main); + fs.writeFileSync(`${dbPath}.wal.checkpoint`, wal); + fs.writeFileSync(`${dbPath}.wal`, ''); + fs.writeFileSync(`${dbPath}.shadow`, ''); + fs.writeFileSync(`${dbPath}.checkpoint.intent.lock`, ''); + fs.writeFileSync(`${dbPath}.checkpoint.apply.lock`, ''); +} else if (mode === 'hard-kill') { + await row('unsafe', 'unsafe-prefix', 0); + process.kill(process.pid, 'SIGKILL'); +} else { + await row('gap', 'gap', 1); + await row('duplicate-0', 'duplicate', 0); + await row('duplicate-1', 'duplicate', 0); + await row('mixed-0', 'mixed', 0, 'old'); + await row('mixed-1', 'mixed', 1, 'new'); + await row('missing-hash-0', 'missing-hash', 0); + await row('missing-hash-1', 'missing-hash', 1, null); + await row('bad-line', 'bad-line', 0, 'same', -1, 3); + await row('nan-0', 'nan', 0); + await query("CREATE (:CodeEmbedding {id: 'nan-1', nodeId: 'nan', chunkIndex: 1, startLine: 1, endLine: 3, embedding: [CAST('NaN', 'FLOAT'), 2.0], contentHash: 'same'})"); + await conn.close(); + await db.close(); +} diff --git a/gitnexus/test/fixtures/swift-captures-golden/expected-captures.json b/gitnexus/test/fixtures/swift-captures-golden/expected-captures.json index b6c40d1b1..037d5d01c 100644 --- a/gitnexus/test/fixtures/swift-captures-golden/expected-captures.json +++ b/gitnexus/test/fixtures/swift-captures-golden/expected-captures.json @@ -127,6 +127,14 @@ "captureGroups": 9, "digest": "c4d442b67247406de3b1158b958f6f3e294f11c0bbb360ed32489582769a1187" }, + "swift-injected-closure-call/Caller.swift": { + "captureGroups": 69, + "digest": "6768d62722a6b173dbe953507fec53661ea6c9786a49d8edf7caedb0a3cef0e4" + }, + "swift-injected-closure-call/Helpers.swift": { + "captureGroups": 48, + "digest": "6e6bfe83084bf189a7789702cfccb5fcc5753986b0de3c0d7f81fc3ae59270b2" + }, "swift-member-write-access/App.swift": { "captureGroups": 14, "digest": "7be68b7e510fefac99c94a4ebab3eb96556a0e587155b6c82b7602705e270987" diff --git a/gitnexus/test/integration/analyze-atomic-swap.test.ts b/gitnexus/test/integration/analyze-atomic-swap.test.ts index f664a6bb6..9731bf51b 100644 --- a/gitnexus/test/integration/analyze-atomic-swap.test.ts +++ b/gitnexus/test/integration/analyze-atomic-swap.test.ts @@ -19,32 +19,55 @@ import { promises as fs } from 'node:fs'; import path from 'node:path'; type LbugAdapter = typeof import('../../src/core/lbug/lbug-adapter.js'); +type RepoManager = typeof import('../../src/storage/repo-manager.js'); +type FsAtomic = typeof import('../../src/storage/fs-atomic.js'); const ctx = vi.hoisted(() => ({ loadMock: vi.fn(), realLoad: null as LbugAdapter['loadGraphToLbug'] | null, deleteMock: vi.fn(), realDelete: null as LbugAdapter['deleteNodesForFiles'] | null, + closeMock: vi.fn(), + realClose: null as LbugAdapter['closeLbug'] | null, + saveMetaMock: vi.fn(), + realSaveMeta: null as RepoManager['saveMeta'] | null, + renameMock: vi.fn(), + realRename: null as FsAtomic['retryRename'] | null, })); -// Delegating mock: overrides only loadGraphToLbug so a rebuild can be made to -// fail on demand (mirrors run-analyze-adopt-failure.test.ts). +// Delegating mocks keep real graph, metadata, and registry writes while +// allowing failures at the write and publication boundaries. vi.mock('../../src/core/lbug/lbug-adapter.js', async (importOriginal) => { const actual = await importOriginal(); ctx.realLoad = actual.loadGraphToLbug; ctx.realDelete = actual.deleteNodesForFiles; + ctx.realClose = actual.closeLbug; ctx.loadMock.mockImplementation(actual.loadGraphToLbug); ctx.deleteMock.mockImplementation(actual.deleteNodesForFiles); + ctx.closeMock.mockImplementation(actual.closeLbug); return { ...actual, loadGraphToLbug: ctx.loadMock, deleteNodesForFiles: ctx.deleteMock, + closeLbug: ctx.closeMock, }; }); +vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => { + const actual = await importOriginal(); + ctx.realSaveMeta = actual.saveMeta; + ctx.saveMetaMock.mockImplementation(actual.saveMeta); + return { ...actual, saveMeta: ctx.saveMetaMock }; +}); +vi.mock('../../src/storage/fs-atomic.js', async (importOriginal) => { + const actual = await importOriginal(); + ctx.realRename = actual.retryRename; + ctx.renameMock.mockImplementation(actual.retryRename); + return { ...actual, retryRename: ctx.renameMock }; +}); import { analyzeFailureMayHaveMutatedLiveIndex, runFullAnalysis, } from '../../src/core/run-analyze.js'; -import { getStoragePaths } from '../../src/storage/repo-manager.js'; +import { getStoragePaths, loadMeta, readRegistry } from '../../src/storage/repo-manager.js'; import { initLbug as poolInit, executeQuery as poolQuery, @@ -83,6 +106,16 @@ describe.skipIf(isWin)('atomic full-rebuild swap (#2)', () => { ctx.deleteMock.mockImplementation((...a: Parameters) => ctx.realDelete!(...a), ); + ctx.closeMock.mockReset(); + ctx.closeMock.mockImplementation(() => ctx.realClose!()); + ctx.saveMetaMock.mockReset(); + ctx.saveMetaMock.mockImplementation((...a: Parameters) => + ctx.realSaveMeta!(...a), + ); + ctx.renameMock.mockReset(); + ctx.renameMock.mockImplementation((...a: Parameters) => + ctx.realRename!(...a), + ); }); afterEach(async () => { @@ -144,6 +177,102 @@ describe.skipIf(isWin)('atomic full-rebuild swap (#2)', () => { } }, 180_000); + it.each( + (['full', 'incremental'] as const).flatMap((writeMode) => + (['close', 'swap', 'metadata'] as const).map((failurePoint) => ({ + writeMode, + failurePoint, + })), + ), + )( + 'keeps registry freshness unchanged when $writeMode publication fails at $failurePoint', + async ({ writeMode, failurePoint }) => { + const { repo, cleanup } = await makeRepo(); + const repoId = `atomic-publication-${writeMode}-${failurePoint}`; + try { + await runFullAnalysis(repo, {}, { onProgress: () => {} }); + const { lbugPath, storagePath } = getStoragePaths(repo); + const oldGraph = await fs.readFile(lbugPath); + const oldMeta = await loadMeta(storagePath); + const oldRegistry = await readRegistry(); + expect(oldRegistry).toHaveLength(1); + expect(oldRegistry[0]).toMatchObject({ + lastCommit: oldMeta!.lastCommit, + indexedAt: oldMeta!.indexedAt, + }); + + await fs.writeFile( + path.join(repo, 'a.ts'), + 'export function replacement() { return "new graph"; }\n', + ); + execSync('git -c user.name=t -c user.email=t@t commit -am replacement', { + cwd: repo, + stdio: 'pipe', + }); + const nextCommit = execSync('git rev-parse HEAD', { + cwd: repo, + encoding: 'utf8', + }).trim(); + expect(nextCommit).not.toBe(oldMeta!.lastCommit); + + const injected = new Error(`injected final ${failurePoint} failure`); + ctx.loadMock.mockClear(); + ctx.deleteMock.mockClear(); + if (failurePoint === 'close') { + ctx.closeMock.mockImplementation(async () => { + // Actually release native handles, then inject the rejection only + // after loading the new graph, not at the pre-rebuild close. + await ctx.realClose!(); + if (ctx.loadMock.mock.calls.length > 0) throw injected; + }); + } else if (failurePoint === 'swap') { + ctx.renameMock.mockImplementation( + async (...args: Parameters) => { + const [from, to] = args; + if (from.startsWith(`${lbugPath}.staging.`) && to === lbugPath) throw injected; + await ctx.realRename!(...args); + }, + ); + } else { + ctx.saveMetaMock.mockImplementation( + async (...args: Parameters) => { + const [, meta] = args; + if (meta.lastCommit === nextCommit && !meta.incrementalInProgress) throw injected; + await ctx.realSaveMeta!(...args); + }, + ); + } + + const failure = await runFullAnalysis( + repo, + writeMode === 'full' ? { force: true } : { atomicIncremental: true }, + { onProgress: () => {} }, + ).catch((error: unknown) => error); + expect(failure).toBe(injected); + expect(ctx.loadMock).toHaveBeenCalled(); + expect(ctx.deleteMock.mock.calls.length > 0).toBe(writeMode === 'incremental'); + expect(await readRegistry()).toEqual(oldRegistry); + expect(await loadMeta(storagePath)).toMatchObject({ lastCommit: oldMeta!.lastCommit }); + expect(await lingeringTemp(lbugPath)).toEqual([]); + + // A metadata failure occurs after the swap; close/swap failures keep + // the old bytes. Both cases must retain the old registry receipt. + const published = failurePoint === 'metadata'; + expect(analyzeFailureMayHaveMutatedLiveIndex(failure)).toBe(published); + expect((await fs.readFile(lbugPath)).equals(oldGraph)).toBe(!published); + await poolInit(repoId, lbugPath); + const names = (await poolQuery(repoId, 'MATCH (f:Function) RETURN f.name AS n')).flatMap( + (row) => Object.values(row as Record).map(String), + ); + expect(names.sort()).toEqual(published ? ['replacement'] : ['caller', 'greet']); + } finally { + await poolClose(repoId); + await cleanup(); + } + }, + 180_000, + ); + it('marks a failure after an atomic publish as potentially live-mutating', async () => { const { repo, cleanup } = await makeRepo(); try { diff --git a/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts b/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts new file mode 100644 index 000000000..dbb5b893e --- /dev/null +++ b/gitnexus/test/integration/analyze-staged-embedding-recovery.test.ts @@ -0,0 +1,449 @@ +/** + * Exercise the CLI -> native staged DB -> durable checkpoint -> SIGKILL -> + * isolated recovery -> graph rebuild -> publication chain. The endpoint is + * local and deterministic; request text is the billing/reuse oracle. + */ +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; +import fs from 'node:fs'; +import http from 'node:http'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { CLI_SPAWN_PREFIX, tsxLoaderUrl } from '../helpers/cli-entry.js'; + +const DIMS = 8; +const NODE_COUNT = 5_128; +const DEADLINE = process.env.CI ? 180_000 : 120_000; +const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const adapterUrl = pathToFileURL(path.join(packageRoot, 'src/core/lbug/lbug-adapter.ts')).href; + +interface CheckpointMeta { + repoPath: string; + stats?: { embeddings?: number }; + embeddingCheckpoint?: { + nodesProcessed: number; + chunksProcessed: number; + pendingNodeIds?: string[]; + recovery?: { stagingFile: string; schemaFingerprint: string; unsafeNodeIds: string[] }; + }; +} + +interface Run { + child: ChildProcess; + output: () => string; + done: Promise<{ code: number | null; signal: NodeJS.Signals | null; output: string }>; +} + +let root: string; +let stoppedRepo: string; +let server: http.Server; +let endpoint: string; +let submitted: string[] = []; +let completed: string[] = []; +let durableTexts: string[] = []; +const activeWindowCompleted: string[] = []; +let held: string[] = []; +let stopAfterCheckpoint = false; +let currentRepo: string; +let gateResolve: (() => void) | undefined; +const running = new Set(); + +function readMeta(repo: string): CheckpointMeta { + return JSON.parse(fs.readFileSync(path.join(repo, '.gitnexus', 'gitnexus.json'), 'utf8')); +} + +function cliEnv(repo: string): NodeJS.ProcessEnv { + const env = { ...process.env }; + for (const key of Object.keys(env)) { + if (key.startsWith('GITNEXUS_EMBEDDING_') || key.startsWith('GITNEXUS_STORAGE_')) + delete env[key]; + } + return { + ...env, + GITNEXUS_HOME: path.join(root, `home-${path.basename(repo)}`), + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_LBUG_EXTENSION_INSTALL: 'never', + GITNEXUS_LBUG_BUFFER_POOL_SIZE: String(256 * 1024 * 1024), + GITNEXUS_EMBEDDING_URL: endpoint, + GITNEXUS_EMBEDDING_MODEL: 'staged-recovery-fixture', + GITNEXUS_EMBEDDING_DIMS: String(DIMS), + GITNEXUS_EMBEDDING_BATCH_SIZE: '5000', + GITNEXUS_EMBEDDING_SUB_BATCH_SIZE: '64', + GITNEXUS_EMBEDDING_MAX_ATTEMPTS: '1', + GITNEXUS_EMBEDDING_CACHE_IN_MEMORY_LIMIT: '0', + GITNEXUS_MEMORY: 'off', + // SIGKILL skips process exit hooks; keep orphaned cache/export spills in + // this suite's owned directory so afterAll can remove them as well. + TMPDIR: path.join(root, 'tmp'), + NODE_OPTIONS: `${process.env.NODE_OPTIONS || ''} --max-old-space-size=2048`.trim(), + CI: '1', + }; +} + +function runAnalyze(repo: string, flags: string[] = [], onOutput?: (output: string) => void): Run { + let output = ''; + const child = spawn( + process.execPath, + [ + ...CLI_SPAWN_PREFIX, + 'analyze', + repo, + '--no-share', + '--skip-skills', + '--skip-fts', + '--workers', + '1', + ...flags, + ], + { cwd: repo, env: cliEnv(repo), stdio: ['ignore', 'pipe', 'pipe'] }, + ); + running.add(child); + const timeout = setTimeout(() => child.kill('SIGKILL'), DEADLINE); + const collect = (chunk: Buffer) => { + output += chunk.toString(); + onOutput?.(output); + }; + child.stdout?.on('data', collect); + child.stderr?.on('data', collect); + const done = new Promise<{ code: number | null; signal: NodeJS.Signals | null; output: string }>( + (resolve, reject) => { + child.once('error', reject); + child.once('close', (code, signal) => { + clearTimeout(timeout); + running.delete(child); + resolve({ code, signal, output }); + }); + }, + ); + return { child, done, output: () => output }; +} + +async function successfulAnalyze(repo: string, flags: string[] = []): Promise { + submitted = []; + stopAfterCheckpoint = false; + currentRepo = repo; + const result = await runAnalyze(repo, flags).done; + expect(result.output, `CLI exited ${result.code}, signal ${result.signal}`).not.toContain( + 'SIGABRT', + ); + expect(result.code, result.output).toBe(0); + return [...submitted]; +} + +function cloneStoppedRepo(name: string): string { + const repo = path.join(root, name); + fs.cpSync(stoppedRepo, repo, { recursive: true }); + for (const filename of ['gitnexus.json', 'meta.json']) { + const target = path.join(repo, '.gitnexus', filename); + const meta = JSON.parse(fs.readFileSync(target, 'utf8')); + meta.repoPath = repo; + meta.storagePath = path.join(repo, '.gitnexus'); + fs.writeFileSync(target, JSON.stringify(meta)); + } + return repo; +} + +function readPublishedRows( + repo: string, +): Array<{ nodeId: string; chunkIndex: number; embedding: number[] }> { + // Keep native handles out of the vitest fork, and wait for clean teardown + // before accepting the receipt. Every read opens only a published DB. + const receiptPath = path.join(root, `rows-${path.basename(repo)}.json`); + const script = ` + const adapter = await import(${JSON.stringify(adapterUrl)}); + const fs = await import('node:fs'); + await adapter.initLbug(${JSON.stringify(path.join(repo, '.gitnexus', 'lbug'))}); + try { + const rows = await adapter.executeQuery('MATCH (e:CodeEmbedding) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.embedding AS embedding'); + fs.writeFileSync(${JSON.stringify(receiptPath)}, JSON.stringify(rows)); + console.log('ROWS_RECEIPT:' + rows.length); + } finally { await adapter.closeLbug(); } + `; + const result = spawnSync( + process.execPath, + ['--import', tsxLoaderUrl(), '--input-type=module', '-e', script], + { + cwd: packageRoot, + env: cliEnv(repo), + encoding: 'utf8', + timeout: 30_000, + }, + ); + try { + const diagnostic = + `${result.error ?? ''} ${result.signal ?? ''}\n${result.stderr}\n${result.stdout}`.slice( + 0, + 4096, + ); + expect(result.status, diagnostic).toBe(0); + expect(result.stdout).toMatch(/ROWS_RECEIPT:\d+/); + return JSON.parse(fs.readFileSync(receiptPath, 'utf8')); + } finally { + fs.rmSync(receiptPath, { force: true }); + } +} + +function expectPublishedComplete(repo: string, expectedNodes: number): void { + const meta = readMeta(repo); + expect(meta.embeddingCheckpoint).toBeUndefined(); + const rows = readPublishedRows(repo); + expect(new Set(rows.map((row) => row.nodeId)).size).toBe(expectedNodes); + expect(meta.stats?.embeddings).toBe(rows.length); + const byNode = new Map(); + for (const row of rows) { + expect(row.embedding).toHaveLength(DIMS); + expect(row.embedding.every(Number.isFinite)).toBe(true); + const indices = byNode.get(row.nodeId) ?? []; + indices.push(row.chunkIndex); + byNode.set(row.nodeId, indices); + } + for (const indices of byNode.values()) { + expect(indices.sort((a, b) => a - b)).toEqual( + Array.from({ length: indices.length }, (_, index) => index), + ); + } + expect( + fs.readdirSync(path.join(repo, '.gitnexus')).filter((name) => name.startsWith('lbug.staging.')), + ).toEqual([]); +} + +beforeAll(async () => { + if (process.platform === 'win32') return; + root = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-staged-recovery-e2e-')); + fs.mkdirSync(path.join(root, 'tmp')); + stoppedRepo = path.join(root, 'interrupted'); + currentRepo = stoppedRepo; + fs.mkdirSync(stoppedRepo); + const shortFunctions = Array.from( + { length: NODE_COUNT - 1 }, + (_, index) => `export function recoverable${index}() { return ${index}; }`, + ); + // Multi-chunk nodes must be reused as a complete group, including their tail. + const longFunction = `export function longRecoverable() {\n${Array.from({ length: 80 }, (_, index) => ` // retained chunk marker ${index} ${'x'.repeat(80)}`).join('\n')}\n return 42;\n}`; + fs.writeFileSync( + path.join(stoppedRepo, 'functions.ts'), + `${longFunction}\n${shortFunctions.join('\n')}\n`, + ); + const gitEnv = { + ...process.env, + GIT_AUTHOR_NAME: 'test', + GIT_AUTHOR_EMAIL: 'test@test', + GIT_COMMITTER_NAME: 'test', + GIT_COMMITTER_EMAIL: 'test@test', + }; + for (const args of [['init'], ['add', 'functions.ts'], ['commit', '-m', 'recovery fixture']]) { + const result = spawnSync('git', args, { cwd: stoppedRepo, env: gitEnv, encoding: 'utf8' }); + expect(result.status, result.stderr).toBe(0); + } + server = http.createServer((request, response) => { + let body = ''; + request.on('data', (chunk) => { + body += chunk; + }); + request.on('end', () => { + const { input } = JSON.parse(body) as { input: string[] }; + submitted.push(...input); + if ( + stopAfterCheckpoint && + (readMeta(currentRepo).embeddingCheckpoint?.nodesProcessed ?? 0) >= 5000 + ) { + if (activeWindowCompleted.length > 0) { + held = [...input]; + gateResolve?.(); + // One sub-batch has already inserted rows inside the unsafe window. + // Awaiting the next real request gates a crash before it completes. + return; + } + durableTexts = [...completed]; + activeWindowCompleted.push(...input); + } + completed.push(...input); + response.writeHead(200, { 'content-type': 'application/json' }); + response.end( + JSON.stringify({ + data: input.map((text, index) => ({ + index, + embedding: Array.from( + { length: DIMS }, + (_, dimension) => ((text.length + dimension * 17) % 101) / 101, + ), + })), + }), + ); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as { port: number }; + endpoint = `http://127.0.0.1:${address.port}/v1`; + await successfulAnalyze(stoppedRepo, ['--index-only']); + expect(readMeta(stoppedRepo).stats?.embeddings ?? 0).toBe(0); + completed = []; + held = []; + stopAfterCheckpoint = true; + const gate = new Promise((resolve) => { + gateResolve = resolve; + }); + const first = runAnalyze(stoppedRepo, ['--force', '--embeddings']); + await Promise.race([ + gate, + first.done.then((result) => { + throw new Error(`CLI exited before recovery gate: ${result.output}`); + }), + ]); + first.child.kill('SIGKILL'); + expect((await first.done).signal).toBe('SIGKILL'); + const interrupted = readMeta(stoppedRepo); + expect(interrupted.embeddingCheckpoint?.nodesProcessed).toBe(5000); + expect(interrupted.embeddingCheckpoint?.recovery?.stagingFile).toMatch(/^lbug\.staging\./); + expect(interrupted.stats?.embeddings ?? 0).toBe(0); + expect(completed.length).toBeGreaterThanOrEqual(5000); + expect(completed.filter((text) => text.includes('longRecoverable')).length).toBeGreaterThan(1); + expect(activeWindowCompleted).toHaveLength(64); + expect(held.length).toBeGreaterThan(0); + gateResolve = undefined; + stopAfterCheckpoint = false; +}, DEADLINE * 2); + +afterAll(async () => { + for (const child of running) child.kill('SIGKILL'); + await Promise.all( + [...running].map( + (child) => new Promise((resolve) => child.once('close', () => resolve())), + ), + ); + if (server) { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } + if (root) fs.rmSync(root, { recursive: true, force: true }); +}); + +// Atomic publication is POSIX-specific; Windows uses the in-place path. +describe + .skipIf(process.platform === 'win32') + .sequential('interrupted staged embedding recovery (real CLI and native DB)', () => { + it.each([ + ['plain', []], + ['forced', ['--force', '--embeddings']], + ] as const)( + '%s retry bills only unfinished groups and publishes an honest count', + async (name, flags) => { + const repo = cloneStoppedRepo(name); + const texts = await successfulAnalyze(repo, [...flags]); + const durable = new Set(durableTexts); + expect(texts.filter((text) => durable.has(text))).toEqual([]); + for (const text of held) expect(texts).toContain(text); + for (const text of activeWindowCompleted) expect(texts).toContain(text); + expect(texts.length).toBe(128); + expectPublishedComplete(repo, NODE_COUNT); + }, + DEADLINE, + ); + + it( + 'manual checkpoint opt-out completes a staged retry without resubmitting durable chunks', + async () => { + const repo = cloneStoppedRepo('manual-checkpoint-opt-out'); + const previous = process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT; + process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT = '0'; + try { + const texts = await successfulAnalyze(repo, ['--force', '--embeddings']); + expect(texts.filter((text) => new Set(durableTexts).has(text))).toEqual([]); + expect(texts.length).toBe(128); + expectPublishedComplete(repo, NODE_COUNT); + } finally { + if (previous === undefined) delete process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT; + else process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT = previous; + } + }, + DEADLINE, + ); + + it( + 'survives another crash after harvesting but before the replacement is durable', + async () => { + const repo = cloneStoppedRepo('crash-again'); + const source = readMeta(repo).embeddingCheckpoint?.recovery?.stagingFile; + expect(source).toBeDefined(); + if (!source) throw new Error('fixture has no retained generation'); + submitted = []; + currentRepo = repo; + let killed = false; + const retry = runAnalyze(repo, [], (output) => { + if (!killed && /Recovered \d+ complete staged embedding chunk/.test(output)) { + killed = true; + retry.child.kill('SIGKILL'); + } + }); + const result = await retry.done; + expect(killed, result.output).toBe(true); + expect(result.signal).toBe('SIGKILL'); + expect(submitted).toEqual([]); + expect(readMeta(repo).embeddingCheckpoint?.recovery?.stagingFile).toBe(source); + expect(fs.existsSync(path.join(repo, '.gitnexus', source))).toBe(true); + const finalTexts = await successfulAnalyze(repo); + expect(finalTexts.filter((text) => new Set(durableTexts).has(text))).toEqual([]); + expect(finalTexts.length).toBe(128); + expectPublishedComplete(repo, NODE_COUNT); + }, + DEADLINE * 2, + ); + + it( + 'regenerates changed content and removes deleted nodes while reusing the other complete groups', + async () => { + const repo = cloneStoppedRepo('changed'); + const source = path.join(repo, 'functions.ts'); + const content = fs + .readFileSync(source, 'utf8') + .replace( + 'export function recoverable5() { return 5; }', + 'export function recoverable5() { return 999999; }', + ) + .replace( + 'export function recoverable6() { return 6; }', + '// deleted function retains line offsets', + ); + fs.writeFileSync(source, content); + const texts = await successfulAnalyze(repo); + expect(texts.some((text) => text.includes('recoverable5') && text.includes('999999'))).toBe( + true, + ); + expect(texts.some((text) => text.includes('recoverable6'))).toBe(false); + expect(texts.filter((text) => new Set(durableTexts).has(text))).toEqual([]); + expect(texts.length).toBe(129); + expectPublishedComplete(repo, NODE_COUNT - 1); + }, + DEADLINE, + ); + + it( + 'a forced retry with a different model does not import the staged cache', + async () => { + const repo = cloneStoppedRepo('different-model'); + const texts = await successfulAnalyze(repo, [ + '--force', + '--embeddings', + '--embedding-model', + 'different-model', + ]); + for (const text of durableTexts) expect(texts).toContain(text); + expect(texts.length).toBeGreaterThanOrEqual(NODE_COUNT); + expectPublishedComplete(repo, NODE_COUNT); + }, + DEADLINE, + ); + + it( + 'explicit drop abandons staged vectors without contacting the provider', + async () => { + const repo = cloneStoppedRepo('drop'); + expect(await successfulAnalyze(repo, ['--force', '--drop-embeddings'])).toEqual([]); + expect(readMeta(repo).embeddingCheckpoint).toBeUndefined(); + expect(readMeta(repo).stats?.embeddings ?? 0).toBe(0); + expect(readPublishedRows(repo)).toEqual([]); + }, + DEADLINE, + ); + }); diff --git a/gitnexus/test/integration/context-resource-staleness.test.ts b/gitnexus/test/integration/context-resource-staleness.test.ts index ab94fcab8..c5f58454d 100644 --- a/gitnexus/test/integration/context-resource-staleness.test.ts +++ b/gitnexus/test/integration/context-resource-staleness.test.ts @@ -207,4 +207,26 @@ describe('context resource freshness — out-of-process analyze (#2438)', () => expect(result).toContain('symbols: 333'); expect(result).toContain('processes: 4'); }); + + it('does not claim divergence when disk metadata and the cached handle have no indexed commit', async function missingIndexedCommitDoesNotClaimDivergence() { + writeFileSync(path.join(repoPath, 'a.ts'), 'export const a = 1;\n'); + runGit(repoPath, 'add', 'a.ts'); + runGit(repoPath, 'commit', '-m', 'c1'); + + // A legacy metadata/registry entry can omit lastCommit despite RepoMeta's + // required field; seed that persisted shape rather than a symbolic ref. + await seedIndexedRepo(repoPath, storagePath, { + repoPath, + indexedAt: '2024-01-01T00:00:00Z', + stats: { files: 1, nodes: 1, processes: 0 }, + } as RepoMeta); + + const backend = new LocalBackend(); + await backend.init(); + expect((await backend.resolveRepo('test-repo')).lastCommit).toBeUndefined(); + + const result = await readResource('gitnexus://repo/test-repo/context', backend); + expect(result).not.toContain('staleness:'); + expect(result).toContain(' commit: ""'); + }); }); diff --git a/gitnexus/test/integration/detect-changes-path-anchoring.test.ts b/gitnexus/test/integration/detect-changes-path-anchoring.test.ts index f4bd381d8..f65cdec6b 100644 --- a/gitnexus/test/integration/detect-changes-path-anchoring.test.ts +++ b/gitnexus/test/integration/detect-changes-path-anchoring.test.ts @@ -14,6 +14,7 @@ * 0-based line space (#2377, #2915). */ import { it, expect, beforeAll, vi } from 'vitest'; +import { execFileSync } from 'node:child_process'; import { mkdirSync, writeFileSync } from 'fs'; import path from 'path'; import { LocalBackend } from '../../src/mcp/local/local-backend.js'; @@ -58,8 +59,10 @@ function makeWorkingCopy(): string { /** The fields these tests read off one `detect_changes` run. */ type DetectChangesResult = { error?: unknown; - summary: { changed_count: number }; + summary: { changed_count: number; risk_level: string }; changed_symbols: { name: string; filePath: string }[]; + partial?: boolean; + unmapped_files?: string[]; }; withTestLbugDB( @@ -118,3 +121,51 @@ withTestLbugDB( }, }, ); + +withTestLbugDB( + 'detect-changes-unindexed-sibling', + (handle) => { + it('does not certify a new root index.ts from an indexed pkg/index.ts suffix match', async () => { + const backend = (handle as typeof handle & { _backend: LocalBackend })._backend; + const result = (await backend.callTool('detect_changes', { + scope: 'staged', + })) as DetectChangesResult; + expect(result.error).toBeUndefined(); + // The fallback remains visible, but cannot certify the identity of the new file. + expect(result.changed_symbols).toEqual([ + expect.objectContaining({ name: 'helper', filePath: 'pkg/index.ts' }), + ]); + expect(result.summary.risk_level).toBe('unknown'); + expect(result.partial).toBe(true); + expect(result.unmapped_files).toEqual(['index.ts']); + }); + }, + { + seed: [ + "CREATE (fn:Function {id: 'Function:pkg/index.ts:helper', name: 'helper', filePath: 'pkg/index.ts', startLine: 0, endLine: 1, isExported: true})", + ], + poolAdapter: true, + afterSetup: async (handle) => { + const repoDir = tempDirs.dir(); + mkdirSync(path.join(repoDir, 'pkg'), { recursive: true }); + writeFileSync(path.join(repoDir, 'pkg/index.ts'), 'export function helper() {\n}\n'); + initGitRepo(repoDir); + commitAll(repoDir, 'indexed sibling'); + writeFileSync(path.join(repoDir, 'index.ts'), 'export function added() {\n}\n'); + execFileSync('git', ['add', 'index.ts'], { cwd: repoDir }); + vi.mocked(listRegisteredRepos).mockResolvedValue([ + { + name: 'sibling-repo', + path: repoDir, + storagePath: handle.tmpHandle.dbPath, + indexedAt: new Date().toISOString(), + lastCommit: 'abc1234', + stats: { files: 1, nodes: 1, edges: 0, communities: 0, processes: 0 }, + }, + ]); + const backend = new LocalBackend(); + await backend.init(); + (handle as typeof handle & { _backend: LocalBackend })._backend = backend; + }, + }, +); diff --git a/gitnexus/test/integration/fts-extension-e2e.test.ts b/gitnexus/test/integration/fts-extension-e2e.test.ts index 47f79d137..9bc6ad544 100644 --- a/gitnexus/test/integration/fts-extension-e2e.test.ts +++ b/gitnexus/test/integration/fts-extension-e2e.test.ts @@ -25,6 +25,7 @@ import { spawnSync } from 'child_process'; import path from 'path'; import fs from 'fs'; import os from 'os'; +import { inspect } from 'node:util'; import { getExtensionInstallChildProcessArgs } from '../../src/core/lbug/extension-loader.js'; import { @@ -144,6 +145,8 @@ const makeFixtureRepo = (label: string): string => { interface CliResult { status: number | null; + /** Keep native termination and spawn failures visible in assertion messages. */ + diagnostics: string; /** stdout + stderr combined — warn lines and progress renderer interleave streams. */ output: string; } @@ -172,7 +175,20 @@ const runCli = ( NODE_OPTIONS: `${process.env.NODE_OPTIONS || ''} --max-old-space-size=8192`.trim(), }, }); - return { status: result.status, output: `${result.stdout ?? ''}\n${result.stderr ?? ''}` }; + return { + status: result.status, + diagnostics: inspect( + { + status: result.status, + signal: result.signal, + error: result.error, + stdout: result.stdout, + stderr: result.stderr, + }, + { depth: null, maxStringLength: null }, + ), + output: `${result.stdout ?? ''}\n${result.stderr ?? ''}`, + }; }; beforeAll(() => { @@ -208,7 +224,7 @@ describe('happy path — extension pre-installed, fully offline (load-only)', () it('analyze builds the index with FTS and emits no degradation warning', () => { const result = runCli(['analyze'], repo, home, 'load-only'); - expect(result.status).toBe(0); + expect(result.status, result.diagnostics).toBe(0); expect(result.output).toContain('indexed successfully'); expect(result.output).not.toContain('FTS extension unavailable'); expect(result.output).not.toContain('search is disabled'); @@ -216,14 +232,14 @@ describe('happy path — extension pre-installed, fully offline (load-only)', () it('query finds the symbol via BM25 with no degradation warning', () => { const result = runCli(['query', 'greetE2eSymbol'], repo, home, 'load-only'); - expect(result.status).toBe(0); + expect(result.status, result.diagnostics).toBe(0); expect(result.output).toContain('greetE2eSymbol'); expect(result.output).not.toContain('keyword search degraded'); }, 60_000); it('doctor reports a live-probed available FTS and a resolved LadybugDB version', () => { const result = runCli(['doctor'], repo, home, 'load-only'); - expect(result.status).toBe(0); + expect(result.status, result.diagnostics).toBe(0); expect(result.output).toContain('Full-text search: available'); // #2374: version used to print as "unknown" on every platform. expect(result.output).toMatch(/LadybugDB:\s*\d+\.\d+\.\d+/); @@ -231,7 +247,7 @@ describe('happy path — extension pre-installed, fully offline (load-only)', () it('analyze --repair-fts rebuilds the search indexes offline', () => { const result = runCli(['analyze', '--repair-fts'], repo, home, 'load-only'); - expect(result.status).toBe(0); + expect(result.status, result.diagnostics).toBe(0); expect(result.output).toContain('FTS indexes repaired successfully'); }, 180_000); }); @@ -250,7 +266,7 @@ describe('packaged vendor survives a broken or missing home copy', () => { it('analyze stays FTS-available when ~/.lbdb is broken', () => { const result = runCli(['analyze'], repo, home, 'load-only'); - expect(result.status).toBe(0); + expect(result.status, result.diagnostics).toBe(0); expect(result.output).toContain('indexed successfully'); expect(result.output).not.toContain('FTS extension unavailable'); expect(result.output).not.toContain('search is disabled'); @@ -258,13 +274,13 @@ describe('packaged vendor survives a broken or missing home copy', () => { it('analyze --repair-fts succeeds from the packaged artifact', () => { const result = runCli(['analyze', '--repair-fts'], repo, home, 'load-only'); - expect(result.status).toBe(0); + expect(result.status, result.diagnostics).toBe(0); expect(result.output).toContain('FTS indexes repaired successfully'); }, 180_000); it('query finds the symbol with no HOME-copy degradation warning', () => { const result = runCli(['query', 'greetE2eSymbol'], repo, home, 'load-only'); - expect(result.status).toBe(0); + expect(result.status, result.diagnostics).toBe(0); expect(result.output).toContain('greetE2eSymbol'); expect(result.output).not.toContain('keyword search degraded'); expect(result.output).not.toContain('FTS extension failed to load'); @@ -272,7 +288,7 @@ describe('packaged vendor survives a broken or missing home copy', () => { it('doctor reports a live-probed available FTS despite a broken HOME copy', () => { const result = runCli(['doctor'], repo, home, 'load-only'); - expect(result.status).toBe(0); + expect(result.status, result.diagnostics).toBe(0); expect(result.output).toContain('Full-text search: available'); }, 60_000); @@ -280,7 +296,7 @@ describe('packaged vendor survives a broken or missing home copy', () => { const missing = makeHome('missing'); const missingRepo = makeFixtureRepo('missing'); const result = runCli(['analyze'], missingRepo, missing.home, 'load-only'); - expect(result.status).toBe(0); + expect(result.status, result.diagnostics).toBe(0); expect(result.output).toContain('indexed successfully'); expect(result.output).not.toContain('FTS extension unavailable'); expect(result.output).not.toContain('has not been installed'); @@ -295,7 +311,7 @@ describe('regression — the home copy disappears between analyze runs (#2841)', // 1. First analyze with the extension in place: the index ends up carrying // an FTS index on every searchable table. const first = runCli(['analyze'], repo, home, 'load-only'); - expect(first.status).toBe(0); + expect(first.status, first.diagnostics).toBe(0); // This case needs run 1 to actually BUILD the indexes — without them there // is nothing for the gate to trip on and the assertions below would be // vacuous. When the seeded extension cannot load on this host (the same @@ -332,7 +348,7 @@ describe('regression — the home copy disappears between analyze runs (#2841)', // table File but its extension is not loaded" and no mention of FTS at all. // Packaged vendor still loads after HOME vanishes, so incremental stays // incremental (no Binder, no full-DB escalation). - expect(second.status).toBe(0); + expect(second.status, second.diagnostics).toBe(0); expect(second.output).not.toContain('its extension is not loaded'); expect(second.output).not.toContain('full DB write'); expect(second.output).not.toContain('forcing full rebuild'); @@ -346,15 +362,15 @@ describe('auto policy — packaged vendor does not need a HOME reinstall', () => const repo = makeFixtureRepo('heal'); const first = runCli(['analyze'], repo, home, 'load-only'); - expect(first.status).toBe(0); + expect(first.status, first.diagnostics).toBe(0); expect(first.output).not.toContain('FTS extension unavailable'); const repair = runCli(['analyze', '--repair-fts'], repo, home, 'auto'); - expect(repair.status).toBe(0); + expect(repair.status, repair.diagnostics).toBe(0); expect(repair.output).toContain('FTS indexes repaired successfully'); const query = runCli(['query', 'greetE2eSymbol'], repo, home, 'load-only'); - expect(query.status).toBe(0); + expect(query.status, query.diagnostics).toBe(0); expect(query.output).toContain('greetE2eSymbol'); expect(query.output).not.toContain('keyword search degraded'); }, 600_000); @@ -363,7 +379,7 @@ describe('auto policy — packaged vendor does not need a HOME reinstall', () => const { home } = makeHome('missing'); const repo = makeFixtureRepo('fresh'); const result = runCli(['analyze'], repo, home, 'load-only'); - expect(result.status).toBe(0); + expect(result.status, result.diagnostics).toBe(0); expect(result.output).toContain('indexed successfully'); expect(result.output).not.toContain('FTS extension unavailable'); }, 600_000); diff --git a/gitnexus/test/integration/fts-repair-warm-session.test.ts b/gitnexus/test/integration/fts-repair-warm-session.test.ts index 2550ba874..ac9ce2713 100644 --- a/gitnexus/test/integration/fts-repair-warm-session.test.ts +++ b/gitnexus/test/integration/fts-repair-warm-session.test.ts @@ -32,6 +32,7 @@ const REQUIRE_FTS = process.env.GITNEXUS_REQUIRE_FTS === '1'; type QueryResult = { error?: unknown; warning?: string; + partial?: boolean; definitions?: Array<{ id: string }>; process_symbols?: Array<{ id: string }>; }; @@ -40,7 +41,8 @@ const matchedIds = (r: QueryResult): string[] => [...(r.process_symbols ?? []), ...(r.definitions ?? [])].map((s) => s.id); const ftsMissing = (r: QueryResult): boolean => - typeof r.warning === 'string' && /FTS indexes missing/i.test(r.warning); + typeof r.warning === 'string' && + /FTS indexes missing|missing configured indexes/i.test(r.warning); /** * Poll the SAME warm `LocalBackend` until it stops reporting FTS-missing, or @@ -90,14 +92,14 @@ describe('warm MCP session observes an in-place --repair-fts rebuild (#2767)', ( await tmpHandle.cleanup(); }); - it( - 'a warm session transitions from FTS-unavailable to FTS-available without restarting, after an out-of-band --repair-fts', + it.each(['all', 'Function'] as const)( + 'a warm session repairs missing %s indexes without restarting', { timeout: 60_000 }, - async (ctx) => { + async (missing, ctx) => { const adapter = await import('../../src/core/lbug/lbug-adapter.js'); const { createSearchFTSIndexes } = await import('../../src/core/search/fts-indexes.js'); - // ── Step 1: build the index WITHOUT FTS (analyzed before repair) ──── + // ── Step 1: build an index with all or just one FTS index missing ── await adapter.initLbug(lbugPath); const ftsAvailable = await adapter.loadFTSExtension(undefined, { @@ -118,6 +120,10 @@ describe('warm MCP session observes an in-place --repair-fts rebuild (#2767)', ( await adapter.executeQuery( `CREATE (n:Function {id: 'func:login', name: 'login', filePath: 'src/auth.ts', startLine: 1, endLine: 3, content: 'function login() { return true; }'})`, ); + if (missing === 'Function') { + await createSearchFTSIndexes(); + await adapter.dropFTSIndex('Function', 'function_fts'); + } await adapter.flushWAL(); await adapter.closeLbug(); @@ -129,7 +135,10 @@ describe('warm MCP session observes an in-place --repair-fts rebuild (#2767)', ( stats: { files: 1, nodes: 1 }, capabilities: { graph: { provider: 'ladybugdb', status: 'available' }, - fts: { provider: 'ladybugdb-fts', status: 'unavailable' }, + fts: { + provider: 'ladybugdb-fts', + status: missing === 'all' ? 'unavailable' : 'available', + }, vectorSearch: { provider: 'exact-scan', status: 'unavailable', exactScanLimit: 0 }, }, }; @@ -142,6 +151,10 @@ describe('warm MCP session observes an in-place --repair-fts rebuild (#2767)', ( const before = await backend.callTool('query', { query: 'login' }); expect(before.error).toBeUndefined(); expect(ftsMissing(before)).toBe(true); + if (missing === 'Function') { + expect(before.partial).toBe(true); + expect(before.warning).toContain('Function.function_fts'); + } // ── Step 3: out-of-band --repair-fts (separate writable session) ──── // Same production functions the repair-fts branch of runFullAnalysis @@ -163,6 +176,7 @@ describe('warm MCP session observes an in-place --repair-fts rebuild (#2767)', ( const after = await waitForFtsRecognized(backend, 'login'); expect(after.error).toBeUndefined(); expect(ftsMissing(after)).toBe(false); + expect(after.partial).toBeUndefined(); expect(matchedIds(after)).toContain('func:login'); }, ); diff --git a/gitnexus/test/integration/impact-context-integrity.test.ts b/gitnexus/test/integration/impact-context-integrity.test.ts new file mode 100644 index 000000000..0e42de484 --- /dev/null +++ b/gitnexus/test/integration/impact-context-integrity.test.ts @@ -0,0 +1,758 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import * as adapter from '../../src/core/lbug/lbug-adapter.js'; +import { executeParameterized } from '../../src/core/lbug/pool-adapter.js'; +import { closeQueryResults } from '../../src/core/lbug/query-result-utils.js'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { retryRename } from '../../src/storage/fs-atomic.js'; +import { getStoragePaths, registerRepo, saveMeta } from '../../src/storage/repo-manager.js'; +import { createTempDir } from '../helpers/test-db.js'; + +const REPO = 'query-integrity'; +const nodes = { + alpha: { id: 'Function:src/alpha.ts:runSweep', name: 'runSweep', filePath: 'src/alpha.ts' }, + alphaCaller: { + id: 'Function:src/α/caller.ts:appelÉ', + name: 'appelÉ', + filePath: 'src/α/caller.ts', + }, + alphaOtherCaller: { + id: 'Function:src/other.ts:runOther', + name: 'runOther', + filePath: 'src/other.ts', + }, + alphaRoot: { id: 'Function:src/root.ts:startSweep', name: 'startSweep', filePath: 'src/root.ts' }, + alphaReader: { + id: 'Function:src/reader.ts:readSweep', + name: 'readSweep', + filePath: 'src/reader.ts', + }, + beta: { + id: 'Function:src/beta.ts:extractLeadingNumber', + name: 'extractLeadingNumber', + filePath: 'src/beta.ts', + }, + betaCaller: { + id: 'Function:src/number.ts:parseNumber', + name: 'parseNumber', + filePath: 'src/number.ts', + }, + betaReader: { + id: 'Function:src/number-view.ts:readNumber', + name: 'readNumber', + filePath: 'src/number-view.ts', + }, +} as const; + +const processes = { + alpha: { + id: 'process:alpha', + label: 'Sweep flow', + entry: nodes.alphaRoot, + terminal: nodes.alpha, + stepCount: 3, + }, + alphaOther: { + id: 'process:alpha-other', + label: 'Other sweep flow', + entry: nodes.alphaOtherCaller, + terminal: nodes.alpha, + stepCount: 2, + }, + beta: { + id: 'process:beta', + label: 'Number flow', + entry: nodes.betaCaller, + terminal: nodes.beta, + stepCount: 2, + }, +} as const; + +type NodeIdentity = { id: string; name: string; filePath: string }; +type Membership = { id: string; label: string; processType: string; step: number }; +type ImpactRow = NodeIdentity & { + relationType: string; + confidence: number; + processes: Membership[]; +}; +type ContextRef = { uid: string; name: string; filePath: string }; +type Target = 'alpha' | 'beta'; + +const membership = ( + process: (typeof processes)[keyof typeof processes], + step: number, +): Membership => ({ + id: process.id, + label: process.label, + processType: 'intra_community', + step, +}); + +const affectedProcess = (process: (typeof processes)[keyof typeof processes], hits: number) => ({ + name: process.entry.name, + type: 'Function', + filePath: process.entry.filePath, + affected_process_count: 1, + total_hits: hits, + earliest_broken_step: 0, +}); + +const oracle = { + alpha: { + count: 3, + direct: 2, + byDepth: { + 1: [ + { + ...nodes.alphaOtherCaller, + relationType: 'CALLS', + confidence: 1, + processes: [membership(processes.alphaOther, 0)], + }, + { + ...nodes.alphaCaller, + relationType: 'CALLS', + confidence: 1, + processes: [membership(processes.alpha, 1)], + }, + ], + 2: [ + { + ...nodes.alphaRoot, + relationType: 'CALLS', + confidence: 1, + processes: [membership(processes.alpha, 0)], + }, + ], + }, + callers: [nodes.alphaOtherCaller, nodes.alphaCaller], + accesses: [nodes.alphaReader], + processes: [ + { id: processes.alpha.id, name: processes.alpha.label, step_index: 2, step_count: 3 }, + { + id: processes.alphaOther.id, + name: processes.alphaOther.label, + step_index: 1, + step_count: 2, + }, + ], + affectedProcesses: [ + affectedProcess(processes.alpha, 2), + affectedProcess(processes.alphaOther, 1), + ], + }, + beta: { + count: 1, + direct: 1, + byDepth: { + 1: [ + { + ...nodes.betaCaller, + relationType: 'CALLS', + confidence: 1, + processes: [membership(processes.beta, 0)], + }, + ], + }, + callers: [nodes.betaCaller], + accesses: [nodes.betaReader], + processes: [ + { id: processes.beta.id, name: processes.beta.label, step_index: 1, step_count: 2 }, + ], + affectedProcesses: [affectedProcess(processes.beta, 1)], + }, +} as const; + +// Compare the values returned by the native engine with a hand-written graph +// oracle, rather than accepting a repeated (and potentially wrong) first result. +function expectImpact( + result: Awaited>, + target: Target, + summaryOnly: boolean, +): void { + const expected = oracle[target]; + expect(result).not.toHaveProperty('error'); + expect(result).not.toHaveProperty('partial'); + expect(result.target).toMatchObject(nodes[target]); + expect(result.direction).toBe('upstream'); + expect(result.impactedCount).toBe(expected.count); + expect(result.risk).toBe('LOW'); + expect(result.epistemic).toBe('exact'); + expect(result.summary).toEqual({ + direct: expected.direct, + processes_affected: expected.affectedProcesses.length, + modules_affected: 0, + }); + expect(result.byDepthCounts).toEqual(target === 'alpha' ? { 1: 2, 2: 1 } : { 1: 1 }); + expect(result.affected_processes).toEqual(expected.affectedProcesses); + expect(result.affected_modules).toEqual([]); + expect(result.affected_routes).toEqual([]); + if (summaryOnly) { + expect(result).not.toHaveProperty('byDepth'); + } else { + const byDepth = Object.fromEntries( + Object.entries(result.byDepth).map(([depth, rows]) => [ + depth, + (rows as ImpactRow[]).map( + ({ id, name, filePath, relationType, confidence, processes: memberships }) => ({ + id, + name, + filePath, + relationType, + confidence, + processes: memberships, + }), + ), + ]), + ); + expect(byDepth).toEqual(expected.byDepth); + } +} + +function expectContext( + result: Awaited>, + target: Target, +): void { + const expected = oracle[target]; + expect(result).not.toHaveProperty('error'); + expect(result.status).toBe('found'); + expect(result.symbol).toMatchObject({ + uid: nodes[target].id, + name: nodes[target].name, + filePath: nodes[target].filePath, + }); + expect(result.epistemic).toBe('exact'); + const incoming = Object.fromEntries( + Object.entries(result.incoming).map(([type, refs]) => [ + type, + (refs as ContextRef[]).map(({ uid, name, filePath }) => ({ id: uid, name, filePath })), + ]), + ); + expect(incoming).toEqual({ calls: expected.callers, accesses: expected.accesses }); + expect(result.outgoing).toEqual({}); + expect(result.processes).toEqual(expected.processes); +} + +function edge(source: NodeIdentity, target: NodeIdentity, type: 'CALLS' | 'ACCESSES'): string { + return `MATCH (a:Function {id: '${source.id}'}), (b:Function {id: '${target.id}'}) CREATE (a)-[:CodeRelation {type: '${type}', confidence: 1.0, reason: 'direct', step: 0}]->(b)`; +} + +function processStep( + node: NodeIdentity, + process: (typeof processes)[keyof typeof processes], + step: number, +): string { + return `MATCH (n:Function {id: '${node.id}'}), (p:Process {id: '${process.id}'}) CREATE (n)-[:CodeRelation {type: 'STEP_IN_PROCESS', confidence: 1.0, reason: 'trace-detection', step: ${step}}]->(p)`; +} + +describe('native impact/context result integrity (#3354)', () => { + let temp: Awaited>; + let backend: LocalBackend; + let lbugPath: string; + + beforeAll(async () => { + temp = await createTempDir(); + vi.stubEnv('GITNEXUS_HOME', path.join(temp.dbPath, 'home')); + vi.stubEnv('GITNEXUS_STORAGE_PATH', path.join(temp.dbPath, 'index')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + const paths = getStoragePaths(temp.dbPath); + lbugPath = paths.lbugPath; + + // Close the writer before LocalBackend opens its ordinary read pool. No + // mocked registry or injected writable Database bypasses the read path. + await adapter.initLbug(lbugPath); + try { + const seed = [ + ...Object.values(nodes).map( + (node) => + `CREATE (:Function {id: '${node.id}', name: '${node.name}', filePath: '${node.filePath}', startLine: 1, endLine: 3})`, + ), + ...Object.values(processes).map( + (process) => + `CREATE (:Process {id: '${process.id}', label: '${process.label}', heuristicLabel: '${process.label}', processType: 'intra_community', stepCount: ${process.stepCount}, communities: [], entryPointId: '${process.entry.id}', terminalId: '${process.terminal.id}'})`, + ), + edge(nodes.alphaCaller, nodes.alpha, 'CALLS'), + edge(nodes.alphaOtherCaller, nodes.alpha, 'CALLS'), + edge(nodes.alphaRoot, nodes.alphaCaller, 'CALLS'), + edge(nodes.alphaReader, nodes.alpha, 'ACCESSES'), + edge(nodes.betaCaller, nodes.beta, 'CALLS'), + edge(nodes.betaReader, nodes.beta, 'ACCESSES'), + processStep(nodes.alphaRoot, processes.alpha, 0), + processStep(nodes.alphaCaller, processes.alpha, 1), + processStep(nodes.alpha, processes.alpha, 2), + processStep(nodes.alphaOtherCaller, processes.alphaOther, 0), + processStep(nodes.alpha, processes.alphaOther, 1), + processStep(nodes.betaCaller, processes.beta, 0), + processStep(nodes.beta, processes.beta, 1), + ]; + for (const query of seed) await adapter.executeQuery(query); + await adapter.flushWAL(); + } finally { + await adapter.closeLbug(); + } + const meta = { + repoPath: temp.dbPath, + storagePath: paths.storagePath, + lastCommit: 'integrity-fixture', + indexedAt: new Date().toISOString(), + scopeExtractionReceipt: 1 as const, + stats: { files: 8, nodes: 11, processes: 3, communities: 0 }, + }; + await saveMeta(paths.storagePath, meta); + await registerRepo(temp.dbPath, meta, { name: REPO }); + backend = new LocalBackend(); + expect(await backend.init()).toBe(true); + }); + + afterAll(async () => { + try { + await backend?.dispose(); + } finally { + await adapter.closeLbug(); + vi.unstubAllEnvs(); + await temp?.cleanup(); + } + }); + + const impact = (target: Target, summaryOnly = false) => + backend.callTool('impact', { + repo: REPO, + target: nodes[target].name, + direction: 'upstream', + summaryOnly, + }); + const context = (target: Target) => + backend.callTool('context', { repo: REPO, uid: nodes[target].id }); + + it('returns exact values across identical sequential requests', async () => { + for (let repeat = 0; repeat < 6; repeat++) { + expectImpact(await impact('alpha', true), 'alpha', true); + expectContext(await context('alpha'), 'alpha'); + expectImpact(await impact('alpha'), 'alpha', false); + } + }); + + it('keeps unrelated targets isolated across mixed requests', async () => { + for (const target of ['alpha', 'beta', 'beta', 'alpha'] as const) { + expectImpact(await impact(target, true), target, true); + expectContext(await context(target), target); + expectImpact(await impact(target), target, false); + } + }); + + it('keeps mixed concurrent prepared reads symbol-specific', async () => { + for (let repeat = 0; repeat < 3; repeat++) { + const results = await Promise.all([ + impact('alpha', true), + context('beta'), + impact('beta'), + impact('beta', true), + context('alpha'), + impact('alpha'), + ]); + expectImpact(results[0], 'alpha', true); + expectContext(results[1], 'beta'); + expectImpact(results[2], 'beta', false); + expectImpact(results[3], 'beta', true); + expectContext(results[4], 'alpha'); + expectImpact(results[5], 'alpha', false); + } + }); + + it('returns exact relation rows directly from the pooled prepared adapter', async () => { + // Warm the pool through the same backend, then check the native row + // boundary independently of the tool's normalization and aggregation. + expectContext(await context('alpha'), 'alpha'); + const read = (target: Target) => + executeParameterized( + lbugPath, + ` + MATCH (caller:Function)-[r:CodeRelation]->(target:Function {id: $id}) + WHERE r.type IN ['CALLS', 'ACCESSES'] + RETURN caller.id AS id, caller.name AS name, caller.filePath AS filePath, r.type AS relationType + ORDER BY id + `, + { id: nodes[target].id }, + ); + const expectedRows = (target: Target) => + [ + ...oracle[target].callers.map((caller) => ({ ...caller, relationType: 'CALLS' })), + ...oracle[target].accesses.map((reader) => ({ ...reader, relationType: 'ACCESSES' })), + ].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + for (let repeat = 0; repeat < 4; repeat++) { + expect(await read('alpha')).toEqual(expectedRows('alpha')); + const [beta, alpha] = await Promise.all([read('beta'), read('alpha')]); + expect(beta).toEqual(expectedRows('beta')); + expect(alpha).toEqual(expectedRows('alpha')); + } + }); +}); + +describe('native string projections after checkpointed deletion (#3354)', () => { + it('keeps long symbol identities associated with their source rows across segments', async () => { + const temp = await createTempDir(); + const db = new lbug.Database(path.join(temp.dbPath, 'scan.lbug'), 128 * 1024 * 1024); + const conn = new lbug.Connection(db, 4); + const source = Array.from({ length: 10_000 }, (_, startLine) => ({ + id: `Function:src/generated/rené-${String(startLine).padStart(5, '0')}.ts:fn${startLine}`, + name: `generated_function_${startLine}_é`, + filePath: `src/generated/rené-${String(startLine).padStart(5, '0')}.ts`, + startLine, + })); + const projection = + 'RETURN n.id AS id, n.name AS name, n.filePath AS filePath, n.startLine AS startLine'; + const read = async (query: string) => { + const result = await conn.query(query); + try { + const cursor = Array.isArray(result) ? result[0] : result; + return await cursor.getAll(); + } finally { + await closeQueryResults(result); + } + }; + + try { + await read( + 'CREATE NODE TABLE Function(id STRING, name STRING, filePath STRING, startLine INT64, PRIMARY KEY(id))', + ); + // Separate checkpoints create segment boundaries inside scan vectors. + // LadybugDB 0.18.3's filtered STRING scan could retain another row's + // printable identities here (LadybugDB/ladybug#678, fixed by #737). + for (let batch = 0; batch < 4; batch++) { + const csvPath = path.join(temp.dbPath, `rows-${batch}.csv`); + const csv = source + .slice(batch * 2500, (batch + 1) * 2500) + .map((row) => + Object.values(row) + .map((value) => JSON.stringify(value)) + .join(','), + ) + .join('\n'); + await fs.writeFile(csvPath, `${csv}\n`); + await read( + `COPY Function FROM ${JSON.stringify(csvPath.replaceAll('\\', '/'))} (HEADER=false)`, + ); + await read('CHECKPOINT'); + } + expect(await read(`MATCH (n:Function) ${projection} ORDER BY n.startLine`)).toEqual(source); + + await read( + 'MATCH (n:Function) WHERE n.startLine >= 3000 AND n.startLine < 3400 DETACH DELETE n', + ); + await read('CHECKPOINT'); + const surviving = source.filter((row) => row.startLine < 3000 || row.startLine >= 3400); + for (let repeat = 0; repeat < 3; repeat++) { + for (const order of ['', ' ORDER BY n.startLine']) { + const rows = await read(`MATCH (n:Function) ${projection}${order}`); + expect(new Set(rows.map((row) => row.id)).size).toBe(surviving.length); + expect(rows.sort((a, b) => a.startLine - b.startLine)).toEqual(surviving); + } + } + // Point lookups independently verify values in the affected segments; + // a repeatably wrong scan must never become the test's reference answer. + for (const startLine of [1600, 7486]) { + expect( + await read(`MATCH (n:Function {id: '${source[startLine].id}'}) ${projection}`), + ).toEqual([source[startLine]]); + } + expect(await read(`MATCH (n:Function {id: '${source[3000].id}'}) ${projection}`)).toEqual([]); + } finally { + try { + await conn.close(); + } finally { + try { + await db.close(); + } finally { + await temp.cleanup(); + } + } + } + }); +}); + +// Windows graph replacement is opt-in in production. The repeated-read +// characterization above remains enabled there; only this POSIX swap is skipped. +describe.skipIf(process.platform === 'win32')('warm backend index replacement (#3354)', () => { + it('reads changed callers, processes and a new symbol through the real freshness window', async () => { + const temp = await createTempDir(); + let backend: LocalBackend | undefined; + vi.stubEnv('GITNEXUS_HOME', path.join(temp.dbPath, 'home')); + vi.stubEnv('GITNEXUS_STORAGE_PATH', path.join(temp.dbPath, 'index')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + const paths = getStoragePaths(temp.dbPath); + const stagedPath = `${paths.lbugPath}.replacement`; + const replacement = { + entry: { + id: 'Function:src/replacement-entry.ts:startReplacement', + name: 'startReplacement', + filePath: 'src/replacement-entry.ts', + }, + caller: { + id: 'Function:src/replacement-caller.ts:callReplacement', + name: 'callReplacement', + filePath: 'src/replacement-caller.ts', + }, + reader: { + id: 'Function:src/replacement-reader.ts:readReplacement', + name: 'readReplacement', + filePath: 'src/replacement-reader.ts', + }, + }; + const nextProcess = { id: 'process:replacement', label: 'Replacement flow' }; + const oldProcess = processes.alphaOther; + + const seed = async (dbPath: string, next: boolean) => { + await adapter.initLbug(dbPath); + try { + const caller = next ? replacement.caller : nodes.alphaOtherCaller; + const reader = next ? replacement.reader : nodes.alphaReader; + const process = next ? nextProcess : oldProcess; + const entry = next ? replacement.entry : caller; + for (const node of [nodes.alpha, caller, reader, ...(next ? [entry] : [])]) { + await adapter.executeQuery( + `CREATE (:Function {id: '${node.id}', name: '${node.name}', filePath: '${node.filePath}', startLine: 1, endLine: 3})`, + ); + } + await adapter.executeQuery( + `CREATE (:Process {id: '${process.id}', label: '${process.label}', heuristicLabel: '${process.label}', processType: 'intra_community', stepCount: ${next ? 3 : 2}, communities: [], entryPointId: '${entry.id}', terminalId: '${nodes.alpha.id}'})`, + ); + await adapter.executeQuery(edge(caller, nodes.alpha, 'CALLS')); + await adapter.executeQuery(edge(reader, nodes.alpha, 'ACCESSES')); + if (next) await adapter.executeQuery(edge(entry, caller, 'CALLS')); + const steps = next ? [entry, caller, nodes.alpha] : [caller, nodes.alpha]; + for (const [step, node] of steps.entries()) { + await adapter.executeQuery( + `MATCH (n:Function {id: '${node.id}'}), (p:Process {id: '${process.id}'}) CREATE (n)-[:CodeRelation {type: 'STEP_IN_PROCESS', confidence: 1.0, reason: 'trace-detection', step: ${step}}]->(p)`, + ); + } + await adapter.flushWAL(); + } finally { + await adapter.closeLbug(); + } + }; + const processMembership = (next: boolean, step: number) => ({ + ...(next ? nextProcess : { id: oldProcess.id, label: oldProcess.label }), + processType: 'intra_community', + step, + }); + const expectedCaller = (node: NodeIdentity, next: boolean, step: number) => ({ + ...node, + relationType: 'CALLS', + confidence: 1, + processes: [processMembership(next, step)], + }); + const expectGeneration = ( + impact: Awaited>, + context: Awaited>, + next: boolean, + ) => { + const caller = next ? replacement.caller : nodes.alphaOtherCaller; + const reader = next ? replacement.reader : nodes.alphaReader; + const entry = next ? replacement.entry : caller; + const process = next ? nextProcess : oldProcess; + expect(impact).not.toHaveProperty('error'); + expect(impact).not.toHaveProperty('partial'); + expect(impact.target).toMatchObject(nodes.alpha); + expect(impact.risk).toBe('LOW'); + expect(impact.epistemic).toBe('exact'); + expect(impact.impactedCount).toBe(next ? 2 : 1); + expect(impact.summary).toEqual({ direct: 1, processes_affected: 1, modules_affected: 0 }); + expect(impact.byDepthCounts).toEqual(next ? { 1: 1, 2: 1 } : { 1: 1 }); + const byDepth = Object.fromEntries( + Object.entries(impact.byDepth).map(([depth, rows]) => [ + depth, + (rows as ImpactRow[]).map( + ({ id, name, filePath, relationType, confidence, processes: memberships }) => ({ + id, + name, + filePath, + relationType, + confidence, + processes: memberships, + }), + ), + ]), + ); + expect(byDepth).toEqual({ + 1: [expectedCaller(caller, next, next ? 1 : 0)], + ...(next ? { 2: [expectedCaller(entry, true, 0)] } : {}), + }); + expect(impact.affected_processes).toEqual([ + { + name: entry.name, + type: 'Function', + filePath: entry.filePath, + affected_process_count: 1, + total_hits: next ? 2 : 1, + earliest_broken_step: 0, + }, + ]); + expect(impact.affected_modules).toEqual([]); + expect(impact.affected_routes).toEqual([]); + expect(context).not.toHaveProperty('error'); + expect(context.status).toBe('found'); + expect(context.epistemic).toBe('exact'); + expect(context.symbol).toMatchObject({ + uid: nodes.alpha.id, + name: nodes.alpha.name, + filePath: nodes.alpha.filePath, + }); + expect( + Object.fromEntries( + Object.entries(context.incoming).map(([type, refs]) => [ + type, + (refs as ContextRef[]).map(({ uid, name, filePath }) => ({ id: uid, name, filePath })), + ]), + ), + ).toEqual({ calls: [caller], accesses: [reader] }); + expect(context.outgoing).toEqual({}); + expect(context.processes).toEqual([ + { + id: process.id, + name: process.label, + step_index: next ? 2 : 1, + step_count: next ? 3 : 2, + }, + ]); + }; + + try { + await seed(paths.lbugPath, false); + const meta = { + repoPath: temp.dbPath, + storagePath: paths.storagePath, + lastCommit: 'graph-a', + indexedAt: new Date().toISOString(), + scopeExtractionReceipt: 1 as const, + stats: { files: 3, nodes: 4, processes: 1, communities: 0 }, + }; + await saveMeta(paths.storagePath, meta); + await registerRepo(temp.dbPath, meta, { name: REPO }); + const heldBackend = new LocalBackend(); + backend = heldBackend; + expect(await heldBackend.init()).toBe(true); + const impact = () => + heldBackend.callTool('impact', { + repo: REPO, + target: nodes.alpha.name, + direction: 'upstream', + }); + const context = () => heldBackend.callTool('context', { repo: REPO, uid: nodes.alpha.id }); + expectGeneration(await impact(), await context(), false); + expect( + await heldBackend.callTool('context', { repo: REPO, uid: replacement.entry.id }), + ).toHaveProperty('error'); + + // Keep this backend and its read pool alive. Publish only after the + // separate staged writer has closed, exactly as run-analyze does. + await seed(stagedPath, true); + for (const suffix of ['.wal', '.shadow', '.wal.checkpoint']) { + await expect(fs.stat(`${stagedPath}${suffix}`)).rejects.toMatchObject({ code: 'ENOENT' }); + } + await retryRename(stagedPath, paths.lbugPath); + const nextMeta = { + ...meta, + lastCommit: 'graph-b', + indexedAt: new Date(Date.now() + 1).toISOString(), + stats: { files: 4, nodes: 5, processes: 1, communities: 0 }, + }; + await saveMeta(paths.storagePath, nextMeta); + await registerRepo(temp.dbPath, nextMeta, { name: REPO }); + + // An independent native read-only Database opens the published path. + // It does not share LocalBackend's pool or trigger its reinitialization. + const freshDb = new lbug.Database(paths.lbugPath, 128 * 1024 * 1024, true, true); + const freshConn = new lbug.Connection(freshDb); + try { + const read = async (query: string) => { + const result = await freshConn.query(query); + try { + const cursor = Array.isArray(result) ? result[0] : result; + return await cursor.getAll(); + } finally { + await closeQueryResults(result); + } + }; + expect( + await read(` + MATCH (n:Function) + RETURN n.id AS id, n.name AS name, n.filePath AS filePath + ORDER BY id + `), + ).toEqual( + [nodes.alpha, ...Object.values(replacement)].sort((a, b) => + a.id < b.id ? -1 : a.id > b.id ? 1 : 0, + ), + ); + expect( + await read(` + MATCH (n:Function)-[r:CodeRelation]->(target:Function {id: '${nodes.alpha.id}'}) + WHERE r.type IN ['CALLS', 'ACCESSES'] + RETURN n.id AS id, n.name AS name, n.filePath AS filePath, r.type AS relationType + ORDER BY id + `), + ).toEqual([ + { ...replacement.caller, relationType: 'CALLS' }, + { ...replacement.reader, relationType: 'ACCESSES' }, + ]); + expect( + await read(` + MATCH (n:Function)-[r:CodeRelation {type: 'STEP_IN_PROCESS'}]->(p:Process) + RETURN n.id AS id, p.id AS processId, p.heuristicLabel AS label, + r.step AS step, p.stepCount AS stepCount, p.entryPointId AS entryPointId + ORDER BY step + `), + ).toEqual( + [replacement.entry, replacement.caller, nodes.alpha].map((node, step) => ({ + id: node.id, + processId: nextProcess.id, + label: nextProcess.label, + step, + stepCount: 3, + entryPointId: replacement.entry.id, + })), + ); + } finally { + await freshConn.close(); + await freshDb.close(); + } + + // Poll the SAME backend through its unchanged five-second throttle. + // No private watermark override, poolInit, reset or restart is used. + const deadline = Date.now() + 15_000; + let refreshed = await context(); + while (refreshed.processes?.[0]?.id !== nextProcess.id && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 300)); + refreshed = await context(); + } + expectGeneration(await impact(), refreshed, true); + for (let repeat = 0; repeat < 3; repeat++) { + expectGeneration(await impact(), await context(), true); + const introduced = await heldBackend.callTool('context', { + repo: REPO, + name: replacement.entry.name, + }); + expect(introduced).not.toHaveProperty('error'); + expect(introduced.status).toBe('found'); + expect(introduced.symbol).toMatchObject({ + uid: replacement.entry.id, + name: replacement.entry.name, + filePath: replacement.entry.filePath, + }); + expect(introduced.processes).toEqual([ + { id: nextProcess.id, name: nextProcess.label, step_index: 0, step_count: 3 }, + ]); + } + } finally { + try { + await backend?.dispose(); + } finally { + await adapter.closeLbug(); + vi.unstubAllEnvs(); + await temp.cleanup(); + } + } + }); +}); diff --git a/gitnexus/test/integration/lbug-core-adapter.test.ts b/gitnexus/test/integration/lbug-core-adapter.test.ts index 814fd60a4..89dd1c8a8 100644 --- a/gitnexus/test/integration/lbug-core-adapter.test.ts +++ b/gitnexus/test/integration/lbug-core-adapter.test.ts @@ -11,6 +11,8 @@ import { describe, it, expect } from 'vitest'; import fs from 'fs/promises'; import path from 'path'; +import { PassThrough } from 'node:stream'; +import type { Response } from 'express'; import type { GraphRelationship } from 'gitnexus-shared'; import { withTestLbugDB } from '../helpers/test-indexed-db.js'; import { skipUnlessFtsAvailable } from '../helpers/fts-availability.js'; @@ -61,6 +63,39 @@ withTestLbugDB( expect(folderRows).toHaveLength(1); }); + it('serializes CSV-loaded empty relationship reasons as strings in both graph APIs', async () => { + const { executeQuery } = await import('../../src/core/lbug/lbug-adapter.js'); + const { buildGraph, streamGraphNdjson } = await import('../../src/server/api.js'); + const stored = await executeQuery( + 'MATCH ()-[r:CodeRelation]->() RETURN r.reason AS reason', + ); + expect(stored).toEqual(Array.from({ length: 4 }, () => ({ reason: null }))); + + const buffered = JSON.parse(JSON.stringify(await buildGraph())); + const response = new PassThrough(); + let ndjson = ''; + response.on('data', (chunk: Buffer) => { + ndjson += chunk.toString(); + }); + try { + await streamGraphNdjson(response as unknown as Response); + } finally { + response.destroy(); + } + const streamed = ndjson + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + .filter((record) => record.type === 'relationship') + .map((record) => record.data); + expect(buffered.relationships).toHaveLength(4); + expect({ + buffered: buffered.relationships.map((rel: GraphRelationship) => rel.reason), + streamed: streamed.map((rel: GraphRelationship) => rel.reason), + }).toEqual({ buffered: ['', '', '', ''], streamed: ['', '', '', ''] }); + expect(streamed).toEqual(expect.arrayContaining(buffered.relationships)); + }); + it('createFTSIndex: creates FTS index on Function table without error', async (ctx) => { await skipUnlessFtsAvailable(ctx); const { createFTSIndex } = await import('../../src/core/lbug/lbug-adapter.js'); diff --git a/gitnexus/test/integration/lbug-interrupted-checkpoint-recovery.test.ts b/gitnexus/test/integration/lbug-interrupted-checkpoint-recovery.test.ts index d2dea28cf..287f2333b 100644 --- a/gitnexus/test/integration/lbug-interrupted-checkpoint-recovery.test.ts +++ b/gitnexus/test/integration/lbug-interrupted-checkpoint-recovery.test.ts @@ -20,58 +20,28 @@ * and the behavioral contract is held on every pin by the mocked * forced-refusal suites instead. */ -import { afterAll, describe, expect, it } from 'vitest'; +import { afterAll, describe, expect, it, vi } from 'vitest'; +import { spawnSync } from 'node:child_process'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { closeLbug, executeQuery, initLbug } from '../../src/core/lbug/pool-adapter.js'; import lbug from '@ladybugdb/core'; +import { closeQueryResults } from '../../src/core/lbug/query-result-utils.js'; const REPO = 'test-interrupted-checkpoint'; const ROWS = 300; /** - * Windows: the native close() resolves before the kernel releases the file's - * handles and byte-range locks — the next open then dies with Win32 Error 33 - * ("another process has locked a portion of the file"), which is exactly how - * this fixture failed its first hosted run. Probe-read both the db and its - * residual WAL until the engine's locks are gone. Bounded, so a real handle - * leak fails loudly instead of hanging; a pass-through on POSIX (first probe - * always succeeds). - */ -async function waitForFixtureRelease(dbPath: string): Promise { - for (const target of [dbPath, `${dbPath}.wal`]) { - for (let attempt = 0; ; attempt++) { - try { - const fh = await fs.open(target, 'r'); - try { - await fh.read(Buffer.alloc(1), 0, 1, 0); - } finally { - await fh.close(); - } - break; - } catch (err) { - if ((err as NodeJS.ErrnoException).code === 'ENOENT') break; // nothing planted there - if (attempt >= 40) throw err; // ~6s of retries: report the leak - await new Promise((resolve) => setTimeout(resolve, 150)); - } - } - } -} - -/** - * Deterministic interrupted-checkpoint signature: build rows on a writable - * session with AUTO-CHECKPOINT DISABLED and close WITHOUT checkpointing, so — - * exactly like a CHECKPOINT killed mid-flight — the main file is stale and - * every row lives only in the WAL. Then rename that WAL to the - * `lbug.wal.checkpoint` name the engine gives it during checkpoint, and plant - * the shadow + intent/apply lock files it leaves behind. + * Preserve the main file and WAL before native close forces a checkpoint. + * Restoring both snapshots models a killed checkpoint with every row still + * WAL-only. All query results must be closed before releasing the database: + * native results retain the database and its writer lock until closed or GC'd. */ async function plantInterruptedCheckpoint(dbPath: string): Promise { - // Raw constructor (positional args mirror createLbugDatabase) because the - // autoCheckpoint toggle is not exposed through the config helpers — and - // auto-checkpoint-on-close is precisely what must NOT happen here. + // Disable automatic checkpoints while building the WAL. Native close still + // forces a checkpoint, so the pre-close main file must also be preserved. const db = new lbug.Database( dbPath, 128 * 1024 * 1024, // bufferManagerSize @@ -83,40 +53,61 @@ async function plantInterruptedCheckpoint(dbPath: string): Promise { false, // throwOnWalReplayFailure true, // enableChecksums ); - await db.init(); - const conn = new lbug.Connection(db); + let conn: lbug.Connection | undefined; + let mainBuffer: Buffer; + let walBuffer: Buffer; try { - await conn.query('CREATE NODE TABLE Person (name STRING, PRIMARY KEY(name))'); + await db.init(); + conn = new lbug.Connection(db); + const statements = ['CREATE NODE TABLE Person (name STRING, PRIMARY KEY(name))']; for (let i = 0; i < ROWS; i += 100) { const batch = Array.from({ length: 100 }, (_, j) => `{name: 'p${i + j}'}`).join(', '); - await conn.query(`UNWIND [${batch}] AS r CREATE (:Person {name: r.name})`); + statements.push(`UNWIND [${batch}] AS r CREATE (:Person {name: r.name})`); } - const walBuffer = await fs.readFile(`${dbPath}.wal`); - // Honesty check: the rows must actually LIVE in the WAL — on an engine - // that tolerates the planted state this is the only proof the plant is - // not an empty shell (review finding: unused walBuffer). + for (const statement of statements) { + const result = await conn.query(statement); + try { + for (const cursor of Array.isArray(result) ? result : [result]) await cursor.getAll(); + } finally { + await closeQueryResults(result); + } + } + [mainBuffer, walBuffer] = await Promise.all([ + fs.readFile(dbPath), + fs.readFile(`${dbPath}.wal`), + ]); expect(walBuffer.byteLength).toBeGreaterThan(0); - // Close WITHOUT checkpoint: rows stay WAL-only, main file stays stale. - // Explicitly awaited release BEFORE the rename/reopen — on Windows the - // kernel releases the engine's handles/locks asynchronously and the WAL - // rename + pooled reopen race them (Win32 Error 33, seen in CI). - await conn.close().catch(() => {}); + } finally { + await conn?.close().catch(() => {}); await db.close().catch(() => {}); - await waitForFixtureRelease(dbPath); - // Re-plant the captured WAL bytes rather than renaming the original: a - // close-time auto-checkpoint can consume the live .wal file out from - // under the rename (ENOENT — the fixture's other CI flake), while the - // captured buffer is what a killed checkpoint would have left behind. - await fs.writeFile(`${dbPath}.wal.checkpoint`, walBuffer); - await fs.writeFile(`${dbPath}.wal`, ''); - await fs.writeFile(`${dbPath}.shadow`, ''); - await fs.writeFile(`${dbPath}.checkpoint.intent.lock`, ''); - await fs.writeFile(`${dbPath}.checkpoint.apply.lock`, ''); - } catch (err) { - await conn.close().catch(() => {}); - await db.close().catch(() => {}); - throw err; } + // A separate process must acquire the native writer lock before the + // fixture files are restored. A byte-zero file read cannot test that lock. + const probe = spawnSync( + process.execPath, + [ + '-e', + `const lbug = require(process.argv[1]); + const db = new lbug.Database(process.argv[2], 128 * 1024 * 1024, false, + false, 16 * 1024 * 1024 * 1024); + db.init().then(() => db.close()).catch((error) => { + console.error(error); + process.exitCode = 1; + });`, + fileURLToPath(new URL('../../node_modules/@ladybugdb/core', import.meta.url)), + dbPath, + ], + { encoding: 'utf8', timeout: 15_000 }, + ); + expect(probe.error).toBeUndefined(); + expect(probe.status, probe.stderr).toBe(0); + + await fs.writeFile(dbPath, mainBuffer); + await fs.writeFile(`${dbPath}.wal.checkpoint`, walBuffer); + await fs.writeFile(`${dbPath}.wal`, ''); + await fs.writeFile(`${dbPath}.shadow`, ''); + await fs.writeFile(`${dbPath}.checkpoint.intent.lock`, ''); + await fs.writeFile(`${dbPath}.checkpoint.apply.lock`, ''); } describe('interrupted-checkpoint recovery (pooled read path self-heal)', () => { @@ -128,6 +119,38 @@ describe('interrupted-checkpoint recovery (pooled read path self-heal)', () => { if (tmpDir) await fs.rm(tmpDir, { recursive: true, force: true }); }); + it('preserves the setup error when both native closes reject', async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-lbug-cp-cleanup-')); + const setupError = new Error('injected query failure'); + const closeConnection = lbug.Connection.prototype.close; + const closeDatabase = lbug.Database.prototype.close; + const query = vi.spyOn(lbug.Connection.prototype, 'query').mockRejectedValueOnce(setupError); + const connectionClose = vi + .spyOn(lbug.Connection.prototype, 'close') + .mockImplementation(async function (this: lbug.Connection) { + await closeConnection.call(this); + throw new Error('injected connection close failure'); + }); + const databaseClose = vi + .spyOn(lbug.Database.prototype, 'close') + .mockImplementation(async function (this: lbug.Database) { + await closeDatabase.call(this); + throw new Error('injected database close failure'); + }); + try { + await expect(plantInterruptedCheckpoint(path.join(directory, 'lbug'))).rejects.toBe( + setupError, + ); + expect(connectionClose).toHaveBeenCalledOnce(); + expect(databaseClose).toHaveBeenCalledOnce(); + } finally { + query.mockRestore(); + connectionClose.mockRestore(); + databaseClose.mockRestore(); + await fs.rm(directory, { recursive: true, force: true }); + } + }); + it('opens read-only through the pool refusal and answers queries', async (ctx) => { // Engine-version honesty: only 0.19+ treats the planted signature as an // interrupted checkpoint ("Cannot open database in read-only mode while @@ -180,9 +203,6 @@ describe('interrupted-checkpoint recovery (pooled read path self-heal)', () => { } })(), ).rejects.toThrow(/checkpoint is in progress/i); - // The probe's native close is best-effort; its handles must be gone - // before the pooled open below (Windows Error 33 otherwise). - await waitForFixtureRelease(dbPath); // The wiki path: pooled READ-ONLY open. Before the fix this refused with // "Cannot open database in read-only mode while checkpoint is in @@ -212,7 +232,6 @@ describe('interrupted-checkpoint recovery (pooled read path self-heal)', () => { // A second open must answer without needing recovery again. await closeLbug(REPO); - await waitForFixtureRelease(dbPath); await initLbug(REPO, dbPath); const again = await executeQuery(REPO, 'MATCH (n:Person) RETURN count(n) AS c'); expect(again.length).toBe(1); diff --git a/gitnexus/test/integration/lbug-load-overlap-errors.test.ts b/gitnexus/test/integration/lbug-load-overlap-errors.test.ts index d72477a4e..f75c87e3a 100644 --- a/gitnexus/test/integration/lbug-load-overlap-errors.test.ts +++ b/gitnexus/test/integration/lbug-load-overlap-errors.test.ts @@ -81,6 +81,52 @@ afterAll(async () => { }); describe('loadGraphToLbug overlap error paths (#2226 F1)', () => { + it.each([1000, 0])( + 'retains COPY failures and rejects skipped rows with warning_limit=%i', + async (warningLimit) => { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + const { _captureLogger } = await import('../../src/core/logger.js'); + const graph = buildTestGraph([], []); + emitMock.mockImplementation( + async (_g: unknown, _r: unknown, dir: string, onNodes?: (n: NodeFiles) => void) => { + await fs.mkdir(dir, { recursive: true }); + const csvPath = path.join(dir, 'function.csv'); + await fs.writeFile( + csvPath, + 'id,name,filePath,startLine,endLine,isExported,content,description,convexEndpointFactory\n' + + '"Function:bad.ts:copy","copy","bad.ts",bad,2,false,"","",""\n', + ); + const nodeFiles = new Map([['Function', { csvPath, rows: 1 }]]) as NodeFiles; + onNodes?.(nodeFiles); + return { ...emptyResult(), nodeFiles }; + }, + ); + const capture = _captureLogger(); + try { + await adapter.executeQuery(`CALL warning_limit=${warningLimit}`); + await expect(adapter.loadGraphToLbug(graph, tmpBase, storagePath)).rejects.toThrow( + /skipped/i, + ); + const records = capture.records(); + expect(records.some((r) => /first COPY failure/i.test(String(r.msg)))).toBe(true); + expect(records).toContainEqual( + expect.objectContaining({ + expectedRows: 1, + copiedRows: 0, + skippedRows: 1, + retainedWarnings: warningLimit === 0 ? 0 : 1, + }), + ); + } finally { + try { + await adapter.executeQuery('CALL warning_limit=1000'); + } finally { + capture.restore(); + } + } + }, + ); + it('relationship-emit failure with node COPY in flight surfaces the emit error and leaks no unhandled rejection', async () => { const adapter = await import('../../src/core/lbug/lbug-adapter.js'); const graph = buildTestGraph( diff --git a/gitnexus/test/integration/local-backend-calltool.test.ts b/gitnexus/test/integration/local-backend-calltool.test.ts index f618a3c5b..35eabbdd4 100644 --- a/gitnexus/test/integration/local-backend-calltool.test.ts +++ b/gitnexus/test/integration/local-backend-calltool.test.ts @@ -956,3 +956,98 @@ withTestLbugDB( }, }, ); + +const PYTHON_METHOD_ID = 'Method:tests/test_supervisor.py:Supervisor.run'; +const PYTHON_CALLER_ID = 'Function:tests/test_supervisor.py:test_run'; +const SWIFT_METHOD_ID = 'Method:Sources/Supervisor.swift:Supervisor.run'; +const SWIFT_CALLER_ID = 'Constructor:Sources/Supervisor.swift:Supervisor.init'; + +withTestLbugDB( + 'symbol-identity-isolation-3424', + (handle) => { + describe('mixed Python/Swift symbol identity isolation (#3424)', () => { + let backend: LocalBackend; + + beforeAll(() => { + backend = (handle as typeof handle & { _backend: LocalBackend })._backend; + }); + + it.each(['name and file', 'UID'])('keeps Python context isolated by %s', async (lookup) => { + const params = + lookup === 'UID' + ? { uid: PYTHON_METHOD_ID } + : { name: 'run', file_path: 'tests/test_supervisor.py' }; + const result = await backend.callTool('context', params); + expect(result).not.toHaveProperty('error'); + expect(result.symbol.uid).toBe(PYTHON_METHOD_ID); + expect(result.incoming.calls.map((caller: { uid: string }) => caller.uid)).toEqual([ + PYTHON_CALLER_ID, + ]); + }); + + it.each(['name and file', 'UID'])('keeps Python impact isolated by %s', async (lookup) => { + const params = + lookup === 'UID' + ? { target_uid: PYTHON_METHOD_ID } + : { target: 'run', file_path: 'tests/test_supervisor.py' }; + const result = await backend.callTool('impact', { + ...params, + direction: 'upstream', + includeTests: true, + }); + expect(result).not.toHaveProperty('error'); + expect(result.target.id).toBe(PYTHON_METHOD_ID); + expect(result.impactedCount).toBe(1); + expect(result.byDepth[1].map((caller: { id: string }) => caller.id)).toEqual([ + PYTHON_CALLER_ID, + ]); + }); + + it('keeps the unrelated Swift constructor queryable', async () => { + const context = await backend.callTool('context', { uid: SWIFT_METHOD_ID }); + expect(context).not.toHaveProperty('error'); + expect(context.symbol.uid).toBe(SWIFT_METHOD_ID); + expect(context.incoming.calls.map((caller: { uid: string }) => caller.uid)).toEqual([ + SWIFT_CALLER_ID, + ]); + const impact = await backend.callTool('impact', { + target_uid: SWIFT_METHOD_ID, + direction: 'upstream', + includeTests: true, + }); + expect(impact).not.toHaveProperty('error'); + expect(impact.target.id).toBe(SWIFT_METHOD_ID); + expect(impact.impactedCount).toBe(1); + expect(impact.byDepth[1].map((caller: { id: string }) => caller.id)).toEqual([ + SWIFT_CALLER_ID, + ]); + }); + }); + }, + { + seed: [ + `CREATE (:Method {id: '${PYTHON_METHOD_ID}', name: 'run', filePath: 'tests/test_supervisor.py', startLine: 3, endLine: 5})`, + `CREATE (:Function {id: '${PYTHON_CALLER_ID}', name: 'test_run', filePath: 'tests/test_supervisor.py', startLine: 7, endLine: 9})`, + `CREATE (:Method {id: '${SWIFT_METHOD_ID}', name: 'run', filePath: 'Sources/Supervisor.swift', startLine: 3, endLine: 5})`, + `CREATE (:Constructor {id: '${SWIFT_CALLER_ID}', name: 'init', filePath: 'Sources/Supervisor.swift', startLine: 7, endLine: 9})`, + `MATCH (a:Function), (b:Method) WHERE a.id = '${PYTHON_CALLER_ID}' AND b.id = '${PYTHON_METHOD_ID}' CREATE (a)-[:CodeRelation {type: 'CALLS', confidence: 1.0, reason: 'direct', step: 0}]->(b)`, + `MATCH (a:Constructor), (b:Method) WHERE a.id = '${SWIFT_CALLER_ID}' AND b.id = '${SWIFT_METHOD_ID}' CREATE (a)-[:CodeRelation {type: 'CALLS', confidence: 1.0, reason: 'direct', step: 0}]->(b)`, + ], + poolAdapter: true, + afterSetup: async (handle) => { + vi.mocked(listRegisteredRepos).mockResolvedValue([ + { + name: 'mixed-language-repo', + path: '/mixed-language/repo', + storagePath: handle.tmpHandle.dbPath, + indexedAt: new Date().toISOString(), + lastCommit: 'abc123', + stats: { files: 2, nodes: 4, communities: 0, processes: 0 }, + }, + ]); + const backend = new LocalBackend(); + await backend.init(); + (handle as typeof handle & { _backend: LocalBackend })._backend = backend; + }, + }, +); diff --git a/gitnexus/test/integration/resolvers/swift.test.ts b/gitnexus/test/integration/resolvers/swift.test.ts index 5d3e0bd85..76b25a565 100644 --- a/gitnexus/test/integration/resolvers/swift.test.ts +++ b/gitnexus/test/integration/resolvers/swift.test.ts @@ -359,6 +359,95 @@ describe.skipIf(!swiftAvailable)('Swift protocol-extension implicit self (#3273) }); }); +describe.skipIf(!swiftAvailable)('Swift injected closure property call (#3425)', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo( + path.join(FIXTURES, 'swift-injected-closure-call'), + () => {}, + ); + }, 60000); + + it('does not resolve an injected closure call to an unrelated method', () => { + expect( + getNodesByLabelFull(result, 'Property').some( + (node) => node.name === 'clock' && node.properties.filePath === 'Caller.swift', + ), + ).toBe(true); + expect( + getNodesByLabelFull(result, 'Function').some( + (node) => node.name === 'clock' && node.properties.filePath === 'Helpers.swift', + ), + ).toBe(true); + const calls = getRelationships(result, 'CALLS').filter((c) => c.source === 'refreshValue'); + expect(calls.filter((c) => c.target === 'clock')).toEqual([]); + }); + + it('does not resolve an inherited closure property call to the unrelated method', () => { + const extendsEdges = getRelationships(result, 'EXTENDS'); + expect( + extendsEdges.some( + (edge) => edge.source === 'DerivedService' && edge.target === 'BaseService', + ), + ).toBe(true); + const calls = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'refreshInheritedValue', + ); + expect(calls.filter((c) => c.target === 'clock')).toEqual([]); + }); + + it('keeps inherited closure calls in extensions unlinked', () => { + expect( + getNodesByLabelFull(result, 'Function').some( + (node) => + node.name === 'refreshInheritedFromExtension' && + node.properties.filePath === 'Helpers.swift', + ), + ).toBe(true); + const calls = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'refreshInheritedFromExtension', + ); + expect(calls.filter((c) => c.target === 'clock')).toEqual([]); + }); + + it('keeps same-type closure calls in extensions unlinked', () => { + const calls = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'refreshOwnFromExtension', + ); + expect(calls.filter((c) => c.target === 'clock')).toEqual([]); + }); + + it('still resolves the concrete-type extension call', () => { + const calls = getRelationships(result, 'CALLS').filter((c) => c.source === 'runScenario'); + expect(calls.map((c) => c.rel.targetId)).toEqual([ + 'Function:Helpers.swift:Example.makeValue#1', + ]); + }); + + it('keeps a nested function that shadows the stored closure', () => { + const calls = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'refreshWithLocalClock', + ); + expect(calls.filter((c) => c.target === 'clock')).toHaveLength(1); + expect(calls.find((c) => c.target === 'clock')?.targetFilePath).toBe('Caller.swift'); + }); + + it('keeps a labeled method selected alongside a same-name property', () => { + const calls = getRelationships(result, 'CALLS').filter((c) => c.source === 'refreshLabeled'); + expect(calls.map((c) => c.target)).toContain('first'); + expect(calls.find((c) => c.target === 'first')?.targetFilePath).toBe('Caller.swift'); + }); + + it('keeps a derived method despite an inaccessible ancestor property', () => { + const calls = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'refreshPrivateAncestor', + ); + expect(calls.map((c) => c.target)).toContain('clock'); + expect(calls.find((c) => c.target === 'clock')?.targetFilePath).toBe('Helpers.swift'); + }); +}); + // --------------------------------------------------------------------------- // Constructor fallback: Swift constructors look like free function calls // (no `new` keyword). The resolver retries with constructor form when diff --git a/gitnexus/test/integration/resolvers/typescript-mcp-tools.test.ts b/gitnexus/test/integration/resolvers/typescript-mcp-tools.test.ts new file mode 100644 index 000000000..fba040759 --- /dev/null +++ b/gitnexus/test/integration/resolvers/typescript-mcp-tools.test.ts @@ -0,0 +1,283 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs'; +import os from 'node:os'; +import { + loadParseCache, + PARSE_CACHE_VERSION, + pruneCache, + saveParseCache, + type ParseCache, +} from '../../../src/storage/parse-cache.js'; +import { + getDurableParsedFileDir, + pruneAndSaveDurableParsedFileStore, +} from '../../../src/storage/parsedfile-store.js'; +import { + FIXTURES, + findDanglingEdges, + getNodesByLabel, + getNodesByLabelFull, + getRelationships, + runPipelineFromRepo, + type PipelineResult, +} from './helpers.js'; + +describe('JavaScript and TypeScript SDK tool registrations', () => { + let result: PipelineResult; + const unresolved = [ + 'inline_callback', + 'imported_callback', + 'parameter_callback', + 'mutable_callback', + 'reassigned_callback', + 'shadowed_callback', + 'alias_callback', + 'first_block_callback', + 'second_block_callback', + ]; + + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'typescript-mcp-tools'), () => {}); + }, 60000); + + it.each(['ts', 'js'])( + 'does not emit tools for a destructured method replacement in %s', + async (extension) => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-mcp-tool-write-')); + try { + fs.writeFileSync( + path.join(repo, `server.${extension}`), + ` + import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; + const replaced = new McpServer({ name: 'replaced', version: '1' }); + ({ registerTool: replaced.registerTool } = { registerTool: () => undefined }); + replaced.registerTool('fake', {}, () => ({ content: [] })); + const actual = new McpServer({ name: 'actual', version: '1' }); + actual.registerTool('real', {}, () => ({ content: [] })); + `, + ); + const pipeline = await runPipelineFromRepo(repo, () => {}, { workerPoolSize: 1 }); + expect(getNodesByLabel(pipeline, 'Tool')).toEqual(['real']); + expect(findDanglingEdges(pipeline, ['HANDLES_TOOL', 'ENTRY_POINT_OF'])).toEqual([]); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }, + ); + + it.each(['ts', 'js'])( + 'preserves namespace registrations through cold/warm %s parsing', + async (extension) => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-mcp-namespace-')); + const storageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-mcp-namespace-cache-')); + try { + fs.writeFileSync( + path.join(repo, `server.${extension}`), + ` + import * as SDK from '@modelcontextprotocol/sdk/server/mcp.js'; + const server = new SDK.McpServer({}); + function handleNamespace() { return { content: [] }; } + server.registerTool('namespace', { description: 'Namespace tool' }, handleNamespace); + const replaced = new SDK.McpServer({}); + ({ method: replaced.registerTool } = other); + replaced.registerTool('fake', {}, handleNamespace); + ${extension === 'ts' ? "function install(typed: SDK.McpServer) { typed.tool('typed_namespace', handleNamespace); }" : ''} + `, + ); + const cache: ParseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map(), + usedKeys: new Set(), + storagePath: storageDir, + onDiskKeys: new Set(), + }; + const cold = await runPipelineFromRepo(repo, () => {}, { + parseCache: cache, + workerPoolSize: 1, + }); + expect(cold.usedWorkerPool).toBe(true); + pruneCache(cache, cache.usedKeys); + const keys = await saveParseCache(storageDir, cache); + await pruneAndSaveDurableParsedFileStore( + getDurableParsedFileDir(storageDir), + PARSE_CACHE_VERSION, + new Set(keys), + ); + const warmCache = await loadParseCache(storageDir); + expect(warmCache).not.toBeNull(); + const warm = await runPipelineFromRepo(repo, () => {}, { + parseCache: warmCache!, + workerPoolSize: 1, + }); + expect(warm.usedWorkerPool).toBe(false); + for (const pipeline of [cold, warm]) { + expect(getNodesByLabel(pipeline, 'Tool')).toEqual( + extension === 'ts' ? ['namespace', 'typed_namespace'] : ['namespace'], + ); + expect( + getNodesByLabelFull(pipeline, 'Tool').find((tool) => tool.name === 'namespace') + ?.properties.description, + ).toBe('Namespace tool'); + expect( + getRelationships(pipeline, 'HANDLES_TOOL').filter( + (edge) => edge.target === 'namespace', + ), + ).toMatchObject([{ source: 'handleNamespace', sourceLabel: 'Function' }]); + expect(findDanglingEdges(pipeline, ['HANDLES_TOOL', 'ENTRY_POINT_OF'])).toEqual([]); + } + expect(getNodesByLabelFull(warm, 'Tool')).toEqual(getNodesByLabelFull(cold, 'Tool')); + expect(getRelationships(warm, 'HANDLES_TOOL')).toEqual( + getRelationships(cold, 'HANDLES_TOOL'), + ); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + fs.rmSync(storageDir, { recursive: true, force: true }); + } + }, + 120_000, + ); + + it('discovers ordinary server files alongside deduplicated object manifests', () => { + expect(getNodesByLabel(result, 'Tool')).toEqual( + [ + ...unresolved, + 'search-files', + 'read_file', + 'function_expression_tool', + 'js_ping', + 'manifest_tool', + ].sort(), + ); + const tools = new Map( + getNodesByLabelFull(result, 'Tool').map((tool) => [tool.name, tool.properties]), + ); + expect(tools.get('search-files')).toMatchObject({ + filePath: 'src/server.ts', + description: 'Search files', + }); + expect(tools.get('read_file')).toMatchObject({ + filePath: 'src/server.ts', + description: 'Read a file', + }); + expect(tools.get('js_ping')).toMatchObject({ + filePath: 'src/server.js', + description: 'Ping JavaScript', + }); + expect(tools.get('manifest_tool')).toMatchObject({ + filePath: 'src/tools.ts', + description: 'Existing object manifest', + }); + expect(tools.get('inline_callback')?.description).toBe(''); + }); + + it('uses actual emitted callable nodes for supported local handlers', () => { + const edges = getRelationships(result, 'HANDLES_TOOL'); + for (const [tool, handler] of [ + ['search-files', 'searchFiles'], + ['read_file', 'readFile'], + ['function_expression_tool', 'expressionHandler'], + ['js_ping', 'jsPing'], + ]) { + expect(edges.filter((edge) => edge.target === tool)).toMatchObject([ + { source: handler, sourceLabel: 'Function' }, + ]); + } + expect(findDanglingEdges(result, ['HANDLES_TOOL', 'ENTRY_POINT_OF'])).toEqual([]); + }); + + it('links each same-file named handler only to its own execution flow', () => { + const edges = getRelationships(result, 'ENTRY_POINT_OF').filter( + (edge) => edge.sourceLabel === 'Tool', + ); + for (const [tool, handler] of [ + ['search-files', 'searchFiles'], + ['read_file', 'readFile'], + ]) { + const flows = edges.filter((edge) => edge.source === tool); + expect(flows).toHaveLength(1); + const process = result.graph.getNode(flows[0].rel.targetId)!; + const entry = result.graph.getNode(process.properties.entryPointId as string)!; + expect(entry.properties.name).toBe(handler); + } + }); + + it('keeps unresolved callbacks at file attribution without unrelated same-file flows', () => { + const handles = getRelationships(result, 'HANDLES_TOOL'); + const flows = getRelationships(result, 'ENTRY_POINT_OF'); + expect( + getNodesByLabelFull(result, 'Process').some((process) => { + const entry = result.graph.getNode(process.properties.entryPointId as string); + return entry?.properties.name === 'unrelatedEntry'; + }), + ).toBe(true); + for (const name of unresolved) { + expect(handles.filter((edge) => edge.target === name)).toMatchObject([ + { sourceLabel: 'File', sourceFilePath: 'src/server.ts' }, + ]); + expect(flows.filter((edge) => edge.sourceLabel === 'Tool' && edge.source === name)).toEqual( + [], + ); + } + }); + + it('preserves tool metadata, handler identities, and flow attribution on warm replay', async () => { + const storageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-mcp-tools-cache-')); + try { + const cold: ParseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map(), + usedKeys: new Set(), + storagePath: storageDir, + onDiskKeys: new Set(), + }; + const fixture = path.join(FIXTURES, 'typescript-mcp-tools'); + const initial = await runPipelineFromRepo(fixture, () => {}, { + parseCache: cold, + workerPoolSize: 1, + }); + expect(initial.usedWorkerPool).toBe(true); + pruneCache(cold, cold.usedKeys); + const savedKeys = await saveParseCache(storageDir, cold); + await pruneAndSaveDurableParsedFileStore( + getDurableParsedFileDir(storageDir), + PARSE_CACHE_VERSION, + new Set(savedKeys), + ); + const warm = await loadParseCache(storageDir); + expect(warm).not.toBeNull(); + const replay = await runPipelineFromRepo(fixture, () => {}, { + parseCache: warm!, + workerPoolSize: 1, + }); + expect(replay.usedWorkerPool).toBe(false); + + const project = (pipeline: PipelineResult) => ({ + tools: getNodesByLabelFull(pipeline, 'Tool'), + edges: [ + ...getRelationships(pipeline, 'HANDLES_TOOL'), + ...getRelationships(pipeline, 'ENTRY_POINT_OF').filter( + (edge) => edge.sourceLabel === 'Tool', + ), + ] + .map(({ rel, source }) => ({ + type: rel.type, + source, + sourceId: rel.sourceId, + targetId: rel.targetId, + })) + .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))), + }); + const expected = project(initial); + expect(expected.tools).toHaveLength(unresolved.length + 5); + expect(project(replay)).toEqual(expected); + for (const name of unresolved) { + expect( + expected.edges.filter((edge) => edge.type === 'ENTRY_POINT_OF' && edge.source === name), + ).toEqual([]); + } + } finally { + fs.rmSync(storageDir, { recursive: true, force: true }); + } + }, 120_000); +}); diff --git a/gitnexus/test/integration/staged-embedding-recovery.test.ts b/gitnexus/test/integration/staged-embedding-recovery.test.ts new file mode 100644 index 000000000..0a52c67ab --- /dev/null +++ b/gitnexus/test/integration/staged-embedding-recovery.test.ts @@ -0,0 +1,208 @@ +import { spawnSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + disposeEmbeddingSpill, + materializeCachedEmbeddings, + type CachedEmbeddingsSnapshot, +} from '../../src/core/embeddings/embedding-restore-spill.js'; +import { recoverStagedEmbeddings } from '../../src/core/embeddings/staged-embedding-recovery.js'; + +describe('isolated native staged embedding recovery', () => { + let tmp: string | undefined; + let recovered: CachedEmbeddingsSnapshot | undefined; + afterEach(() => { + disposeEmbeddingSpill(recovered?.spill); + recovered = undefined; + if (tmp) fs.rmSync(tmp, { recursive: true, force: true }); + tmp = undefined; + }); + function stagePath() { + tmp ??= fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-stage-native-')); + return path.join(tmp, `lbug.staging.${randomUUID()}`); + } + function seed(dbPath: string, mode = 'clean') { + const result = spawnSync( + process.execPath, + [ + '--import', + 'tsx', + fileURLToPath(new URL('../fixtures/staged-embedding-recovery/seed.mjs', import.meta.url)), + dbPath, + mode, + ], + { + encoding: 'utf8', + timeout: 20_000, + env: { ...process.env, GITNEXUS_LBUG_BUFFER_POOL_SIZE: String(128 * 1024 * 1024) }, + }, + ); + expect(result.error, result.stderr).toBeUndefined(); + if (mode === 'hard-kill') expect(result.signal, result.stderr).toBe('SIGKILL'); + else expect(result.status, result.stderr).toBe(0); + } + + function snapshotSourceFamily(dbPath: string) { + const basename = path.basename(dbPath); + return Object.fromEntries( + fs + .readdirSync(path.dirname(dbPath)) + .filter((name) => name === basename || name.startsWith(`${basename}.`)) + .sort() + .map((name) => [name, fs.readFileSync(path.join(path.dirname(dbPath), name))]), + ); + } + + it('streams complete same-hash groups and rejects malformed whole nodes', async () => { + const dbPath = stagePath(); + seed(dbPath); + recovered = await recoverStagedEmbeddings(dbPath, { dimensions: 2 }); + expect([...recovered.embeddingNodeIds].sort()).toEqual(['complete', 'other']); + expect(recovered.rows).toHaveLength(3); + expect(recovered.embeddings).toEqual([]); + expect(materializeCachedEmbeddings(recovered, recovered.rows)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + nodeId: 'complete', + chunkIndex: 0, + embedding: [1, 2], + contentHash: 'same', + }), + expect.objectContaining({ + nodeId: 'complete', + chunkIndex: 1, + embedding: [1, 2], + contentHash: 'same', + }), + ]), + ); + expect(fs.existsSync(dbPath)).toBe(true); + }, 30_000); + + it('replays a hard-killed native writer strictly and excludes the incomplete active window', async () => { + const dbPath = stagePath(); + seed(dbPath, 'hard-kill'); + const sourceBefore = snapshotSourceFamily(dbPath); + recovered = await recoverStagedEmbeddings(dbPath, { + dimensions: 2, + excludedNodeIds: ['unsafe-prefix', 'other'], + }); + expect([...recovered.embeddingNodeIds]).toEqual(['complete']); + expect(recovered.rows).toHaveLength(2); + expect( + materializeCachedEmbeddings(recovered, recovered.rows) + .map((row) => row.chunkIndex) + .sort(), + ).toEqual([0, 1]); + expect(snapshotSourceFamily(dbPath)).toEqual(sourceBefore); + }, 30_000); + + it('rejects vectors from a different dimension instead of coercing them', async () => { + const dbPath = stagePath(); + seed(dbPath); + recovered = await recoverStagedEmbeddings(dbPath, { dimensions: 3 }); + expect(recovered.rows).toEqual([]); + }, 30_000); + + it('strictly replays an interrupted checkpoint copy with both checkpoint locks retained', async (ctx) => { + const version = JSON.parse( + fs.readFileSync( + new URL('../../node_modules/@ladybugdb/core/package.json', import.meta.url), + 'utf8', + ), + ).version as string; + const [major, minor] = version.split('.').map(Number); + // Older pins do not support the deterministic interrupted-checkpoint plant. + if (Number.isFinite(major) && Number.isFinite(minor) && major === 0 && minor < 19) ctx.skip(); + const dbPath = stagePath(); + seed(dbPath, 'interrupted-checkpoint'); + const sourceBefore = snapshotSourceFamily(dbPath); + expect(fs.statSync(`${dbPath}.wal.checkpoint`).size).toBeGreaterThan(0); + expect(fs.existsSync(`${dbPath}.checkpoint.intent.lock`)).toBe(true); + expect(fs.existsSync(`${dbPath}.checkpoint.apply.lock`)).toBe(true); + + recovered = await recoverStagedEmbeddings(dbPath, { dimensions: 2 }); + + expect([...recovered.embeddingNodeIds].sort()).toEqual([ + 'checkpoint-only', + 'complete', + 'other', + ]); + expect(recovered.rows).toHaveLength(4); + expect(snapshotSourceFamily(dbPath)).toEqual(sourceBefore); + }, 30_000); + + it('contains a malformed native source in a subprocess and preserves it', async () => { + const dbPath = stagePath(); + fs.writeFileSync(dbPath, 'not a ladybug database'); + await expect(recoverStagedEmbeddings(dbPath, { dimensions: 2 })).rejects.toThrow( + /extraction failed/, + ); + expect(fs.readFileSync(dbPath, 'utf8')).toBe('not a ladybug database'); + }, 30_000); + + it('rejects a malformed WAL without deleting or quarantining it to reopen the source', async () => { + const dbPath = stagePath(); + seed(dbPath, 'hard-kill'); + const walPath = `${dbPath}.wal`; + fs.writeFileSync(walPath, Buffer.alloc(128, 0xff)); + const sourceBefore = snapshotSourceFamily(dbPath); + await expect(recoverStagedEmbeddings(dbPath, { dimensions: 2 })).rejects.toThrow( + /extraction failed/, + ); + expect(fs.existsSync(walPath)).toBe(true); + expect(fs.readdirSync(path.dirname(dbPath)).some((name) => /bad|quarantine/i.test(name))).toBe( + false, + ); + expect(snapshotSourceFamily(dbPath)).toEqual(sourceBefore); + }, 30_000); + + it('does not create a missing source and refuses a symlink', async () => { + const dbPath = stagePath(); + await expect(recoverStagedEmbeddings(dbPath, { dimensions: 2 })).rejects.toThrow(/ENOENT/); + expect(fs.existsSync(dbPath)).toBe(false); + const realPath = path.join(path.dirname(dbPath), 'real'); + fs.writeFileSync(realPath, 'fixture'); + fs.symlinkSync(realPath, dbPath); + await expect(recoverStagedEmbeddings(dbPath, { dimensions: 2 })).rejects.toThrow( + /regular file/, + ); + }); + + it('can kill a timed out native subprocess without aborting analyze', async () => { + const dbPath = stagePath(); + seed(dbPath); + const sourceBefore = snapshotSourceFamily(dbPath); + await expect(recoverStagedEmbeddings(dbPath, { dimensions: 2, timeoutMs: 1 })).rejects.toThrow( + /timeout/, + ); + expect(fs.existsSync(dbPath)).toBe(true); + expect(snapshotSourceFamily(dbPath)).toEqual(sourceBefore); + }, 30_000); + + it('loads the source child when analyze runs in another repository directory', () => { + const dbPath = stagePath(); + seed(dbPath); + const result = spawnSync( + process.execPath, + [ + '--import', + import.meta.resolve('tsx'), + '--input-type=module', + '-e', + 'const { recoverStagedEmbeddings } = await import(process.argv[1]); const { disposeEmbeddingSpill } = await import(process.argv[2]); const recovered = await recoverStagedEmbeddings(process.argv[3], {dimensions: 2}); process.stdout.write(String(recovered.rows.length)); disposeEmbeddingSpill(recovered.spill);', + new URL('../../src/core/embeddings/staged-embedding-recovery.ts', import.meta.url).href, + new URL('../../src/core/embeddings/embedding-restore-spill.ts', import.meta.url).href, + dbPath, + ], + { cwd: path.dirname(dbPath), encoding: 'utf8', timeout: 20_000 }, + ); + expect(result.error, result.stderr).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe('3'); + }, 30_000); +}); diff --git a/gitnexus/test/unit/api-fts-mode.test.ts b/gitnexus/test/unit/api-fts-mode.test.ts index 632bd37d3..ab8eaf401 100644 --- a/gitnexus/test/unit/api-fts-mode.test.ts +++ b/gitnexus/test/unit/api-fts-mode.test.ts @@ -7,7 +7,12 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } const mocks = vi.hoisted(() => ({ loadMeta: vi.fn(), + saveMeta: vi.fn(), listRegisteredRepos: vi.fn(), + acquireIndexLock: vi.fn(), + releaseIndexLock: vi.fn(), + ensurePrivateSharedGraph: vi.fn(), + runEmbeddingPipeline: vi.fn(), withLbugDb: vi.fn(), search: vi.fn(), updateJob: vi.fn(), @@ -16,8 +21,19 @@ const mocks = vi.hoisted(() => ({ vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => ({ ...(await importOriginal()), loadMeta: mocks.loadMeta, + saveMeta: mocks.saveMeta, listRegisteredRepos: mocks.listRegisteredRepos, })); +vi.mock('../../src/storage/index-lock.js', async (importOriginal) => ({ + ...(await importOriginal()), + acquireIndexLock: mocks.acquireIndexLock, +})); +vi.mock('../../src/core/shared-store-analyze.js', () => ({ + ensurePrivateSharedGraph: mocks.ensurePrivateSharedGraph, +})); +vi.mock('../../src/core/embeddings/embedding-pipeline.js', () => ({ + runEmbeddingPipeline: mocks.runEmbeddingPipeline, +})); vi.mock('../../src/storage/storage-resolver.js', async (importOriginal) => ({ ...(await importOriginal()), requireRegisteredStoragePath: vi.fn(async (entry: { storagePath: string }) => entry.storagePath), @@ -116,6 +132,19 @@ afterAll(() => { beforeEach(() => { vi.clearAllMocks(); + mocks.acquireIndexLock.mockResolvedValue({ + release: mocks.releaseIndexLock, + record: { + v: 1, + pid: process.pid, + hostname: 'test-host', + startTime: null, + token: 'test-lock', + invocationId: 'test-run', + acquiredAt: '', + }, + }); + mocks.ensurePrivateSharedGraph.mockResolvedValue(true); mocks.listRegisteredRepos.mockResolvedValue([entry]); mocks.withLbugDb.mockImplementation(async (_path, callback) => callback()); mocks.search.mockImplementation(async (_query, _limit, _exec, reason) => ({ @@ -124,6 +153,145 @@ beforeEach(() => { })); }); +describe('POST /api/embed staged recovery preflight', () => { + afterEach(() => { + vi.unstubAllEnvs(); + fs.rmSync(entry.storagePath, { recursive: true, force: true }); + fs.mkdirSync(entry.storagePath, { recursive: true }); + }); + + async function useRealIndexLock() { + vi.stubEnv('GITNEXUS_INDEX_LOCK_BACKEND', 'file'); + const actual = await vi.importActual( + '../../src/storage/index-lock.js', + ); + mocks.acquireIndexLock.mockImplementation( + async (...args: Parameters) => { + const lock = await actual.acquireIndexLock(...args); + return { + ...lock, + release: () => { + lock.release(); + mocks.releaseIndexLock(); + }, + }; + }, + ); + } + + it.each([ + { + name: 'valid staged receipt', + recovery: { + stagingFile: 'lbug.staging.12345678-1234-4123-8123-123456789abc', + schemaFingerprint: 'test-schema', + unsafeNodeIds: ['n2', 'inherited-window-node'], + }, + }, + { name: 'null receipt', recovery: null }, + { name: 'malformed receipt', recovery: { stagingFile: 'invalid' } }, + { name: 'false receipt', recovery: false }, + ])('preserves a $name and releases the job locks', async ({ recovery }) => { + await useRealIndexLock(); + const lockPath = path.join(entry.storagePath, 'analyze.lock'); + const lbugPath = path.join(entry.storagePath, 'lbug'); + const metaPath = path.join(entry.storagePath, 'gitnexus.json'); + const sourcePath = path.join( + entry.storagePath, + 'lbug.staging.12345678-1234-4123-8123-123456789abc', + ); + fs.writeFileSync(lbugPath, 'published graph'); + fs.writeFileSync(sourcePath, 'completed paid vectors'); + fs.writeFileSync(`${sourcePath}.wal`, 'unfinished window'); + const metadataBytes = JSON.stringify({ + repoPath: entry.path, + lastCommit: 'abc123', + indexedAt: '2026-01-01T00:00:00.000Z', + stats: { embeddings: 7 }, + embeddingCheckpoint: { + at: '2026-01-01T00:00:00.000Z', + nodesProcessed: 1, + totalNodes: 2, + chunksProcessed: 1, + model: 'test-model', + dimensions: 768, + provider: 'local', + kind: 'interrupted', + pendingNodeIds: ['n2'], + recovery, + }, + }); + fs.writeFileSync(metaPath, metadataBytes); + mocks.loadMeta.mockImplementation(async () => { + expect(fs.existsSync(lockPath)).toBe(true); + return JSON.parse(fs.readFileSync(metaPath, 'utf8')); + }); + mocks.withLbugDb.mockResolvedValue(undefined); + + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledTimes(1)); + + expect(mocks.updateJob).toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ + status: 'failed', + error: expect.stringMatching( + /staged embeddings.*Run `gitnexus analyze` to recover them first/, + ), + }), + ); + expect(mocks.acquireIndexLock).toHaveBeenCalledWith(entry.storagePath, { sweep: false }); + expect(mocks.loadMeta.mock.invocationCallOrder[0]).toBeGreaterThan( + mocks.acquireIndexLock.mock.invocationCallOrder[0]!, + ); + expect(mocks.ensurePrivateSharedGraph).not.toHaveBeenCalled(); + expect(mocks.withLbugDb).not.toHaveBeenCalled(); + expect(mocks.runEmbeddingPipeline).not.toHaveBeenCalled(); + expect(mocks.saveMeta).not.toHaveBeenCalled(); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.readFileSync(metaPath, 'utf8')).toBe(metadataBytes); + expect(fs.readFileSync(lbugPath, 'utf8')).toBe('published graph'); + expect(fs.readFileSync(sourcePath, 'utf8')).toBe('completed paid vectors'); + expect(fs.readFileSync(`${sourcePath}.wal`, 'utf8')).toBe('unfinished window'); + + // A second accepted job proves the in-memory repo lock was also released. + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledTimes(2)); + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it('sweeps orphaned staging files before a writable job without a recovery receipt', async () => { + await useRealIndexLock(); + const metaPath = path.join(entry.storagePath, 'gitnexus.json'); + fs.writeFileSync(metaPath, JSON.stringify({ repoPath: entry.path })); + const sourcePath = path.join( + entry.storagePath, + 'lbug.staging.12345678-1234-4123-8123-123456789abc', + ); + fs.writeFileSync(sourcePath, 'orphaned database'); + fs.writeFileSync(`${sourcePath}.wal`, 'orphaned WAL'); + mocks.loadMeta.mockImplementation(async () => JSON.parse(fs.readFileSync(metaPath, 'utf8'))); + mocks.ensurePrivateSharedGraph.mockImplementation(async () => { + expect(fs.existsSync(path.join(entry.storagePath, 'analyze.lock'))).toBe(true); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.wal`)).toBe(false); + return true; + }); + mocks.withLbugDb.mockResolvedValue(undefined); + + await invoke('/api/embed'); + await vi.waitFor(() => expect(mocks.releaseIndexLock).toHaveBeenCalledTimes(1)); + + expect(mocks.updateJob).toHaveBeenCalledWith( + 'embed-job', + expect.objectContaining({ status: 'complete' }), + ); + expect(mocks.ensurePrivateSharedGraph).toHaveBeenCalledTimes(1); + expect(mocks.withLbugDb).toHaveBeenCalledTimes(1); + expect(fs.existsSync(path.join(entry.storagePath, 'analyze.lock'))).toBe(false); + }); +}); + async function invoke(route: string, query: Record = {}) { const layer = app.router.stack.find((item: any) => item.route?.path === route); expect(layer, route).toBeDefined(); @@ -257,7 +425,9 @@ describe('serve uses one metadata-derived FTS mode on every DB-open path', () => expect.any(Function), skip ? { skipFts: true } : {}, ); - expect(mocks.loadMeta).toHaveBeenCalledExactlyOnceWith(entry.storagePath); + expect(mocks.loadMeta).toHaveBeenCalledTimes(2); + expect(mocks.loadMeta).toHaveBeenNthCalledWith(1, entry.storagePath); + expect(mocks.loadMeta).toHaveBeenNthCalledWith(2, entry.storagePath); }, ); }); diff --git a/gitnexus/test/unit/api-graph-streaming.test.ts b/gitnexus/test/unit/api-graph-streaming.test.ts index 58de01f84..f79a1ea5a 100644 --- a/gitnexus/test/unit/api-graph-streaming.test.ts +++ b/gitnexus/test/unit/api-graph-streaming.test.ts @@ -1,4 +1,5 @@ import { EventEmitter } from 'node:events'; +import type { GraphRelationship } from 'gitnexus-shared'; import { describe, expect, it, vi, beforeEach } from 'vitest'; const { lbugMocks } = vi.hoisted(() => ({ @@ -23,6 +24,60 @@ const createMockResponse = (writeImpl?: (chunk: string) => boolean) => { return response; }; +describe('graph relationship reason contract', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it.each([ + { stored: null, expected: '' }, + { stored: undefined, expected: '' }, + { stored: '', expected: '' }, + { stored: 'branch → next', expected: 'branch → next' }, + ])( + 'returns reason=$expected for stored $stored in JSON and NDJSON', + async ({ stored, expected }) => { + const row = { + sourceId: 'BasicBlock:src/app.ts:1', + targetId: 'BasicBlock:src/app.ts:2', + type: 'CFG' as const, + confidence: 1, + reason: stored, + step: 0, + }; + lbugMocks.executeQuery.mockImplementation(async (query: string) => + query.includes('CodeRelation') ? [row] : [], + ); + lbugMocks.streamQuery.mockImplementation( + async (query: string, onRow: (row: unknown) => Promise) => { + if (!query.includes('CodeRelation')) return 0; + await onRow(row); + return 1; + }, + ); + + const buffered = JSON.parse(JSON.stringify(await buildGraph())); + const writes: string[] = []; + await streamGraphNdjson( + createMockResponse((chunk) => { + writes.push(chunk); + return true; + }), + ); + const streamed = writes.map((chunk) => JSON.parse(chunk)); + const relationship: GraphRelationship = { + ...row, + id: `${row.sourceId}_${row.type}_${row.targetId}`, + reason: expected, + }; + expect({ buffered: buffered.relationships, streamed }).toEqual({ + buffered: [relationship], + streamed: [{ type: 'relationship', data: relationship }], + }); + }, + ); +}); + describe('streamGraphNdjson', () => { beforeEach(() => { vi.clearAllMocks(); diff --git a/gitnexus/test/unit/basicblock-callee-ids-schema.test.ts b/gitnexus/test/unit/basicblock-callee-ids-schema.test.ts index a9e7ab1ad..d6dbb8154 100644 --- a/gitnexus/test/unit/basicblock-callee-ids-schema.test.ts +++ b/gitnexus/test/unit/basicblock-callee-ids-schema.test.ts @@ -74,7 +74,7 @@ describe('BasicBlock calleeIds — CSV header + row builder', () => { expect(rowCells).toHaveLength(headerCols.length); const calleeIdsIdx = headerCols.indexOf('calleeIds'); const calleesIdx = headerCols.indexOf('callees'); - // escapeCSVField always wraps the cell in double quotes; the space-joined + // Non-empty string fields are quoted; the space-joined // id list contains no comma, so the cell is a single CSV column. expect(rowCells[calleeIdsIdx]).toBe('"id1 id2"'); expect(rowCells[calleesIdx]).toBe('"foo bar"'); @@ -85,7 +85,8 @@ describe('BasicBlock calleeIds — CSV header + row builder', () => { const headerCols = BASICBLOCK_CSV_HEADER.split(','); const rowCells = buildBasicBlockRow(node).split(','); const calleeIdsIdx = headerCols.indexOf('calleeIds'); - expect(rowCells[calleeIdsIdx]).toBe('""'); + // An absent string must stay unquoted so native COPY loads SQL NULL. + expect(rowCells[calleeIdsIdx]).toBe(''); expect(rowCells[calleeIdsIdx]).not.toContain('undefined'); }); }); diff --git a/gitnexus/test/unit/bm25-search.test.ts b/gitnexus/test/unit/bm25-search.test.ts index 793cf7aae..08b2fc46a 100644 --- a/gitnexus/test/unit/bm25-search.test.ts +++ b/gitnexus/test/unit/bm25-search.test.ts @@ -647,3 +647,125 @@ describe('BM25 search', () => { }); }); }); + +describe('FTS index completeness', () => { + beforeEach(async () => { + resetExtensionState(); + mockExecuteParameterized.mockReset(); + const { queryFTS } = await import('../../src/core/lbug/lbug-adapter.js'); + vi.mocked(queryFTS).mockReset(); + }); + + const missing = (table: string, indexName: string) => + new Error(`Binder exception: Table ${table} doesn't have an index with name ${indexName}.`); + + for (const pooled of [false, true]) { + const mode = pooled ? 'pool' : 'core'; + const repo = pooled ? 'completeness-test' : undefined; + + async function useOutcome(outcome: (table: string, indexName: string) => Promise) { + const { queryFTS } = await import('../../src/core/lbug/lbug-adapter.js'); + vi.mocked(queryFTS).mockImplementation((table, indexName) => outcome(table, indexName)); + mockExecuteParameterized.mockImplementation(async (_repo: string, cypher: string) => { + const match = cypher.match(/QUERY_FTS_INDEX\('([^']+)', '([^']+)'/); + return outcome(match![1], match![2]); + }); + } + + it(`${mode}: distinguishes healthy zero matches from missing indexes`, async () => { + await useOutcome(async () => []); + const result = await searchFTSFromLbug('no matches', 5, repo); + expect(result).toEqual({ results: [], ftsAvailable: true }); + }); + + it(`${mode}: reports one missing index while retaining successful matches`, async () => { + await useOutcome(async (table, indexName) => { + if (table === 'Function') throw missing(table, indexName); + if (table !== 'File') return []; + return pooled + ? [{ node: { filePath: 'src/auth.ts', id: 'file:auth' }, score: 3 }] + : [{ filePath: 'src/auth.ts', nodeId: 'file:auth', score: 3 }]; + }); + const result = await searchFTSFromLbug('auth', 5, repo); + expect(result.ftsAvailable).toBe(true); + expect(result.results[0]).toMatchObject({ filePath: 'src/auth.ts', score: 3 }); + expect(result.missingIndexes).toEqual(['Function.function_fts']); + expect(result.nonBenignErrors).toBeUndefined(); + }); + + it(`${mode}: reports all missing indexes without claiming FTS is available`, async () => { + await useOutcome(async (table, indexName) => { + throw missing(table, indexName); + }); + const result = await searchFTSFromLbug('auth', 5, repo); + expect(result.ftsAvailable).toBe(false); + expect(result.results).toEqual([]); + if (!pooled) { + const { queryFTS } = await import('../../src/core/lbug/lbug-adapter.js'); + for (const { table, indexName } of FTS_INDEXES) { + expect(queryFTS).toHaveBeenCalledWith(table, indexName, 'auth', 5, false, 'throw'); + } + } + expect(result.missingIndexes).toEqual( + FTS_INDEXES.map(({ table, indexName }) => `${table}.${indexName}`), + ); + expect(result.nonBenignErrors).toBeUndefined(); + }); + + it(`${mode}: preserves redacted real errors alongside missing-index diagnostics`, async () => { + await useOutcome(async (table, indexName) => { + if (table === 'Function') throw missing(table, indexName); + if (table === 'Class') + throw new Error('connection reset at /home/alice/private/index.lbug'); + return []; + }); + const result = await searchFTSFromLbug('auth', 5, repo); + expect(result.ftsAvailable).toBe(true); + expect(result.missingIndexes).toEqual(['Function.function_fts']); + expect(result.nonBenignErrors).toHaveLength(1); + expect(result.nonBenignErrors![0]).toContain('connection reset'); + expect(JSON.stringify(result)).not.toContain('/home/alice'); + }); + + it(`${mode}: retains both failure causes when no configured index query succeeds`, async () => { + await useOutcome(async (table, indexName) => { + if (table === 'Function') throw missing(table, indexName); + throw new Error('connection reset at /home/alice/private/index.lbug'); + }); + + const result = await searchFTSFromLbug('auth', 5, repo); + + expect(result.ftsAvailable).toBe(false); + expect(result.results).toEqual([]); + expect(result.missingIndexes).toEqual(['Function.function_fts']); + expect(result.nonBenignErrors).toEqual( + Array(FTS_INDEXES.length - 1).fill('connection reset at '), + ); + }); + + it(`${mode}: clears missing-index diagnostics after a repaired query`, async () => { + await useOutcome(async (table, indexName) => { + if (table === 'Function') throw missing(table, indexName); + return []; + }); + expect((await searchFTSFromLbug('auth', 5, repo)).missingIndexes).toEqual([ + 'Function.function_fts', + ]); + await useOutcome(async () => []); + expect(await searchFTSFromLbug('auth', 5, repo)).toEqual({ results: [], ftsAvailable: true }); + }); + + it(`${mode}: explicit opt-out makes no queries and emits no missing-index diagnosis`, async () => { + await useOutcome(async (table, indexName) => { + throw missing(table, indexName); + }); + expect(await searchFTSFromLbug('auth', 5, repo, 'disabled-by-flag')).toEqual({ + results: [], + ftsAvailable: false, + }); + const { queryFTS } = await import('../../src/core/lbug/lbug-adapter.js'); + expect(queryFTS).not.toHaveBeenCalled(); + expect(mockExecuteParameterized).not.toHaveBeenCalled(); + }); + } +}); diff --git a/gitnexus/test/unit/calltool-dispatch-id-bridge.test.ts b/gitnexus/test/unit/calltool-dispatch-id-bridge.test.ts index 0390caab6..be6d32215 100644 --- a/gitnexus/test/unit/calltool-dispatch-id-bridge.test.ts +++ b/gitnexus/test/unit/calltool-dispatch-id-bridge.test.ts @@ -159,9 +159,8 @@ describe('LocalBackend PDG impact — resolved-callee-id bridge (U6)', () => { if (query.includes('r.type IN $relTypes') && !query.includes('STEP_IN_PROCESS')) { return [frontierRow('func:callee-A', 'callee')]; } - if (query.includes('COUNT(DISTINCT s.id)') || query.includes('RETURN s.id AS sid')) return []; - // Target resolution (WHERE n.name = $symName) and any other read. - return [TARGET_ROW]; + if (query.includes('WHERE n.name = $symName')) return [TARGET_ROW]; + return []; }); const result = await backend.callTool('impact', { @@ -195,8 +194,8 @@ describe('LocalBackend PDG impact — resolved-callee-id bridge (U6)', () => { if (query.includes('r.type IN $relTypes') && !query.includes('STEP_IN_PROCESS')) { return [frontierRow('func:callee-A', 'callee')]; } - if (query.includes('COUNT(DISTINCT s.id)') || query.includes('RETURN s.id AS sid')) return []; - return [TARGET_ROW]; + if (query.includes('WHERE n.name = $symName')) return [TARGET_ROW]; + return []; }); const result = await backend.callTool('impact', { @@ -230,8 +229,8 @@ describe('LocalBackend PDG impact — resolved-callee-id bridge (U6)', () => { if (query.includes('r.type IN $relTypes') && !query.includes('STEP_IN_PROCESS')) { return [frontierRow('func:callee-A', 'callee'), frontierRow('func:callee-B', 'callee')]; } - if (query.includes('COUNT(DISTINCT s.id)') || query.includes('RETURN s.id AS sid')) return []; - return [TARGET_ROW]; + if (query.includes('WHERE n.name = $symName')) return [TARGET_ROW]; + return []; }); const result = await backend.callTool('impact', { @@ -266,8 +265,8 @@ describe('LocalBackend PDG impact — resolved-callee-id bridge (U6)', () => { if (query.includes('r.type IN $relTypes') && !query.includes('STEP_IN_PROCESS')) { return [frontierRow('func:callee-A', 'callee'), frontierRow('*', 'callee')]; } - if (query.includes('COUNT(DISTINCT s.id)') || query.includes('RETURN s.id AS sid')) return []; - return [TARGET_ROW]; + if (query.includes('WHERE n.name = $symName')) return [TARGET_ROW]; + return []; }); const result = await backend.callTool('impact', { diff --git a/gitnexus/test/unit/calltool-dispatch.test.ts b/gitnexus/test/unit/calltool-dispatch.test.ts index 6a0a98d4a..7eaf2e1ff 100644 --- a/gitnexus/test/unit/calltool-dispatch.test.ts +++ b/gitnexus/test/unit/calltool-dispatch.test.ts @@ -183,6 +183,16 @@ function setupNoRepos() { (listRegisteredRepos as any).mockResolvedValue([]); } +/** Seed resolver rows without inventing relationship/process rows for other projections. */ +function mockSymbolRows(rows: Record[]) { + (executeParameterized as any).mockImplementation( + async (_repo: string, query: string, params: Record) => { + if (query.includes('COUNT(*) AS total')) return [{ total: rows.length }]; + return params?.symName || params?.uid ? rows : []; + }, + ); +} + const duplicateFixtureDirs: string[] = []; function makeDuplicateNameFixture() { @@ -624,8 +634,8 @@ describe('LocalBackend.callTool', () => { }); it('reports UNKNOWN instead of a blast radius when the target resolves without a node id (#3354)', async () => { - // Every query returns the same id-less row: the resolver picks it as the - // single match, and the frontier query would answer for no symbol at all. + // Every query returns the same id-less row: resolution must reject it + // before a frontier query could answer for no symbol at all. (executeParameterized as any).mockResolvedValue([{ name: 'runSweep', type: 'Function' }]); const result = await backend.callTool('impact', { target: 'runSweep', direction: 'upstream' }); @@ -635,7 +645,8 @@ describe('LocalBackend.callTool', () => { impactedCount: null, risk: 'UNKNOWN', }); - expect(result.error).toMatch(/without a node id/); + expect(result.error).toMatch(/invalid symbol identity/); + expect(result.recoverySuggestion).toContain('gitnexus analyze --force'); expect(result).not.toHaveProperty('byDepthCounts'); }); @@ -1320,7 +1331,7 @@ describe('LocalBackend.callTool', () => { }); it('dispatches context tool', async () => { - (executeParameterized as any).mockResolvedValue([ + mockSymbolRows([ { id: 'func:main', name: 'main', @@ -1662,27 +1673,22 @@ describe('LocalBackend.callTool', () => { }); it('exact File path wins over suffixed matches during qualified resolution (#3084 review P2)', async () => { - (executeParameterized as any).mockImplementation(async (_repo: string, query: string) => { - if (query.startsWith('MATCH (n)')) { - return [ - { - id: 'File:src/lib/a.ts', - name: 'a.ts', - filePath: 'src/lib/a.ts', - kind: 'File', - total_hits: 1, - }, - { - id: 'File:lib/a.ts', - name: 'a.ts', - filePath: 'lib/a.ts', - kind: 'File', - total_hits: 1, - }, - ]; - } - return [{ total: 2 }]; - }); + mockSymbolRows([ + { + id: 'File:src/lib/a.ts', + name: 'a.ts', + filePath: 'src/lib/a.ts', + kind: 'File', + total_hits: 1, + }, + { + id: 'File:lib/a.ts', + name: 'a.ts', + filePath: 'lib/a.ts', + kind: 'File', + total_hits: 1, + }, + ]); const result = await backend.callTool('context', { name: 'lib/a.ts' }); expect(result).toMatchObject({ @@ -2004,7 +2010,7 @@ describe('LocalBackend.callTool', () => { }); it('context tool ranks file_path match higher than non-match (#470)', async () => { - (executeParameterized as any).mockResolvedValue([ + mockSymbolRows([ { id: 'func:handleConnect:1', name: 'handleConnect', @@ -2043,7 +2049,7 @@ describe('LocalBackend.callTool', () => { // review): both candidates satisfy the file_path hint (so DB // pre-filter would return both in production), and promotion is // determined purely by the combined file_path + kind score. - (executeParameterized as any).mockResolvedValue([ + mockSymbolRows([ { id: 'fn:App:1', name: 'render', @@ -2134,7 +2140,7 @@ describe('LocalBackend.callTool', () => { it('impact tool returns ambiguous shape with ranked candidates when target has multiple matches (#470)', async () => { // resolveSymbolCandidates issues a single name query; mock it to return // two Function rows in different files with no hints. - (executeParameterized as any).mockResolvedValue([ + mockSymbolRows([ { id: 'func:login:1', name: 'login', @@ -2221,7 +2227,7 @@ describe('LocalBackend.callTool', () => { // Resolver returns target; BFS returns one frontier caller; no STEP_IN_PROCESS rows. (executeParameterized as any).mockImplementation((_repoId: string, cypher: string) => { // BFS frontier query is now parameterized (#1907 U3). - if (cypher.includes('r.type IN') && !cypher.includes('STEP_IN_PROCESS')) { + if (cypher.includes('$frontierIds')) { return Promise.resolve([ { id: 'func:caller', @@ -2233,10 +2239,12 @@ describe('LocalBackend.callTool', () => { }, ]); } - // Symbol resolution. - return Promise.resolve([ - { id: 'func:main', name: 'main', type: 'Function', filePath: 'src/index.ts' }, - ]); + // Symbol resolution; unseeded enrichment queries return no rows. + return Promise.resolve( + cypher.includes('$symName') + ? [{ id: 'func:main', name: 'main', type: 'Function', filePath: 'src/index.ts' }] + : [], + ); }); (executeQuery as any).mockResolvedValue([]); @@ -2466,7 +2474,7 @@ describe('LocalBackend.callTool', () => { }, ]); - const result = await backend.impactByUid('test-project', 'uid:main', 'upstream', { + const result = await backend.impactByUid('test-project', 'func:main', 'upstream', { maxDepth: 5, relationTypes: ['CALLS'], minConfidence: 0, @@ -2973,7 +2981,7 @@ describe('LocalBackend.callTool', () => { }); it('dispatches "explore" as alias for context', async () => { - (executeParameterized as any).mockResolvedValue([ + mockSymbolRows([ { id: 'func:main', name: 'main', @@ -3006,9 +3014,7 @@ describe('LocalBackend impact mode (KTD1/KTD5/KTD12)', () => { // dispatch (callgraph BFS or the PDG traversal). The callgraph BFS then issues // executeQuery for its frontier; the PDG path delegates to runImpactPDG. function resolveSingleTarget() { - (executeParameterized as any).mockResolvedValue([ - { id: 'func:main', name: 'main', type: 'Function', filePath: 'src/index.ts' }, - ]); + mockSymbolRows([{ id: 'func:main', name: 'main', type: 'Function', filePath: 'src/index.ts' }]); (executeQuery as any).mockResolvedValue([]); } @@ -3265,18 +3271,13 @@ describe('LocalBackend impact mode (KTD1/KTD5/KTD12)', () => { it("mode:'pdg' + downstream line:8 routes to the PDG traversal and seeds bridge evidence", async () => { resolveSingleTarget(); - // The target-resolution row doubles as the calleesOfBlocks row: `callees` - // ('callee') is the leaf name persisted on the slice's BasicBlock, the - // statement-precise substrate the bridge keys on. - (executeParameterized as any).mockResolvedValue([ - { - id: 'func:main', - name: 'main', - type: 'Function', - filePath: 'src/index.ts', - callees: 'callee', - }, - ]); + // BasicBlock callees and symbol lookup use distinct native projections. + vi.mocked(executeParameterized).mockImplementation(async (_repo, query) => { + if (query.includes('RETURN b.callees')) return [{ callees: 'callee' }]; + return query.includes('$symName') + ? [{ id: 'func:main', name: 'main', type: 'Function', filePath: 'src/index.ts' }] + : []; + }); // A line-seeded downstream slice with one reachable block → the dispatch // queries that block's callees and seeds the bridge with them. const pdgSpy = vi.spyOn(backend as any, '_runImpactPDG').mockResolvedValueOnce({ @@ -3401,11 +3402,13 @@ describe('LocalBackend impact mode (KTD1/KTD5/KTD12)', () => { // is not built and the inter-procedural reach falls back to callgraph-equal — // never surfacing the error or producing a partial proven/unproven labeling. resolveSingleTarget(); - // The slice-callees query (RETURN b.callees) throws; every other query (target - // resolution) returns the resolved symbol row. + // The slice-callees query throws; lookup returns the target and unseeded + // relationship/process projections return no rows. vi.mocked(executeParameterized).mockImplementation(async (_repo, query) => { if (query.includes('RETURN b.callees')) throw new Error('slice-callees query failed'); - return [{ id: 'func:main', name: 'main', type: 'Function', filePath: 'src/index.ts' }]; + return query.includes('$symName') + ? [{ id: 'func:main', name: 'main', type: 'Function', filePath: 'src/index.ts' }] + : []; }); // A line-seeded downstream slice so calleesOfBlocks is attempted. vi.spyOn(backend as any, '_runImpactPDG').mockResolvedValueOnce({ @@ -3460,7 +3463,9 @@ describe('LocalBackend impact mode (KTD1/KTD5/KTD12)', () => { resolveSingleTarget(); vi.mocked(executeParameterized).mockImplementation(async (_repo, query) => { if (query.includes('RETURN b.callees')) throw new Error('Table BasicBlock does not exist'); - return [{ id: 'func:main', name: 'main', type: 'Function', filePath: 'src/index.ts' }]; + return query.includes('$symName') + ? [{ id: 'func:main', name: 'main', type: 'Function', filePath: 'src/index.ts' }] + : []; }); vi.spyOn(backend as any, '_runImpactPDG').mockResolvedValueOnce({ mode: 'pdg', diff --git a/gitnexus/test/unit/cli-group-status-format.test.ts b/gitnexus/test/unit/cli-group-status-format.test.ts index 32278dd04..962efd536 100644 --- a/gitnexus/test/unit/cli-group-status-format.test.ts +++ b/gitnexus/test/unit/cli-group-status-format.test.ts @@ -27,6 +27,29 @@ describe('formatIndexStatusCell', () => { ); }); + it.each([0, 3])( + 'renders a stale diverged row without a commits-behind count of %i', + (commitsBehind) => { + const row = { indexStale: true, commitsBehind, status: 'diverged' as const }; + expect(formatIndexStatusCell(row)).toBe('STALE (index differs from HEAD)'); + }, + ); + + it('renders an explicit behind status with its counted gap', () => { + const row = { indexStale: true, commitsBehind: 3, status: 'behind' as const }; + expect(formatIndexStatusCell(row)).toBe('STALE (3 commits behind)'); + }); + + it('renders an explicit current status as OK', () => { + const row = { indexStale: false, commitsBehind: 0, status: 'current' as const }; + expect(formatIndexStatusCell(row)).toBe('OK '); + }); + + it('preserves the fail-open cell when a diverged probe did not confirm staleness', () => { + const row = { indexStale: false, commitsBehind: 0, status: 'diverged' as const }; + expect(formatIndexStatusCell(row)).toBe('OK '); + }); + it('keeps the OK cell byte-identical, padding included', () => { expect(formatIndexStatusCell({ indexStale: false, commitsBehind: 0 })).toBe('OK '); }); diff --git a/gitnexus/test/unit/community-processor.test.ts b/gitnexus/test/unit/community-processor.test.ts index 2e207a22a..e9b3d07aa 100644 --- a/gitnexus/test/unit/community-processor.test.ts +++ b/gitnexus/test/unit/community-processor.test.ts @@ -448,7 +448,76 @@ module.exports = { const second = await processCommunities(graph); expect(second.memberships).toEqual(first.memberships); + expect(second.rawMemberships).toEqual(first.rawMemberships); expect(second.stats.modularity).toBe(first.stats.modularity); }); }); }); + +describe('community membership integrity', () => { + it('returns empty raw and retained memberships for an empty graph', async () => { + const result = await processCommunities(createKnowledgeGraph()); + + expect(result.communities).toEqual([]); + expect(result.memberships).toEqual([]); + expect(result.rawMemberships).toEqual([]); + expect(result.stats).toMatchObject({ totalCommunities: 0, modularity: 0, nodesProcessed: 0 }); + }); + + it('retains connected members without emitting memberships for a filtered singleton', async () => { + const graph = createKnowledgeGraph(); + for (const id of ['fn:a', 'fn:b', 'fn:c', 'fn:singleton']) { + graph.addNode(makeNode(id, id.slice(3))); + } + graph.addNode(makeNode('file:target', 'target', 'File')); + graph.addRelationship(makeRel('rel:ab', 'fn:a', 'fn:b')); + graph.addRelationship(makeRel('rel:bc', 'fn:b', 'fn:c')); + graph.addRelationship(makeRel('rel:ca', 'fn:c', 'fn:a')); + // The symbol enters the projection, but its non-symbol target does not. + // Leiden therefore partitions it into a singleton, which is not emitted. + graph.addRelationship(makeRel('rel:singleton', 'fn:singleton', 'file:target')); + + const result = await processCommunities(graph, undefined, { engine: 'graphology' }); + const communityIds = new Set(result.communities.map((community) => community.id)); + + expect(result.communities).toHaveLength(1); + expect(result.communities[0].symbolCount).toBe(3); + expect(result.stats).toMatchObject({ totalCommunities: 2, nodesProcessed: 4 }); + expect(result.memberships.map((membership) => membership.nodeId).sort()).toEqual([ + 'fn:a', + 'fn:b', + 'fn:c', + ]); + expect(result.memberships.every((membership) => communityIds.has(membership.communityId))).toBe( + true, + ); + expect(result.rawMemberships?.map((membership) => membership.nodeId)).toEqual([ + 'fn:a', + 'fn:b', + 'fn:c', + 'fn:singleton', + ]); + expect( + result.rawMemberships?.filter((membership) => communityIds.has(membership.communityId)), + ).toEqual(result.memberships); + }); + + it('returns no memberships when every detected community is a filtered singleton', async () => { + const graph = createKnowledgeGraph(); + graph.addNode(makeNode('file:target', 'target', 'File')); + for (const id of ['fn:a', 'fn:b']) { + graph.addNode(makeNode(id, id.slice(3))); + graph.addRelationship(makeRel(`rel:${id}`, id, 'file:target')); + } + + const result = await processCommunities(graph, undefined, { engine: 'graphology' }); + + expect(result.communities).toEqual([]); + expect(result.memberships).toEqual([]); + expect(result.rawMemberships).toEqual([ + { nodeId: 'fn:a', communityId: 'comm_0' }, + { nodeId: 'fn:b', communityId: 'comm_1' }, + ]); + expect(result.stats).toMatchObject({ totalCommunities: 2, nodesProcessed: 2 }); + }); +}); diff --git a/gitnexus/test/unit/csv-escaping.test.ts b/gitnexus/test/unit/csv-escaping.test.ts index 6abb31511..2e8d8a36a 100644 --- a/gitnexus/test/unit/csv-escaping.test.ts +++ b/gitnexus/test/unit/csv-escaping.test.ts @@ -15,16 +15,25 @@ import { // ─── escapeCSVField ────────────────────────────────────────────────── describe('escapeCSVField', () => { - it('returns empty quoted string for null', () => { - expect(escapeCSVField(null)).toBe('""'); + it('returns an unquoted NULL field for null', () => { + expect(escapeCSVField(null)).toBe(''); }); - it('returns empty quoted string for undefined', () => { - expect(escapeCSVField(undefined)).toBe('""'); + it('returns an unquoted NULL field for undefined', () => { + expect(escapeCSVField(undefined)).toBe(''); }); - it('returns quoted empty string for empty input', () => { - expect(escapeCSVField('')).toBe('""'); + it('preserves the legacy NULL meaning of empty input', () => { + expect(escapeCSVField('')).toBe(''); + }); + + it('returns a NULL field when sanitization removes the entire value', () => { + expect(escapeCSVField('\x00\x01')).toBe(''); + }); + + it('keeps whitespace and zero as quoted non-null values', () => { + expect(escapeCSVField(' ')).toBe('" "'); + expect(escapeCSVField(0)).toBe('"0"'); }); it('wraps simple string in quotes', () => { diff --git a/gitnexus/test/unit/detect-changes-format.test.ts b/gitnexus/test/unit/detect-changes-format.test.ts index 18f21eeeb..32e3a57f4 100644 --- a/gitnexus/test/unit/detect-changes-format.test.ts +++ b/gitnexus/test/unit/detect-changes-format.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { formatDetectChangesResult } from '../../src/cli/detect-changes-format.js'; import { setCliLanguage } from '../../src/cli/i18n/index.js'; +import { parseDiffHunks } from '../../src/storage/git.js'; describe('formatDetectChangesResult — zero-symbol honesty (#3131)', () => { beforeEach(() => { @@ -62,6 +63,49 @@ describe('formatDetectChangesResult — zero-symbol honesty (#3131)', () => { expect(text).toBe('No changes detected.'); }); + it('explains unmapped source files without claiming a query failed or retry will repair the index', () => { + const text = formatDetectChangesResult({ + partial: true, + unmapped_files: ['src/index-lock.ts'], + summary: { changed_count: 0, affected_count: 0, changed_files: 1, risk_level: 'unknown' }, + }); + expect(text).toContain('PARTIAL RESULT'); + expect(text).toContain('src/index-lock.ts'); + expect(text).toMatch(/rebuild/i); + expect(text).not.toMatch(/queries failed|No changes detected|no indexed symbols overlap/i); + }); + + it('escapes Git-decoded terminal controls while leaving structured paths intact', () => { + const diff = [ + 'diff --git "a/evil\\033]52;c;VEVTVA==\\007.ts" "b/evil\\033]52;c;VEVTVA==\\007.ts"', + 'old mode 100644', + 'new mode 100755', + ].join('\n'); + const paths = parseDiffHunks(diff).map((file) => file.filePath); + expect(paths[0]).toContain('\u001b'); + const text = formatDetectChangesResult({ + partial: true, + unmapped_files: paths, + summary: { changed_count: 0, changed_files: 1, risk_level: 'unknown' }, + }); + expect(text).toContain('\\u001b]52;c;VEVTVA==\\u0007.ts'); + expect(text).not.toContain('\u001b'); + expect(text).not.toContain('\u0007'); + expect(paths[0]).toContain('\u0007'); + }); + + it('leads a populated summary with the incomplete source-mapping explanation', () => { + const text = formatDetectChangesResult({ + partial: true, + unmapped_files: ['other.ts'], + summary: { changed_count: 1, changed_files: 2, affected_count: 0, risk_level: 'unknown' }, + changed_symbols: [{ type: 'Function', name: 'known', filePath: 'code.py' }], + }); + expect(text.indexOf('other.ts')).toBeLessThan(text.indexOf('Changes:')); + expect(text).toContain('known'); + expect(text).toContain('unknown'); + }); + it('localizes the production clean-tree payload that carries English summary.message', () => { setCliLanguage('zh-CN'); const text = formatDetectChangesResult({ diff --git a/gitnexus/test/unit/detect-changes-hunk-scale.test.ts b/gitnexus/test/unit/detect-changes-hunk-scale.test.ts index a9e1035fc..b7f472dea 100644 --- a/gitnexus/test/unit/detect-changes-hunk-scale.test.ts +++ b/gitnexus/test/unit/detect-changes-hunk-scale.test.ts @@ -133,11 +133,11 @@ interface DetectChangesResult { partial?: boolean; } -async function runDetectChanges(): Promise { +async function runDetectChanges(scope = 'unstaged'): Promise { const backend = new LocalBackend(); await backend.init(); return (await backend.callTool('detect_changes', { - scope: 'unstaged', + scope, repo: 'hunk-scale-repo', })) as DetectChangesResult; } @@ -238,6 +238,135 @@ beforeEach(() => { }); describe('#2915 detect_changes hunk scaling', () => { + it('withholds a low-risk verdict when a changed source file maps to no symbols', async () => { + const result = await detectChangesForCodePy('def vanished():\n return 2\n'); + expect(result.summary.changed_files).toBe(1); + expect(result.summary.changed_count).toBe(0); + expect(result.summary.risk_level).toBe('unknown'); + expect(result.partial).toBe(true); + expect(result).toHaveProperty('unmapped_files', ['code.py']); + }); + + it.each(['notes.txt', 'README.md', 'config.json', 'config.yaml'])( + 'keeps ordinary non-source changes in %s measurable with zero symbols', + async (filePath) => { + const repoDir = makeRepo([filePath], 2); + writeFileSync(path.join(repoDir, filePath), 'changed\nline 2\n'); + registerRepo(repoDir); + const result = await runDetectChanges(); + expect(result.summary.risk_level).toBe('low'); + expect(result.partial).toBeUndefined(); + }, + ); + + it.each(['.html', '.htm', '.ejs', '.hbs', '.blade.php'])( + 'withholds ranked risk when a changed %s template maps to no symbols', + async (extension) => { + const filePath = `views/orders${extension}`; + const repoDir = makeRepo([filePath], 2); + writeFileSync(path.join(repoDir, filePath), '
\n'); + registerRepo(repoDir); + const result = await runDetectChanges(); + expect(result.summary).toMatchObject({ + changed_files: 1, + changed_count: 0, + risk_level: 'unknown', + }); + expect(result.partial).toBe(true); + expect(result).toHaveProperty('unmapped_files', [filePath]); + }, + ); + + it.each(['.html', '.htm', '.ejs', '.hbs', '.blade.php'])( + 'recognizes the template side of a pure %s-to-text rename', + async (extension) => { + const filePath = `orders${extension}`; + const repoDir = makeRepo([filePath], 2); + execFileSync('git', ['mv', filePath, 'orders.txt'], { cwd: repoDir }); + registerRepo(repoDir); + const result = await runDetectChanges('staged'); + expect(result.summary).toMatchObject({ + changed_files: 1, + changed_count: 0, + risk_level: 'unknown', + }); + expect(result.partial).toBe(true); + expect(result).toHaveProperty('unmapped_files', ['orders.txt']); + }, + ); + + it('keeps mapped template changes measurable', async () => { + const filePath = 'views/orders.blade.php'; + const repoDir = makeRepo([filePath], 2); + writeFileSync(path.join(repoDir, filePath), '
\nline 2\n'); + registerRepo(repoDir); + mockSymbolRows([{ name: 'orders', filePath, startLine: 0, endLine: 1 }]); + const result = await runDetectChanges(); + expect(result.summary).toMatchObject({ + changed_files: 1, + changed_count: 1, + risk_level: 'low', + }); + expect(result.changed_symbols.map((symbol) => symbol.name)).toEqual(['orders']); + expect(result.partial).toBeUndefined(); + expect(result).not.toHaveProperty('unmapped_files'); + }); + + it('withholds a low-risk verdict for an unmapped source rename without hunks', async () => { + const repoDir = makeRepo(['code.py'], 2); + execFileSync('git', ['mv', 'code.py', 'renamed.py'], { cwd: repoDir }); + registerRepo(repoDir); + const result = await runDetectChanges('staged'); + expect(result.summary.changed_files).toBe(1); + expect(result.summary.risk_level).toBe('unknown'); + expect(result.partial).toBe(true); + expect(result).toHaveProperty('unmapped_files', ['renamed.py']); + }); + + it('recognizes the source side of a pure source-to-text rename', async () => { + const repoDir = makeRepo(['code.py'], 2); + execFileSync('git', ['mv', 'code.py', 'code.txt'], { cwd: repoDir }); + registerRepo(repoDir); + const result = await runDetectChanges('staged'); + expect(result.summary).toMatchObject({ + changed_files: 1, + changed_count: 0, + risk_level: 'unknown', + }); + expect(result.partial).toBe(true); + expect(result).toHaveProperty('unmapped_files', ['code.txt']); + }); + + it.each(['.jcl', '.job', '.proc', '.copybook', '.JCL', '.COPYBOOK'])( + 'withholds ranked risk for an unmapped ingestion-supported %s rename', + async (extension) => { + const repoDir = makeRepo([`source${extension}`], 2); + execFileSync('git', ['mv', `source${extension}`, `renamed${extension}`], { cwd: repoDir }); + registerRepo(repoDir); + const result = await runDetectChanges('staged'); + expect(result.summary.risk_level).toBe('unknown'); + expect(result.partial).toBe(true); + expect(result).toHaveProperty('unmapped_files', [`renamed${extension}`]); + }, + ); + + it('retains mapped symbols while withholding ranked risk for an unmapped source', async () => { + const repoDir = makeRepo(['code.py', 'other.ts'], 2); + writeFileSync(path.join(repoDir, 'code.py'), 'changed\nline 2\n'); + writeFileSync(path.join(repoDir, 'other.ts'), 'changed\nline 2\n'); + registerRepo(repoDir); + mockSymbolRows([{ name: 'known', startLine: 0, endLine: 1 }]); + const result = await runDetectChanges(); + expect(result.summary).toMatchObject({ + changed_files: 2, + changed_count: 1, + risk_level: 'unknown', + }); + expect(result.changed_symbols.map((symbol) => symbol.name)).toEqual(['known']); + expect(result.partial).toBe(true); + expect(result).toHaveProperty('unmapped_files', ['other.ts']); + }); + it('sends the same query for a 3,000-hunk diff as for a 1-hunk diff', async () => { const oneHunkRepo = makeRepo(['big.txt'], 12000); editEveryNthLine(oneHunkRepo, 'big.txt', 12000, 12000); diff --git a/gitnexus/test/unit/embedding-recovery-race.test.ts b/gitnexus/test/unit/embedding-recovery-race.test.ts new file mode 100644 index 000000000..145f860ec --- /dev/null +++ b/gitnexus/test/unit/embedding-recovery-race.test.ts @@ -0,0 +1,199 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import os from 'node:os'; +import path from 'node:path'; +import { readEmbeddingRecovery } from '../../src/storage/embedding-recovery.js'; + +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + constants: { ...actual.constants }, + lstatSync: vi.fn(actual.lstatSync), + openSync: vi.fn(actual.openSync), + fstatSync: vi.fn(actual.fstatSync), + readFileSync: vi.fn(actual.readFileSync), + closeSync: vi.fn(actual.closeSync), + }; +}); + +const actual = await vi.importActual('node:fs'); +const stagingFile = 'lbug.staging.6e34c761-bf58-46cc-8b54-78d11607bc46'; +const receipt = { + embeddingCheckpoint: { + kind: 'interrupted', + at: '2026-10-03T12:00:00.000Z', + nodesProcessed: 1, + totalNodes: 2, + chunksProcessed: 1, + model: 'test-model', + dimensions: 2, + provider: 'local', + recovery: { stagingFile, schemaFingerprint: 'test-schema', unsafeNodeIds: [] }, + }, +}; + +let dir: string; +let metadataPath: string; +beforeEach(() => { + vi.mocked(fs.lstatSync).mockImplementation(actual.lstatSync); + vi.mocked(fs.openSync).mockImplementation(actual.openSync); + vi.mocked(fs.fstatSync).mockImplementation(actual.fstatSync); + vi.mocked(fs.readFileSync).mockImplementation(actual.readFileSync); + vi.mocked(fs.closeSync).mockImplementation(actual.closeSync); + Object.assign(fs.constants, actual.constants); + vi.clearAllMocks(); + dir = actual.mkdtempSync(path.join(os.tmpdir(), 'gnx-recovery-metadata-race-')); + metadataPath = path.join(dir, 'gitnexus.json'); + actual.writeFileSync(path.join(dir, stagingFile), 'stage'); +}); +afterEach(() => actual.rmSync(dir, { recursive: true, force: true })); + +describe('readEmbeddingRecovery metadata races', () => { + it('does not read replacement metadata after checking the original file', () => { + actual.writeFileSync(metadataPath, JSON.stringify({ embeddingCheckpoint: null })); + let replaced = false; + vi.mocked(fs.lstatSync).mockImplementation((...args) => { + const stat = actual.lstatSync(...args); + if (args[0] === metadataPath && !replaced) { + replaced = true; + actual.renameSync(metadataPath, path.join(dir, 'original-metadata.json')); + actual.writeFileSync(metadataPath, JSON.stringify(receipt)); + } + return stat; + }); + + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + expect(replaced).toBe(true); + const descriptor = vi.mocked(fs.openSync).mock.results[0]?.value; + expect(typeof descriptor).toBe('number'); + expect(fs.readFileSync).toHaveBeenCalledWith(descriptor, 'utf8'); + expect(fs.closeSync).toHaveBeenCalledWith(descriptor); + }); + + it('rejects a file replaced between opening and checking its identity', () => { + actual.writeFileSync(metadataPath, JSON.stringify(receipt)); + let replaced = false; + vi.mocked(fs.openSync).mockImplementation((...args) => { + const descriptor = actual.openSync(...args); + if (args[0] === metadataPath && !replaced) { + replaced = true; + actual.renameSync(metadataPath, path.join(dir, 'original-metadata.json')); + actual.writeFileSync(metadataPath, JSON.stringify(receipt)); + } + return descriptor; + }); + + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + expect(replaced).toBe(true); + expect(fs.readFileSync).not.toHaveBeenCalled(); + expect(fs.closeSync).toHaveBeenCalledOnce(); + }); + + it('reads regular metadata when no-follow opens are unavailable', () => { + Object.assign(fs.constants, { O_NOFOLLOW: 0, O_NONBLOCK: 0 }); + actual.writeFileSync(metadataPath, JSON.stringify(receipt)); + + expect(readEmbeddingRecovery(dir)?.stagingFile).toBe(stagingFile); + expect(fs.closeSync).toHaveBeenCalledOnce(); + }); + + it('refuses unverifiable file identity when no-follow opens are unavailable', () => { + Object.assign(fs.constants, { O_NOFOLLOW: 0, O_NONBLOCK: 0 }); + actual.writeFileSync(metadataPath, JSON.stringify(receipt)); + vi.mocked(fs.fstatSync).mockImplementation((...args) => { + const stat = actual.fstatSync(...args); + Object.defineProperty(stat, 'ino', { value: 0n }); + return stat; + }); + vi.mocked(fs.lstatSync).mockImplementation((...args) => { + const stat = actual.lstatSync(...args); + Object.defineProperty(stat, 'ino', { value: 0n }); + return stat; + }); + + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + expect(fs.readFileSync).not.toHaveBeenCalled(); + expect(fs.closeSync).toHaveBeenCalledOnce(); + }); + + it('rejects a symlink introduced before opening without no-follow support', () => { + Object.assign(fs.constants, { O_NOFOLLOW: 0, O_NONBLOCK: 0 }); + actual.writeFileSync(metadataPath, JSON.stringify({ embeddingCheckpoint: null })); + const target = path.join(dir, 'foreign-metadata.json'); + actual.writeFileSync(target, JSON.stringify(receipt)); + let replaced = false; + vi.mocked(fs.openSync).mockImplementation((...args) => { + if (args[0] === metadataPath && !replaced) { + replaced = true; + actual.rmSync(metadataPath); + actual.symlinkSync(target, metadataPath); + } + return actual.openSync(...args); + }); + + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + expect(replaced).toBe(true); + expect(fs.readFileSync).not.toHaveBeenCalled(); + expect(fs.closeSync).toHaveBeenCalledOnce(); + }); + + it('never falls back from a dangling primary symlink without no-follow support', () => { + Object.assign(fs.constants, { O_NOFOLLOW: 0, O_NONBLOCK: 0 }); + actual.symlinkSync(path.join(dir, 'missing-metadata.json'), metadataPath); + actual.writeFileSync(path.join(dir, 'meta.json'), JSON.stringify(receipt)); + + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + expect(fs.readFileSync).not.toHaveBeenCalled(); + }); + + it('rejects a symlinked legacy receipt when the primary is absent', () => { + Object.assign(fs.constants, { O_NOFOLLOW: 0, O_NONBLOCK: 0 }); + const target = path.join(dir, 'foreign-metadata.json'); + actual.writeFileSync(target, JSON.stringify(receipt)); + actual.symlinkSync(target, path.join(dir, 'meta.json')); + + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + expect(fs.readFileSync).not.toHaveBeenCalled(); + expect(fs.closeSync).toHaveBeenCalledOnce(); + }); + + it('closes a descriptor when fstat fails without falling back to legacy metadata', () => { + actual.writeFileSync(metadataPath, JSON.stringify(receipt)); + actual.writeFileSync(path.join(dir, 'meta.json'), JSON.stringify(receipt)); + vi.mocked(fs.fstatSync).mockImplementationOnce(() => { + throw Object.assign(new Error('stat failed'), { code: 'EIO' }); + }); + + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + expect(fs.readFileSync).not.toHaveBeenCalled(); + expect(fs.closeSync).toHaveBeenCalledOnce(); + }); + + it('closes a descriptor when JSON parsing fails without falling back', () => { + actual.writeFileSync(metadataPath, '{'); + actual.writeFileSync(path.join(dir, 'meta.json'), JSON.stringify(receipt)); + + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + expect(fs.closeSync).toHaveBeenCalledOnce(); + }); + + it.skipIf(process.platform === 'win32')('rejects a substituted FIFO without blocking', () => { + actual.writeFileSync(metadataPath, JSON.stringify(receipt)); + let replaced = false; + vi.mocked(fs.openSync).mockImplementation((...args) => { + if (args[0] === metadataPath && !replaced) { + replaced = true; + actual.rmSync(metadataPath); + execFileSync('mkfifo', [metadataPath]); + } + return actual.openSync(...args); + }); + + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + expect(replaced).toBe(true); + expect(fs.readFileSync).not.toHaveBeenCalled(); + expect(fs.closeSync).toHaveBeenCalledOnce(); + }); +}); diff --git a/gitnexus/test/unit/embedding-recovery.test.ts b/gitnexus/test/unit/embedding-recovery.test.ts new file mode 100644 index 000000000..a8f1064a0 --- /dev/null +++ b/gitnexus/test/unit/embedding-recovery.test.ts @@ -0,0 +1,175 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + readEmbeddingRecovery, + resolveEmbeddingRecovery, +} from '../../src/storage/embedding-recovery.js'; + +const stagingFile = 'lbug.staging.6e34c761-bf58-46cc-8b54-78d11607bc46'; +const familySuffixes = [ + '', + '.wal', + '.shadow', + '.wal.checkpoint', + '.lock', + '.checkpoint.intent.lock', + '.checkpoint.apply.lock', +]; +const checkpoint = () => ({ + kind: 'interrupted', + at: '2026-10-03T12:00:00.000Z', + nodesProcessed: 1, + totalNodes: 3, + chunksProcessed: 2, + model: 'test-model', + dimensions: 2, + provider: 'local', + pendingNodeIds: ['active'], + recovery: { + stagingFile, + schemaFingerprint: 'test-schema', + unsafeNodeIds: ['active', 'incomplete-restore'], + }, +}); + +let dir: string; +beforeEach(() => { + dir = mkdtempSync(path.join(os.tmpdir(), 'gnx-embedding-recovery-')); + writeFileSync(path.join(dir, stagingFile), 'stage'); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +describe('resolveEmbeddingRecovery', () => { + it('resolves the exact staged generation and carries all unsafe node IDs', () => { + expect(resolveEmbeddingRecovery(dir, checkpoint())).toEqual({ + ...checkpoint().recovery, + dbPath: path.join(dir, stagingFile), + familyFiles: familySuffixes.map((suffix) => stagingFile + suffix), + }); + }); + + it.each([ + '../lbug.staging.6e34c761-bf58-46cc-8b54-78d11607bc46', + '/tmp/lbug.staging.6e34c761-bf58-46cc-8b54-78d11607bc46', + 'branches/foreign/lbug.staging.6e34c761-bf58-46cc-8b54-78d11607bc46', + '..\\lbug.staging.6e34c761-bf58-46cc-8b54-78d11607bc46', + 'lbug', + 'lbug.new', + 'lbug.staging.orphan', + `${stagingFile}.wal`, + 'lbug.staging.00000000-0000-4000-8000-000000000000', + ])('rejects a foreign, unrecognized or missing generation: %s', (filename) => { + const marker = checkpoint(); + marker.recovery.stagingFile = filename; + expect(resolveEmbeddingRecovery(dir, marker)).toBeUndefined(); + }); + + it.each([ + null, + [], + { recovery: checkpoint().recovery }, + { ...checkpoint(), kind: 'partial' }, + { ...checkpoint(), kind: 'unverified-count' }, + { ...checkpoint(), kind: 'unknown' }, + { ...checkpoint(), at: 'invalid-time' }, + { ...checkpoint(), nodesProcessed: -1 }, + { ...checkpoint(), nodesProcessed: 4 }, + { ...checkpoint(), totalNodes: 1.5 }, + { ...checkpoint(), chunksProcessed: Number.NaN }, + { ...checkpoint(), model: '' }, + { ...checkpoint(), provider: null }, + { ...checkpoint(), dimensions: 0 }, + { ...checkpoint(), pendingNodeIds: [42] }, + { ...checkpoint(), pendingNodeIds: ['unexcluded'] }, + { ...checkpoint(), recovery: { ...checkpoint().recovery, schemaFingerprint: '' } }, + { ...checkpoint(), recovery: { ...checkpoint().recovery, unsafeNodeIds: undefined } }, + { ...checkpoint(), recovery: { ...checkpoint().recovery, unsafeNodeIds: [''] } }, + ])('rejects malformed or incompatible checkpoint shape %#', (marker) => { + expect(resolveEmbeddingRecovery(dir, marker)).toBeUndefined(); + }); + + it('accepts a durable restored stage even before a new embedding window completes', () => { + expect( + resolveEmbeddingRecovery(dir, { + ...checkpoint(), + nodesProcessed: 0, + chunksProcessed: 0, + pendingNodeIds: [], + }), + ).toBeDefined(); + }); + + it('rejects a directory in place of the staged database', () => { + rmSync(path.join(dir, stagingFile)); + mkdirSync(path.join(dir, stagingFile)); + expect(resolveEmbeddingRecovery(dir, checkpoint())).toBeUndefined(); + }); + + it.each(familySuffixes)('rejects a symlink in the staged family: %s', (suffix) => { + const target = path.join(dir, 'external-file'); + writeFileSync(target, 'external'); + const candidate = path.join(dir, stagingFile + suffix); + rmSync(candidate, { force: true }); + symlinkSync(target, candidate); + expect(resolveEmbeddingRecovery(dir, checkpoint())).toBeUndefined(); + }); + + it.each(familySuffixes.slice(1))('rejects a dangling staged sidecar symlink: %s', (suffix) => { + symlinkSync(path.join(dir, 'missing'), path.join(dir, stagingFile + suffix)); + expect(resolveEmbeddingRecovery(dir, checkpoint())).toBeUndefined(); + }); + + it.each(familySuffixes.slice(1))('rejects a non-file staged sidecar: %s', (suffix) => { + mkdirSync(path.join(dir, stagingFile + suffix)); + expect(resolveEmbeddingRecovery(dir, checkpoint())).toBeUndefined(); + }); +}); + +describe('readEmbeddingRecovery', () => { + const writeMeta = (filename: string, marker: unknown = checkpoint()): void => { + writeFileSync(path.join(dir, filename), JSON.stringify({ embeddingCheckpoint: marker })); + }; + + it('prefers the primary metadata file over a stale legacy reference', () => { + writeMeta('gitnexus.json'); + writeMeta('meta.json', null); + expect(readEmbeddingRecovery(dir)?.stagingFile).toBe(stagingFile); + }); + + it('loads the legacy mirror only when the primary metadata file is absent', () => { + writeMeta('meta.json'); + expect(readEmbeddingRecovery(dir)?.stagingFile).toBe(stagingFile); + }); + + it('does not resurrect a legacy reference when the primary file is malformed', () => { + writeFileSync(path.join(dir, 'gitnexus.json'), '{'); + writeMeta('meta.json'); + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + }); + + it('does not fall back from valid primary metadata with no recovery reference', () => { + writeMeta('gitnexus.json', null); + writeMeta('meta.json'); + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + }); + + it('rejects symlinked primary metadata rather than reading a foreign receipt', () => { + writeMeta('meta.json'); + symlinkSync(path.join(dir, 'meta.json'), path.join(dir, 'gitnexus.json')); + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + }); + + it('resolves branch-slot provenance within that slot despite a flat storagePath', () => { + const branchSlot = path.join(dir, 'branches', 'feature'); + mkdirSync(branchSlot, { recursive: true }); + writeFileSync(path.join(branchSlot, stagingFile), 'branch-stage'); + writeFileSync( + path.join(branchSlot, 'gitnexus.json'), + JSON.stringify({ storagePath: dir, embeddingCheckpoint: checkpoint() }), + ); + expect(readEmbeddingRecovery(branchSlot)?.dbPath).toBe(path.join(branchSlot, stagingFile)); + expect(readEmbeddingRecovery(dir)).toBeUndefined(); + }); +}); diff --git a/gitnexus/test/unit/embeddings-sync-command.test.ts b/gitnexus/test/unit/embeddings-sync-command.test.ts index 01afa40c3..ec33fee6d 100644 --- a/gitnexus/test/unit/embeddings-sync-command.test.ts +++ b/gitnexus/test/unit/embeddings-sync-command.test.ts @@ -3,13 +3,15 @@ * index lock, missing-DB preflight, identity fail-closed, tri-state count, * closeLbug masking, and hash-only cache load. */ -import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const { acquireIndexLockMock, + ensurePrivateSharedGraphMock, releaseMock, getStoragePathsMock, loadMetaMock, @@ -27,6 +29,7 @@ const { reapEmbeddingSidecarMock, } = vi.hoisted(() => ({ acquireIndexLockMock: vi.fn(), + ensurePrivateSharedGraphMock: vi.fn(), releaseMock: vi.fn(), getStoragePathsMock: vi.fn(), loadMetaMock: vi.fn(), @@ -48,6 +51,10 @@ vi.mock('../../src/storage/git.js', () => ({ getGitRoot: () => '/tmp/emb-sync-repo', })); +vi.mock('../../src/core/shared-store-analyze.js', () => ({ + ensurePrivateSharedGraph: (...args: unknown[]) => ensurePrivateSharedGraphMock(...args), +})); + vi.mock('../../src/storage/index-lock.js', async (importOriginal) => ({ ...(await importOriginal()), acquireIndexLock: (...args: unknown[]) => acquireIndexLockMock(...args), @@ -113,6 +120,25 @@ async function run(inputPath = '/tmp/emb-sync-repo') { await embeddingsSyncCommand(inputPath); } +async function useRealIndexLock() { + vi.stubEnv('GITNEXUS_INDEX_LOCK_BACKEND', 'file'); + const actual = await vi.importActual( + '../../src/storage/index-lock.js', + ); + acquireIndexLockMock.mockImplementation( + async (...args: Parameters) => { + const lock = await actual.acquireIndexLock(...args); + return { + ...lock, + release: () => { + lock.release(); + releaseMock(); + }, + }; + }, + ); +} + describe('embeddingsSyncCommand writer safety (#3065)', () => { const tmpDirs: string[] = []; const originalEmbeddingUrl = process.env.GITNEXUS_EMBEDDING_URL; @@ -132,6 +158,7 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { beforeEach(() => { vi.resetModules(); acquireIndexLockMock.mockReset().mockResolvedValue(lockHandle()); + ensurePrivateSharedGraphMock.mockReset().mockResolvedValue(true); releaseMock.mockReset(); getStoragePathsMock.mockReset(); loadMetaMock.mockReset().mockResolvedValue({ ...BASE_META }); @@ -156,6 +183,7 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { }); afterEach(async () => { + vi.unstubAllEnvs(); if (originalEmbeddingUrl === undefined) delete process.env.GITNEXUS_EMBEDDING_URL; else process.env.GITNEXUS_EMBEDDING_URL = originalEmbeddingUrl; if (originalEmbeddingModel === undefined) delete process.env.GITNEXUS_EMBEDDING_MODEL; @@ -183,7 +211,7 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { await run(); - expect(acquireIndexLockMock).toHaveBeenCalledWith(dir); + expect(acquireIndexLockMock).toHaveBeenCalledWith(dir, { sweep: false }); expect(order[0]).toBe('lock'); expect(order.indexOf('loadMeta')).toBeGreaterThan(order.indexOf('lock')); expect(order.indexOf('init')).toBeGreaterThan(order.indexOf('loadMeta')); @@ -299,6 +327,90 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { expect(releaseMock).toHaveBeenCalled(); }); + it.each([ + { + name: 'valid staged receipt', + recovery: { + stagingFile: 'lbug.staging.12345678-1234-4123-8123-123456789abc', + schemaFingerprint: 'test-schema', + unsafeNodeIds: ['n2', 'inherited-window-node'], + }, + }, + { name: 'null receipt', recovery: null }, + { name: 'malformed receipt', recovery: { stagingFile: 'invalid' } }, + { name: 'false receipt', recovery: false }, + ])('preserves a $name before any writable sync work', async ({ recovery }) => { + const { dir, lbugPath, metaPath } = await store(); + await useRealIndexLock(); + const lockPath = path.join(dir, 'analyze.lock'); + const sourcePath = path.join(dir, 'lbug.staging.12345678-1234-4123-8123-123456789abc'); + await writeFile(sourcePath, 'completed paid vectors'); + await writeFile(`${sourcePath}.wal`, 'unfinished window'); + const metadataBytes = JSON.stringify({ + ...BASE_META, + embeddingCheckpoint: { + ...IDENTITY, + at: '2026-01-01T00:00:00.000Z', + nodesProcessed: 1, + totalNodes: 2, + chunksProcessed: 1, + kind: 'interrupted', + pendingNodeIds: ['n2'], + recovery, + }, + }); + await writeFile(metaPath, metadataBytes); + loadMetaMock.mockImplementation(async () => { + expect(existsSync(lockPath)).toBe(true); + return JSON.parse(await readFile(metaPath, 'utf8')); + }); + resolveEmbeddingRuntimeMock.mockReturnValue(null); + + await expect(run()).rejects.toThrow( + /staged embeddings.*Run `gitnexus analyze` to recover them first/, + ); + + expect(acquireIndexLockMock).toHaveBeenCalledWith(dir, { sweep: false }); + expect(loadMetaMock.mock.invocationCallOrder[0]).toBeGreaterThan( + acquireIndexLockMock.mock.invocationCallOrder[0]!, + ); + expect(ensurePrivateSharedGraphMock).not.toHaveBeenCalled(); + expect(resolveEmbeddingIdentityMock).not.toHaveBeenCalled(); + expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled(); + expect(initLbugMock).not.toHaveBeenCalled(); + expect(runEmbeddingPipelineMock).not.toHaveBeenCalled(); + expect(saveMetaMock).not.toHaveBeenCalled(); + expect(releaseMock).toHaveBeenCalledTimes(1); + expect(existsSync(lockPath)).toBe(false); + expect(await readFile(metaPath, 'utf8')).toBe(metadataBytes); + expect(await readFile(lbugPath, 'utf8')).toBe('db'); + expect(await readFile(sourcePath, 'utf8')).toBe('completed paid vectors'); + expect(await readFile(`${sourcePath}.wal`, 'utf8')).toBe('unfinished window'); + }); + + it('sweeps orphaned staging files before writable sync work without a recovery receipt', async () => { + const { dir, metaPath } = await store(); + await useRealIndexLock(); + await writeFile(metaPath, JSON.stringify(BASE_META)); + const sourcePath = path.join(dir, 'lbug.staging.12345678-1234-4123-8123-123456789abc'); + await writeFile(sourcePath, 'orphaned database'); + await writeFile(`${sourcePath}.wal`, 'orphaned WAL'); + loadMetaMock.mockImplementation(async () => JSON.parse(await readFile(metaPath, 'utf8'))); + ensurePrivateSharedGraphMock.mockImplementation(async () => { + expect(existsSync(path.join(dir, 'analyze.lock'))).toBe(true); + expect(existsSync(sourcePath)).toBe(false); + expect(existsSync(`${sourcePath}.wal`)).toBe(false); + return true; + }); + + await run(); + + expect(ensurePrivateSharedGraphMock).toHaveBeenCalledTimes(1); + expect(runEmbeddingPipelineMock).toHaveBeenCalledTimes(1); + expect(releaseMock).toHaveBeenCalledTimes(1); + expect(existsSync(path.join(dir, 'analyze.lock'))).toBe(false); + }); + it('persists an interrupted checkpoint from the pipeline checkpoint callbacks', async () => { // The resume contract lives in these callbacks; a mock that never invokes // them leaves the whole save path unexecuted. diff --git a/gitnexus/test/unit/eval-formatters.test.ts b/gitnexus/test/unit/eval-formatters.test.ts index bfc025432..d43ec5f38 100644 --- a/gitnexus/test/unit/eval-formatters.test.ts +++ b/gitnexus/test/unit/eval-formatters.test.ts @@ -641,7 +641,7 @@ describe('formatDetectChangesResult', () => { // counts at zero. Without the note the pre-commit gate reads as "clean". const result = formatDetectChangesResult({ partial: true, summary: { changed_count: 0 } }); expect(result).toContain('PARTIAL RESULT'); - expect(result).toContain('a graph query failed'); + expect(result).toContain('mapping is incomplete'); expect(result).not.toContain('No changes detected.'); }); diff --git a/gitnexus/test/unit/format-path.test.ts b/gitnexus/test/unit/format-path.test.ts new file mode 100644 index 000000000..a24446fe5 --- /dev/null +++ b/gitnexus/test/unit/format-path.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest'; +import { formatPathForTerminal } from '../../src/cli/format-path.js'; + +describe('terminal path rendering', () => { + it('preserves ordinary Unicode and spaces', () => { + expect(formatPathForTerminal('src/王 name.ts')).toBe('src/王 name.ts'); + }); + + it.each(['\u001b', '\u0007', '\r', '\n', '\u007f', '\u0085', '\u009b'])( + 'renders control %j visibly without changing the decoded path', + (control) => { + const filePath = `src/a${control}.ts`; + const rendered = formatPathForTerminal(filePath); + expect(rendered).not.toMatch(/[\u0000-\u001f\u007f-\u009f]/); + expect(JSON.parse(rendered)).toBe(filePath); + }, + ); +}); diff --git a/gitnexus/test/unit/group/go-gin-route-groups.test.ts b/gitnexus/test/unit/group/go-gin-route-groups.test.ts new file mode 100644 index 000000000..bdb5bde56 --- /dev/null +++ b/gitnexus/test/unit/group/go-gin-route-groups.test.ts @@ -0,0 +1,969 @@ +/** + * Group HTTP-contract layer: Go gin/echo framework routes registered through + * route groups and method-value handlers. Exercises `GO_HTTP_PLUGIN.scan` + * directly with a real tree-sitter parser, asserting the FULL registered path + * (every enclosing `x := y.Group("/p")` prefix joined in) and the handler name + * the group layer resolves by. The last block runs the real extractor on a Go + * provider repo and a fetch() consumer repo and pairs them with `runExactMatch`. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import Parser from 'tree-sitter'; +import Go from 'tree-sitter-go'; +import { GO_HTTP_PLUGIN } from '../../../src/core/group/extractors/http-patterns/go.js'; +import { + HttpRouteExtractor, + RESOLVE_BY_NAME_QUERY, +} from '../../../src/core/group/extractors/http-route-extractor.js'; +import { runExactMatch } from '../../../src/core/group/matching.js'; +import type { RepoHandle, StoredContract } from '../../../src/core/group/types.js'; + +const parser = new Parser(); + +interface Provider { + method: string; + path: string; + name: string | null; +} + +function providers(src: string): Provider[] { + parser.setLanguage(Go); + return GO_HTTP_PLUGIN.scan(parser.parse(src)) + .filter((d) => d.role === 'provider') + .map(({ method, path: p, name }) => ({ method, path: p, name })); +} + +// Mirrors the shapes of a real gin `RegisterRoutes`: an engine-level group, +// `{ }` blocks, nested groups three levels deep, empty-prefix groups used only +// to attach middleware, method-value / package-func / identifier / inline +// handlers, and variadic middleware before the handler. +const GIN_ROUTES = `package handlers + +import "github.com/gin-gonic/gin" + +func RegisterRoutes(r *gin.Engine, svc *service.Service) { + playerHandler := NewPlayerHandler(svc.Player) + matchHandler := NewMatchHandler(svc.Match) + r.GET("/ping", pingHandle) + v1 := r.Group("/api/v1") + { + v1.GET("/health", func(c *gin.Context) { c.Status(200) }) + v1.GET("/players", playerHandler.GetPlayersHandle) + v1.POST("/upload/avatar", UploadAvatarHandle) + v1.GET("/exports", exports.ListExportsHandle) + v1.PATCH("/players/:playerId", middleware.AuthRequired(svc.Auth), playerHandler.UpdatePlayerHandle) + admin := v1.Group("/admin") + admin.Use(middleware.AdminRequired(svc.AdminControl)) + { + admin.POST("/seasons/:seasonId/rounds/:roundId/unfinalize", matchHandler.UnfinalizeRoundHandle) + newsAdmin := admin.Group("/news") + { + newsAdmin.DELETE("/:id", newsHandler.DeleteNewsHandle) + } + knockoutAdmin := admin.Group("", middleware.KnockoutGuard()) + knockoutAdmin.POST("/seasons/:seasonId/knockout", knockoutHandler.CreateKnockoutHandle) + adminOnly := admin.Group("") + adminOnly.PUT("/seasons/:seasonId/streak-config", streakHandler.SaveStreakConfigHandle) + } + } +} +`; + +describe('GO_HTTP_PLUGIN — gin route groups', () => { + const got = providers(GIN_ROUTES); + const find = (method: string, p: string) => got.find((d) => d.method === method && d.path === p); + + it('emits exactly one provider per registered route (Use() and Group() are not routes)', () => { + expect(got).toHaveLength(10); + }); + + it('keeps a route on the engine root, outside any group, at its literal path', () => { + expect(find('GET', '/ping')).toEqual({ method: 'GET', path: '/ping', name: 'pingHandle' }); + }); + + it('prefixes an inline func_literal handler and leaves it unnamed', () => { + expect(find('GET', '/api/v1/health')).toEqual({ + method: 'GET', + path: '/api/v1/health', + name: null, + }); + }); + + it('accepts a method-value handler and names it by its field', () => { + expect(find('GET', '/api/v1/players')?.name).toBe('GetPlayersHandle'); + }); + + it('accepts a package-qualified function handler and names it by its field', () => { + expect(find('GET', '/api/v1/exports')?.name).toBe('ListExportsHandle'); + }); + + it('keeps an identifier handler', () => { + expect(find('POST', '/api/v1/upload/avatar')?.name).toBe('UploadAvatarHandle'); + }); + + it('binds the last argument, not the variadic middleware before it', () => { + expect(find('PATCH', '/api/v1/players/:playerId')?.name).toBe('UpdatePlayerHandle'); + }); + + it('joins a nested group inside a { } block', () => { + expect(find('POST', '/api/v1/admin/seasons/:seasonId/rounds/:roundId/unfinalize')?.name).toBe( + 'UnfinalizeRoundHandle', + ); + }); + + it('joins groups nested three levels deep', () => { + expect(find('DELETE', '/api/v1/admin/news/:id')?.name).toBe('DeleteNewsHandle'); + }); + + it('treats an empty-prefix group (with or without middleware) as its parent prefix', () => { + expect(find('POST', '/api/v1/admin/seasons/:seasonId/knockout')?.name).toBe( + 'CreateKnockoutHandle', + ); + expect(find('PUT', '/api/v1/admin/seasons/:seasonId/streak-config')?.name).toBe( + 'SaveStreakConfigHandle', + ); + }); +}); + +describe('GO_HTTP_PLUGIN — group binding edge cases', () => { + it('follows plain `=` assignment and `var x = …` declarations', () => { + const got = providers(`package main +func routes(r *gin.Engine) { + var api *gin.RouterGroup + api = r.Group("/v2") + api.GET("/a", h.A) + var ops = api.Group("/ops") + ops.GET("/b", h.B) +} +`); + expect(got).toEqual([ + { method: 'GET', path: '/v2/a', name: 'A' }, + { method: 'GET', path: '/v2/ops/b', name: 'B' }, + ]); + }); + + it('joins a Group() call chained directly onto the route call', () => { + expect( + providers(`package main +func routes(r *gin.Engine) { + r.Group("/inline").GET("/y", h.Y) +} +`), + ).toEqual([{ method: 'GET', path: '/inline/y', name: 'Y' }]); + }); + + it('uses the binding visible at the call site when a name is reused in sibling blocks', () => { + expect( + providers(`package main +func routes(r *gin.Engine) { + { + g := r.Group("/a") + g.GET("/x", h.AX) + } + { + g := r.Group("/b") + g.GET("/x", h.BX) + } +} +`), + ).toEqual([ + { method: 'GET', path: '/a/x', name: 'AX' }, + { method: 'GET', path: '/b/x', name: 'BX' }, + ]); + }); + + it('uses the latest assignment that precedes the route, not one after it', () => { + expect( + providers(`package main +func routes(r *gin.Engine) { + g := r.Group("/first") + g.GET("/x", h.X) + g = r.Group("/second") + g.GET("/y", h.Y) +} +`), + ).toEqual([ + { method: 'GET', path: '/first/x', name: 'X' }, + { method: 'GET', path: '/second/y', name: 'Y' }, + ]); + }); + + it('resolves a group captured by a closure registered inside the function', () => { + expect( + providers(`package main +func routes(r *gin.Engine) { + v1 := r.Group("/api/v1") + register := func() { + v1.GET("/inner", h.Inner) + } + register() +} +`), + ).toEqual([{ method: 'GET', path: '/api/v1/inner', name: 'Inner' }]); + }); + + it('keeps the literal path when the receiver is a parameter (group passed from another function)', () => { + expect( + providers(`package main +func registerAdmin(g *gin.RouterGroup) { + g.GET("/extra", extraHandle) +} +`), + ).toEqual([{ method: 'GET', path: '/extra', name: 'extraHandle' }]); + }); + + it('keeps the literal path when the receiver is bound to something other than Group()', () => { + expect( + providers(`package main +func routes() { + g := newRouter("/ignored") + g.GET("/x", h.X) +} +`), + ).toEqual([{ method: 'GET', path: '/x', name: 'X' }]); + }); + + it('does not resolve a group bound in a different function', () => { + expect( + providers(`package main +func a(r *gin.Engine) { + g := r.Group("/a") + g.GET("/in-a", h.A) +} +func b(g *gin.RouterGroup) { + g.GET("/in-b", h.B) +} +`), + ).toEqual([ + { method: 'GET', path: '/a/in-a', name: 'A' }, + { method: 'GET', path: '/in-b', name: 'B' }, + ]); + }); + + it('ignores a Group() whose prefix is not a string literal and keeps the route literal', () => { + expect( + providers(`package main +func routes(r *gin.Engine) { + g := r.Group(prefix) + g.GET("/x", h.X) +} +`), + ).toEqual([{ method: 'GET', path: '/x', name: 'X' }]); + }); + + it('uses the group bound by an if initializer in the body and in the else branch', () => { + expect( + providers(`package main +func routes(r *gin.Engine, cond bool) { + g := r.Group("/outer") + if g := r.Group("/inner"); cond { + g.GET("/x", h.X) + } else { + g.GET("/y", h.Y) + } +} +`), + ).toEqual([ + { method: 'GET', path: '/inner/x', name: 'X' }, + { method: 'GET', path: '/inner/y', name: 'Y' }, + ]); + }); + + it('keeps the outer group when an if initializer binds a different name', () => { + expect( + providers(`package main +func routes(r *gin.Engine, cond bool) { + g := r.Group("/outer") + if x := prepare(); cond { + g.GET("/x", h.X) + } +} +`), + ).toEqual([{ method: 'GET', path: '/outer/x', name: 'X' }]); + }); + + it('stops at an if initializer bound to something other than Group()', () => { + expect( + providers(`package main +func routes(r *gin.Engine, cond bool) { + g := r.Group("/outer") + if g := build(); cond { + g.GET("/x", h.X) + } +} +`), + ).toEqual([{ method: 'GET', path: '/x', name: 'X' }]); + }); + + it('uses a switch initializer group and a group declared inside a case clause', () => { + expect( + providers(`package main +func routes(r *gin.Engine, cond bool) { + g := r.Group("/outer") + switch g := r.Group("/s"); g != nil { + case cond: + g.GET("/x", h.X) + } + switch { + case cond: + g := r.Group("/case") + g.GET("/y", h.Y) + } +} +`), + ).toEqual([ + { method: 'GET', path: '/s/x', name: 'X' }, + { method: 'GET', path: '/case/y', name: 'Y' }, + ]); + }); + + it('stops at a type-switch guard binding and resolves groups declared in a type case', () => { + expect( + providers(`package main +func routes(r *gin.Engine, anyVal any) { + g := r.Group("/outer") + switch g := anyVal.(type) { + case *Router: + g.GET("/x", h.X) + } + switch anyVal.(type) { + case interface{}: + g := r.Group("/t") + g.GET("/y", h.Y) + } +} +`), + ).toEqual([ + { method: 'GET', path: '/x', name: 'X' }, + { method: 'GET', path: '/t/y', name: 'Y' }, + ]); + }); + + it('stops at a select receive binding instead of inheriting an outer group', () => { + // The value received from a channel is statically unknown, so a case that + // rebinds `g` must shadow the outer group with "nothing traceable" (the + // route keeps its literal path) — not leak `/outer` into the id. An + // unrebound case still inherits, and a default clause declaring its own + // group shadows like any other statement list. + expect( + providers(`package main +func routes(r *gin.Engine, ch chan *gin.RouterGroup) { + g := r.Group("/outer") + select { + case g := <-ch: + g.GET("/x", h.X) + } + select { + case <-ch: + g.GET("/y", h.Y) + } + select { + default: + g := r.Group("/d") + g.GET("/z", h.Z) + } +} +`), + ).toEqual([ + { method: 'GET', path: '/x', name: 'X' }, + { method: 'GET', path: '/outer/y', name: 'Y' }, + { method: 'GET', path: '/d/z', name: 'Z' }, + ]); + }); + + it('keeps the outer group through a plain for init and stops at a range shadow binding', () => { + expect( + providers(`package main +func routes(r *gin.Engine, n int, subs []*gin.RouterGroup) { + g := r.Group("/outer") + for i := 0; i < n; i++ { + g.GET("/i", h.I) + } + for _, g := range subs { + g.GET("/r", h.R) + } + for g := r.Group("/loop"); ; { + g.GET("/l", h.L) + } +} +`), + ).toEqual([ + { method: 'GET', path: '/outer/i', name: 'I' }, + { method: 'GET', path: '/r', name: 'R' }, + { method: 'GET', path: '/loop/l', name: 'L' }, + ]); + }); + + it('declines a route whose group a loop reassigns between iterations', () => { + // The post statement (or the body) runs between iterations, so from the + // second pass on the body sees the reassigned group rather than the one + // it entered with: the prefix is control-flow dependent, so the route is + // declined instead of emitting either value. A loop that never writes the + // name keeps its initializer binding. + expect( + providers(`package main +func routes(r *gin.Engine, cond bool, n int) { + g := r.Group("/old") + for ; cond; g = r.Group("/post") { + g.GET("/a", h.A) + } + for g := r.Group("/init"); ; g = r.Group("/post3") { + g.GET("/c", h.C) + } + for k := r.Group("/k"); cond; { + k.GET("/d", h.D) + } +} +`), + ).toEqual([{ method: 'GET', path: '/k/d', name: 'D' }]); + }); + + it('accepts a raw-string (backtick) group prefix', () => { + expect( + providers(`package main +func routes(r *gin.Engine) { + g := r.Group(\`/api\`) + g.GET("/x", h.X) +} +`), + ).toEqual([{ method: 'GET', path: '/api/x', name: 'X' }]); + }); + + it('accepts a raw-string (backtick) route path, as ingestion does', () => { + // Ingestion (Strategy A) decodes both Go string forms for the route path; + // the group layer must emit the same contract for a backtick path, both + // on a bound group and on a chained `Group(...).GET(...)` receiver. + expect( + providers(`package main +func routes(r *gin.Engine) { + g := r.Group("/api") + g.GET(\`/health\`, h.Health) + r.Group("/v1").POST(\`/raw\\x2fy\`, h.Raw) +} +`), + ).toEqual([ + { method: 'GET', path: '/api/health', name: 'Health' }, + { method: 'POST', path: '/v1/raw\\x2fy', name: 'Raw' }, + ]); + }); + + it('decodes Go string escapes in group prefixes and route paths', () => { + // "/api\x2fv1" and "/health\x2fcheck" are `/api/v1` and `/health/check` + // once Go processes the escapes — the id must match the registered URL. + expect( + providers(`package main +func routes(r *gin.Engine) { + g := r.Group("/api\\x2fv1") + g.GET("/health\\x2fcheck", h.H) +} +`), + ).toEqual([{ method: 'GET', path: '/api/v1/health/check', name: 'H' }]); + }); + + it('leaves escapes literal inside a raw-string (backtick) prefix', () => { + // Go raw strings process no escapes: the prefix is `/raw\x2fy`, backslash included. + expect( + providers(`package main +func routes(r *gin.Engine) { + g := r.Group(\`/raw\\x2fy\`) + g.GET("/z", h.Z) +} +`), + ).toEqual([{ method: 'GET', path: '/raw\\x2fy/z', name: 'Z' }]); + }); + + it('collapses duplicate slashes so the path matches ingestion normalization', () => { + // normalizeExtractedRoutePath collapses every "//" run; the downstream + // contract-id normalizer does not — a path keeping "//" would get a + // different id than the graph's route node for the same route. + expect( + providers(`package main +func routes(r *gin.Engine) { + g := r.Group("/api//v1") + g.GET("/a//b", h.A) + r.GET("//root", h.R) + r.GET("/ok", h.Ok) +} +`), + ).toEqual([ + { method: 'GET', path: '/api/v1/a/b', name: 'A' }, + { method: 'GET', path: '/root', name: 'R' }, + { method: 'GET', path: '/ok', name: 'Ok' }, + ]); + }); + + it('forces a leading slash on slashless literals and group prefixes', () => { + // Ingestion's normalizeExtractedRoutePath always adds a leading "/" but + // normalizeHttpPath (the shared contract-id normalizer) does not — a path + // like "x" would become `http::GET::x` here and `http::GET::/x` there, + // splitting the id across the two strategies. Gin likewise panics when a + // route is registered without one. + expect( + providers(`package main +func routes(r *gin.Engine) { + r.GET("x", h.X) + g := r.Group("api") + g.GET("y", h.Y) + r.GET("", h.Root) +} +`), + ).toEqual([ + { method: 'GET', path: '/x', name: 'X' }, + { method: 'GET', path: '/api/y', name: 'Y' }, + { method: 'GET', path: '/', name: 'Root' }, + ]); + }); + + it("binds echo's handler (first argument) when the file imports echo only", () => { + // echo is `GET(path, handler, middleware...)` — the handler is second, + // not last, so a middleware selector must not become the route's name. + expect( + providers(`package main +import "github.com/labstack/echo/v4" + +func routes(e *echo.Echo) { + e.GET("/x", h.Handler, auth.Middleware) + e.POST("/y", handlerID) + e.PUT("/z", func(c echo.Context) error { return nil }) +} +`), + ).toEqual([ + { method: 'GET', path: '/x', name: 'Handler' }, + { method: 'POST', path: '/y', name: 'handlerID' }, + { method: 'PUT', path: '/z', name: null }, + ]); + }); + + it('picks the handler order from a typed receiver parameter in a mixed-import file', () => { + // A parameter's static type proves its framework: `*echo.Echo` / + // `*echo.Group` take echo's order (handler FIRST), gin's types and types + // the file cannot tie to echo keep the last-argument fallback. Method + // receivers and func-literal parameters count the same way. + expect( + providers(`package main +import ( + "github.com/gin-gonic/gin" + "github.com/labstack/echo/v4" +) + +func routes(e *echo.Echo, g *echo.Group, r *gin.RouterGroup, x *Router) { + e.GET("/e", h.Handler, auth.Middleware) + g.GET("/g", h.Handler, auth.Middleware) + r.GET("/r", auth.Middleware, h.Handler) + x.GET("/x", h.Handler, auth.Middleware) + register := func(sub *echo.Group) { + sub.GET("/f", h.Handler, auth.Middleware) + } + _ = register +} + +type Server struct{} + +func (s *Server) routes(api *echo.Group) { + api.GET("/m", h.Handler, auth.Middleware) +} +`), + ).toEqual([ + { method: 'GET', path: '/e', name: 'Handler' }, + { method: 'GET', path: '/g', name: 'Handler' }, + { method: 'GET', path: '/r', name: 'Handler' }, + { method: 'GET', path: '/x', name: 'Middleware' }, + { method: 'GET', path: '/f', name: 'Handler' }, + { method: 'GET', path: '/m', name: 'Handler' }, + ]); + }); + + it('picks the handler order per receiver constructor in a mixed-import file', () => { + // Mixed imports are ambiguous at file scope, but `e := echo.New()` proves + // this call follows echo's order (handler FIRST after the path) and + // `r := gin.Default()` proves gin's (handler LAST). + expect( + providers(`package main +import ( + "github.com/gin-gonic/gin" + "github.com/labstack/echo/v4" +) + +func routes() { + e := echo.New() + r := gin.Default() + e.GET("/x", h.Handler, auth.Middleware) + r.POST("/y", auth.Middleware, h.Post) +} +`), + ).toEqual([ + { method: 'GET', path: '/x', name: 'Handler' }, + { method: 'POST', path: '/y', name: 'Post' }, + ]); + }); + + it('does not treat an unrelated New() as a framework constructor', () => { + // `wrapper.New()` is neither echo's nor gin's constructor: no proof → + // conservative last-argument fallback, not a guessed echo order. + expect( + providers(`package main +import ( + "github.com/gin-gonic/gin" + "github.com/labstack/echo/v4" +) + +func routes() { + e := wrapper.New() + e.GET("/x", h.Handler, auth.Middleware) +} +`), + ).toEqual([{ method: 'GET', path: '/x', name: 'Middleware' }]); + }); + + it('does not mistake a local shadowing the echo import for its constructor', () => { + // A parameter named `echo` shadows the package qualifier: `echo.New()` + // is then a method on that value, not echo's constructor, so the mixed + // file keeps the conservative last-argument fallback. + expect( + providers(`package main +import ( + "github.com/gin-gonic/gin" + "github.com/labstack/echo/v4" +) + +func routes(echo *Factory) { + e := echo.New() + e.GET("/x", h.Handler, auth.Middleware) +} +`), + ).toEqual([{ method: 'GET', path: '/x', name: 'Middleware' }]); + }); + + it('does not mistake a value-less local declaration of echo for the import', () => { + // `var echo Factory` declares a local without an initializer; it still + // shadows the package qualifier, so `echo.New()` proves nothing. + expect( + providers(`package main +import ( + "github.com/gin-gonic/gin" + "github.com/labstack/echo/v4" +) + +func routes() { + var echo Factory + e := echo.New() + e.GET("/x", h.Handler, auth.Middleware) +} +`), + ).toEqual([{ method: 'GET', path: '/x', name: 'Middleware' }]); + }); + + it('declines a route whose Group chain exceeds the depth cap', () => { + // Past MAX_GROUP_DEPTH (32) the full prefix — and, in a mixed file, the + // framework order — is unprovable: emitting the outer prefixes alone (or + // gin's handler order for an echo chain) would be a silent guess. + const chain = (ctor: string, imports: string) => { + const lines = [`g0 := ${ctor}`]; + for (let i = 1; i <= 34; i++) lines.push(`g${i} := g${i - 1}.Group("/p${i}")`); + return `package main +${imports} + +func routes() { + ${lines.join('\n\t')} + g34.GET("/x", h.Handler, auth.Middleware) + g2.GET("/y", h.Handler, auth.Middleware) +} +`; + }; + expect(providers(chain('gin.Default()', 'import "github.com/gin-gonic/gin"'))).toEqual([ + { method: 'GET', path: '/p1/p2/y', name: 'Middleware' }, + ]); + const mixed = `import ( + "github.com/gin-gonic/gin" + "github.com/labstack/echo/v4" +)`; + expect(providers(chain('echo.New()', mixed))).toEqual([ + { method: 'GET', path: '/p1/p2/y', name: 'Handler' }, + ]); + }); + + it('traces a grouped receiver back to its framework constructor in a mixed file', () => { + // The normal grouped shape: `users := api.Group(…)` ← `api := e.Group(…)` + // ← `echo.New()` proves echo order through the Group chain, while the + // gin chain resolves to gin.Default() and keeps the last-argument rule. + expect( + providers(`package main +import ( + "github.com/gin-gonic/gin" + "github.com/labstack/echo/v4" +) + +func routes() { + e := echo.New() + api := e.Group("/api") + users := api.Group("/users") + users.GET("/:id", h.Handler, auth.Middleware) + r := gin.Default() + v1 := r.Group("/v1") + v1.POST("/x", auth.Middleware, h.Post) +} +`), + ).toEqual([ + { method: 'GET', path: '/api/users/:id', name: 'Handler' }, + { method: 'POST', path: '/v1/x', name: 'Post' }, + ]); + }); + + it('marks handler resolution by how the handler is designated', () => { + // `h.List` / `o.List` emit the field name `List`, but the operand does not + // prove where `List` is declared: they are qualifiedHandler (repo-wide + // unique match only, never a same-named local method). A bare name + // declared more than once in the file (`Show` as function and method) + // resolves only when unique in the file; a bare unique name keeps the + // default resolution. + parser.setLanguage(Go); + const flags = GO_HTTP_PLUGIN.scan( + parser.parse(`package main +import "github.com/gin-gonic/gin" + +type A struct{} + +func (a *A) List(c *gin.Context) {} +func (a *A) Show(c *gin.Context) {} +func Show(c *gin.Context) {} +func Ping(c *gin.Context) {} + +func routes(r *gin.Engine, h *A, o *B) { + r.GET("/a", h.List) + r.GET("/b", o.List) + r.GET("/s", Show) + r.GET("/p", Ping) +} +`), + ) + .filter((d) => d.role === 'provider') + .map((d) => [ + d.path, + d.name, + d.qualifiedHandler ?? false, + d.strictHandlerResolution ?? false, + ]); + expect(flags).toEqual([ + ['/a', 'List', true, false], + ['/b', 'List', true, false], + ['/s', 'Show', false, true], + ['/p', 'Ping', false, false], + ]); + }); + + it('resolves grouped var specs that precede the use', () => { + // Earlier specs in a grouped `var (…)` are in scope for later ones; the + // ingestion side emits /api/admin/x for this, so both strategies agree. + expect( + providers(`package main +func routes(r *gin.Engine) { + var ( + api = r.Group("/api") + admin = api.Group("/admin") + ) + admin.GET("/x", handler) +} +`), + ).toEqual([{ method: 'GET', path: '/api/admin/x', name: 'handler' }]); + }); + + it('declines a route whose group is reassigned in an earlier nested scope', () => { + // `{ g = r.Group("/new") }` (or a branch) writes the outer g: which value + // reaches the use depends on control flow, and ingestion declines it too, + // so emitting the older `/old/x` would invent a route. A nested `:=` + // declares a new variable and leaves the outer binding intact. + expect( + providers(`package main +func routes(r *gin.Engine, cond bool) { + g := r.Group("/old") + { g = r.Group("/new") } + g.GET("/x", handler) + k := r.Group("/k") + if cond { k = r.Group("/other") } + k.GET("/y", handler) + m := r.Group("/m") + { m := r.Group("/inner"); m.GET("/i", handler) } + m.GET("/z", handler) +} +`), + ).toEqual([ + { method: 'GET', path: '/inner/i', name: 'handler' }, + { method: 'GET', path: '/m/z', name: 'handler' }, + ]); + }); + + it('resolves a name used in its own statement initializer to the outer binding', () => { + // `if g := g.Group("/inner"); …` — the right-hand g is the OUTER group; + // the new g only scopes over what follows. Same for a for initializer. + expect( + providers(`package main +func routes(r *gin.Engine, enabled bool) { + g := r.Group("/api") + if g := g.Group("/inner"); enabled { + g.GET("/x", handler) + } + for g := g.Group("/loop"); enabled; { + g.GET("/y", handler) + } +} +`), + ).toEqual([ + { method: 'GET', path: '/api/inner/x', name: 'handler' }, + { method: 'GET', path: '/api/loop/y', name: 'handler' }, + ]); + }); + + it('skips comments when locating the path and the handler', () => { + // tree-sitter names comments, so they must not count as arguments: a + // leading comment must not hide the path, and a comment must not be + // picked as the echo (first) or gin (last) handler. + expect( + providers(`package main +import "github.com/labstack/echo/v4" +func routes(e *echo.Echo) { + e.GET("/users", /* description */ users) + e.POST(/* description */ "/posts", posts /* trailing */) +} +`), + ).toEqual([ + { method: 'GET', path: '/users', name: 'users' }, + { method: 'POST', path: '/posts', name: 'posts' }, + ]); + expect( + providers(`package main +import "github.com/gin-gonic/gin" +func routes(r *gin.Engine) { + r.GET(/* description */ "/users", users /* trailing */) + g := r.Group(/* c */ "/api") + g.GET("/x", h.X) +} +`), + ).toEqual([ + { method: 'GET', path: '/users', name: 'users' }, + { method: 'GET', path: '/api/x', name: 'X' }, + ]); + }); + + it('applies the same group logic to echo', () => { + expect( + providers(`package main +func main() { + e := echo.New() + api := e.Group("/api") + users := api.Group("/users") + users.GET("/:id", userHandler.Get) +} +`), + ).toEqual([{ method: 'GET', path: '/api/users/:id', name: 'Get' }]); + }); +}); + +describe('Go gin provider ↔ fetch() consumer pairing', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-go-gin-groups-')); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + const repoHandle = (repoPath: string, id: string): RepoHandle => ({ + id, + path: id, + repoPath, + storagePath: path.join(repoPath, '.gitnexus'), + }); + + it('does not bind a qualified handler to an unrelated same-named local method', async () => { + // routes.go declares A.List; the route's handler is b.List, with B.List in + // b.go. The file-first name lookup would pick A.List, so a qualified + // handler skips it: a repo-wide unique List resolves, an ambiguous one + // keeps the file-level fallback (empty symbolUid) instead of A.List. + const repo = path.join(tmpDir, 'repo'); + fs.mkdirSync(repo, { recursive: true }); + fs.writeFileSync( + path.join(repo, 'routes.go'), + `package main + +import "github.com/gin-gonic/gin" + +type A struct{} + +func (a *A) List(c *gin.Context) {} + +func routes(r *gin.Engine, b *B) { + r.GET("/bs", b.List) +} +`, + ); + const aList = { + uid: 'Method:routes.go:A.List', + name: 'List', + filePath: 'routes.go', + startLine: 6, + endLine: 6, + labels: ['Method'], + }; + const bList = { uid: 'Method:b.go:B.List', name: 'List', filePath: 'b.go' }; + const run = async (repoWide: Record[]) => { + const db = async (query: string, params?: Record) => { + if (query === RESOLVE_BY_NAME_QUERY) return params?.name === 'List' ? repoWide : []; + if (query.includes('UNION ALL') && params?.filePath === 'routes.go') return [aList]; + return []; + }; + const out = await new HttpRouteExtractor().extract(db, repo, repoHandle(repo, 'repo')); + return out.find((c) => c.contractId === 'http::GET::/bs')?.symbolUid; + }; + expect(await run([aList, bList])).toBe(''); + expect(await run([bList])).toBe(bList.uid); + }); + + it('cross-links a grouped method-value route to a ${API_BASE}-prefixed fetch', async () => { + const backend = path.join(tmpDir, 'backend'); + const web = path.join(tmpDir, 'web'); + fs.mkdirSync(path.join(backend, 'internal/handlers'), { recursive: true }); + fs.mkdirSync(path.join(web, 'src/pages'), { recursive: true }); + fs.writeFileSync(path.join(backend, 'internal/handlers/routes.go'), GIN_ROUTES); + fs.writeFileSync( + path.join(web, 'src/pages/AdminMatchesPage.tsx'), + `const API_BASE = import.meta.env.VITE_API_BASE; + +export async function handleUnfinalize(seasonId: string, roundId: string) { + await fetch(\`\${API_BASE}/api/v1/admin/seasons/\${seasonId}/rounds/\${roundId}/unfinalize\`, { + method: 'POST', + }); +} +`, + ); + + const extractor = new HttpRouteExtractor(); + const contracts: StoredContract[] = [ + ...(await extractor.extract(null, backend, repoHandle(backend, 'backend'))).map((c) => ({ + ...c, + repo: 'backend', + })), + ...(await extractor.extract(null, web, repoHandle(web, 'web'))).map((c) => ({ + ...c, + repo: 'web', + })), + ]; + + const { matched } = runExactMatch(contracts); + const link = matched.find( + (l) => l.contractId === 'http::POST::/api/v1/admin/seasons/{param}/rounds/{param}/unfinalize', + ); + expect(link).toMatchObject({ + from: { repo: 'web', symbolRef: { filePath: 'src/pages/AdminMatchesPage.tsx' } }, + to: { + repo: 'backend', + symbolRef: { filePath: 'internal/handlers/routes.go', name: 'UnfinalizeRoundHandle' }, + }, + matchType: 'exact', + }); + }); +}); diff --git a/gitnexus/test/unit/group/service.test.ts b/gitnexus/test/unit/group/service.test.ts index f56522ef9..770e6889f 100644 --- a/gitnexus/test/unit/group/service.test.ts +++ b/gitnexus/test/unit/group/service.test.ts @@ -2,13 +2,17 @@ import { describe, it, expect, vi } from 'vitest'; import * as fs from 'node:fs'; import * as path from 'node:path'; import * as os from 'node:os'; +import { execFileSync } from 'node:child_process'; import { GroupService, type GroupToolPort, type GroupRepoHandle, } from '../../../src/core/group/service.js'; import { writeContractRegistry } from '../../../src/core/group/storage.js'; -import { formatIndexStatusCell } from '../../../src/cli/group-status-format.js'; +import { + formatIndexStatusCell, + type GroupRepoIndexRow, +} from '../../../src/cli/group-status-format.js'; import type { ContractRegistry, StoredContract, CrossLink } from '../../../src/core/group/types.js'; function makeTmpGroup(): { tmpDir: string; groupDir: string; cleanup: () => void } { @@ -717,10 +721,7 @@ repos: const svc = new GroupService(port); const result = (await svc.groupStatus({ name: 'test-group' })) as { - repos: Record< - string, - { indexStale: boolean; commitsBehind?: number; status?: string; missing: boolean } - >; + repos: Record; }; const row = result.repos['app/backend']; @@ -736,5 +737,67 @@ repos: cleanup(); } }); + + it('renders a real rollback as an index that differs from HEAD', async () => { + const { cleanup, tmpDir } = makeTmpGroup(); + try { + vi.stubEnv('GITNEXUS_HOME', tmpDir); + const repoPath = path.join(tmpDir, 'rollback'); + fs.mkdirSync(repoPath); + const git = (...args: string[]): string => + execFileSync( + 'git', + [ + '-c', + 'user.email=t@example.com', + '-c', + 'user.name=T', + '-c', + 'commit.gpgsign=false', + ...args, + ], + { cwd: repoPath, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }, + ).trim(); + git('init', '-q', '--initial-branch=main'); + git('commit', '--allow-empty', '-qm', 'first'); + const firstCommit = git('rev-parse', 'HEAD'); + git('commit', '--allow-empty', '-qm', 'indexed'); + const indexedCommit = git('rev-parse', 'HEAD'); + git('checkout', '-q', '--detach', firstCommit); + + const storagePath = path.join(repoPath, '.gitnexus'); + fs.mkdirSync(storagePath); + fs.writeFileSync( + path.join(storagePath, 'gitnexus.json'), + JSON.stringify({ lastCommit: indexedCommit, indexedAt: '2026-01-01T00:00:00.000Z' }), + ); + const port = makePort({ + resolveRepo: vi.fn( + async (name?: string): Promise => ({ + id: name || 'test', + name: name || 'test', + repoPath, + storagePath, + }), + ), + }); + + const svc = new GroupService(port); + const result = (await svc.groupStatus({ name: 'test-group' })) as { + repos: Record; + }; + const row = result.repos['app/backend']; + expect(row).toMatchObject({ + missing: false, + indexStale: true, + commitsBehind: 0, + status: 'diverged', + }); + expect(formatIndexStatusCell(row)).toBe('STALE (index differs from HEAD)'); + } finally { + vi.unstubAllEnvs(); + cleanup(); + } + }); }); }); diff --git a/gitnexus/test/unit/ignore-service.test.ts b/gitnexus/test/unit/ignore-service.test.ts index 60d6e8f77..307fd5a15 100644 --- a/gitnexus/test/unit/ignore-service.test.ts +++ b/gitnexus/test/unit/ignore-service.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, beforeAll, beforeEach, afterAll, afterEach, vi } import fs from 'fs/promises'; import path from 'path'; import os from 'os'; +import { execFileSync } from 'child_process'; import { shouldIgnorePath, isHardcodedIgnoredDirectory, @@ -687,6 +688,319 @@ describe('createIgnoreFilter', () => { }); }); +describe('createIgnoreFilter with nested .gitignore files (#2675)', () => { + let tmpDir: string; + let originalNoGitignore: string | undefined; + + const asPath = (rel: string) => ({ name: path.basename(rel), relative: () => rel }) as any; + + beforeEach(async () => { + originalNoGitignore = process.env.GITNEXUS_NO_GITIGNORE; + // These tests expect nested rules to apply, so a value inherited from the + // invoking shell must not switch them off. afterEach restores it. + delete process.env.GITNEXUS_NO_GITIGNORE; + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-nested-ignore-test-')); + }); + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + if (originalNoGitignore === undefined) { + delete process.env.GITNEXUS_NO_GITIGNORE; + } else { + process.env.GITNEXUS_NO_GITIGNORE = originalNoGitignore; + } + }); + + it('applies a nested .gitignore relative to its own directory', async () => { + await fs.mkdir(path.join(tmpDir, 'app', 'public', 'generated'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, 'app', '.gitignore'), 'public/generated/\n*.log\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.childrenIgnored(asPath('app/public/generated'))).toBe(true); + expect(filter.ignored(asPath('app/public/generated/bundle.js'))).toBe(true); + expect(filter.ignored(asPath('app/debug.log'))).toBe(true); + expect(filter.ignored(asPath('app/src/deep/debug.log'))).toBe(true); + + // Rules stay scoped to the directory that declares them. + expect(filter.childrenIgnored(asPath('public/generated'))).toBe(false); + expect(filter.ignored(asPath('debug.log'))).toBe(false); + expect(filter.ignored(asPath('other/debug.log'))).toBe(false); + expect(filter.ignored(asPath('app/src/index.ts'))).toBe(false); + }); + + it('preserves independent root exclusions beneath a GitNexus directory negation', async () => { + await fs.mkdir(path.join(tmpDir, 'pkg', 'generated'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, '.gitignore'), '*.log\n**/pkg/generated/\n'); + await fs.writeFile(path.join(tmpDir, '.gitnexusignore'), '!pkg/\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.childrenIgnored(asPath('pkg'))).toBe(false); + expect(filter.ignored(asPath('pkg/debug.log'))).toBe(true); + expect(filter.childrenIgnored(asPath('pkg/generated'))).toBe(true); + expect(filter.ignored(asPath('pkg/generated/index.ts'))).toBe(true); + expect(filter.ignored(asPath('pkg/index.ts'))).toBe(false); + }); + + it('preserves root inclusions after an unrelated nested directory negation', async () => { + await fs.mkdir(path.join(tmpDir, 'pkg', 'reports', '__tests__'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, '.gitignore'), '!__tests__/\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', '.gitignore'), '!reports/\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', 'reports', '__tests__', 'test.ts'), 'export {};\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.childrenIgnored(asPath('pkg/reports/__tests__'))).toBe(false); + expect(filter.ignored(asPath('pkg/reports/__tests__/test.ts'))).toBe(false); + const { walkRepositoryPaths } = await import('../../src/core/ingestion/filesystem-walker.js'); + expect((await walkRepositoryPaths(tmpDir)).map((f) => f.path)).toContain( + 'pkg/reports/__tests__/test.ts', + ); + }); + + it('lets a deeper nested negation re-include what an outer nested file ignored', async () => { + await fs.mkdir(path.join(tmpDir, 'app', 'lib'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, 'app', '.gitignore'), '*.gen.ts\n'); + await fs.writeFile(path.join(tmpDir, 'app', 'lib', '.gitignore'), '!keep.gen.ts\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.ignored(asPath('app/lib/keep.gen.ts'))).toBe(false); + expect(filter.ignored(asPath('app/lib/other.gen.ts'))).toBe(true); + }); + + it('lets a nested negation re-include what the root .gitignore ignored', async () => { + await fs.mkdir(path.join(tmpDir, 'pkg', 'reports'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, '.gitignore'), '*.log\nreports/\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', '.gitignore'), '!keep.log\n!reports/\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.ignored(asPath('pkg/keep.log'))).toBe(false); + expect(filter.ignored(asPath('pkg/other.log'))).toBe(true); + expect(filter.childrenIgnored(asPath('pkg/reports'))).toBe(false); + expect(filter.childrenIgnored(asPath('reports'))).toBe(true); + }); + + it('re-includes the files inside a directory a nested negation un-ignores', async () => { + await fs.mkdir(path.join(tmpDir, 'pkg', 'reports', 'daily'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, '.gitignore'), 'reports/\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', '.gitignore'), '!reports/\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', 'reports', 'summary.ts'), 'export {};\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', 'reports', 'daily', 'run.ts'), 'export {};\n'); + await fs.mkdir(path.join(tmpDir, 'reports'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, 'reports', 'top.ts'), 'export {};\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.ignored(asPath('pkg/reports/summary.ts'))).toBe(false); + expect(filter.childrenIgnored(asPath('pkg/reports/daily'))).toBe(false); + expect(filter.ignored(asPath('pkg/reports/daily/run.ts'))).toBe(false); + expect(filter.ignored(asPath('reports/top.ts'))).toBe(true); + + const { walkRepositoryPaths } = await import('../../src/core/ingestion/filesystem-walker.js'); + const scanned = (await walkRepositoryPaths(tmpDir)).map((f) => f.path); + + expect(scanned).toContain('pkg/reports/summary.ts'); + expect(scanned).toContain('pkg/reports/daily/run.ts'); + expect(scanned).not.toContain('reports/top.ts'); + }); + + it('keeps independent root exclusions inside a directory re-included by nested rules', async () => { + await fs.mkdir(path.join(tmpDir, 'pkg', 'reports', 'private'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, '.gitignore'), '*.ts\nreports/\n**/reports/private/\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', '.gitignore'), '!reports/\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', 'reports', 'file.ts'), 'export {};\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', 'reports', 'keep.js'), 'export {};\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', 'reports', 'private', 'secret.js'), 'export {};\n'); + + const { walkRepositoryPaths } = await import('../../src/core/ingestion/filesystem-walker.js'); + const scanned = (await walkRepositoryPaths(tmpDir)).map((f) => f.path); + + expect(scanned).toContain('pkg/reports/keep.js'); + expect(scanned).not.toContain('pkg/reports/file.ts'); + expect(scanned).not.toContain('pkg/reports/private/secret.js'); + }); + + it('re-includes descendants when a deeper directory negation overrides an outer nested file', async () => { + await fs.mkdir(path.join(tmpDir, 'app', 'pkg', 'reports', 'daily'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, 'app', '.gitignore'), 'reports/\n*.gen.ts\n'); + await fs.writeFile(path.join(tmpDir, 'app', 'pkg', '.gitignore'), '!reports/\n'); + await fs.writeFile(path.join(tmpDir, 'app', 'pkg', 'reports', 'keep.ts'), 'export {};\n'); + await fs.writeFile(path.join(tmpDir, 'app', 'pkg', 'reports', 'drop.gen.ts'), 'export {};\n'); + await fs.writeFile( + path.join(tmpDir, 'app', 'pkg', 'reports', 'daily', 'run.ts'), + 'export {};\n', + ); + + const { walkRepositoryPaths } = await import('../../src/core/ingestion/filesystem-walker.js'); + const scanned = (await walkRepositoryPaths(tmpDir)).map((f) => f.path); + + expect(scanned).toContain('app/pkg/reports/keep.ts'); + expect(scanned).toContain('app/pkg/reports/daily/run.ts'); + expect(scanned).not.toContain('app/pkg/reports/drop.gen.ts'); + }); + + it.each(['reports [daily]', ...(process.platform === 'win32' ? [] : ['reports\ndaily'])])( + 'keeps re-included directory names literal: %j', + async (directory) => { + await fs.mkdir(path.join(tmpDir, 'pkg', directory), { recursive: true }); + await fs.mkdir(path.join(tmpDir, 'other', directory), { recursive: true }); + await fs.writeFile(path.join(tmpDir, '.gitignore'), 'reports*/\n*.ts\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', '.gitignore'), '!reports*/\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', directory, 'keep.js'), 'export {};\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', directory, 'drop.ts'), 'export {};\n'); + await fs.writeFile(path.join(tmpDir, 'other', directory, 'drop.js'), 'export {};\n'); + + const { walkRepositoryPaths } = await import('../../src/core/ingestion/filesystem-walker.js'); + const scanned = (await walkRepositoryPaths(tmpDir)).map((f) => f.path); + + expect(scanned).toContain(`pkg/${directory}/keep.js`); + expect(scanned).not.toContain(`pkg/${directory}/drop.ts`); + expect(scanned).not.toContain(`other/${directory}/drop.js`); + }, + ); + + it('keeps .gitnexusignore above a nested negation', async () => { + await fs.mkdir(path.join(tmpDir, 'pkg'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, 'pkg', '.gitignore'), '!keep.log\n'); + await fs.writeFile(path.join(tmpDir, '.gitnexusignore'), 'pkg/keep.log\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.ignored(asPath('pkg/keep.log'))).toBe(true); + }); + + it('keeps an explicit root .gitnexusignore negation in charge', async () => { + await fs.mkdir(path.join(tmpDir, 'app'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, 'app', '.gitignore'), 'generated/\n'); + await fs.writeFile(path.join(tmpDir, '.gitnexusignore'), '!app/generated/\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.childrenIgnored(asPath('app/generated'))).toBe(false); + expect(filter.ignored(asPath('app/generated/schema.ts'))).toBe(false); + }); + + it('lets a nested ignore beat a root .gitignore file negation', async () => { + await fs.mkdir(path.join(tmpDir, 'pkg'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, '.gitignore'), '*.log\n!pkg/keep.log\n'); + await fs.writeFile(path.join(tmpDir, 'pkg', '.gitignore'), 'keep.log\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.ignored(asPath('pkg/keep.log'))).toBe(true); + }); + + it('lets a nested ignore beat a root .gitignore directory negation', async () => { + await fs.mkdir(path.join(tmpDir, 'app', 'generated'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, '.gitignore'), '!app/generated/\n'); + await fs.writeFile(path.join(tmpDir, 'app', '.gitignore'), 'generated/\n'); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.childrenIgnored(asPath('app/generated'))).toBe(true); + expect(filter.ignored(asPath('app/generated/schema.ts'))).toBe(true); + }); + + it('does not let a nested negation rescue hardcoded defaults', async () => { + await fs.mkdir(path.join(tmpDir, 'pkg', 'node_modules'), { recursive: true }); + await fs.writeFile( + path.join(tmpDir, 'pkg', '.gitignore'), + '!node_modules/\n!package-lock.json\n', + ); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.childrenIgnored(asPath('pkg/node_modules'))).toBe(true); + expect(filter.ignored(asPath('pkg/package-lock.json'))).toBe(true); + }); + + it.skipIf(process.platform === 'win32')('ignores a symlinked nested .gitignore', async () => { + await fs.mkdir(path.join(tmpDir, 'app'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, 'rules.txt'), '*.log\n'); + await fs.symlink(path.join(tmpDir, 'rules.txt'), path.join(tmpDir, 'app', '.gitignore')); + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.ignored(asPath('app/debug.log'))).toBe(false); + }); + + it('skips nested .gitignore files when GITNEXUS_NO_GITIGNORE is set', async () => { + await fs.mkdir(path.join(tmpDir, 'app'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, 'app', '.gitignore'), 'generated/\n'); + process.env.GITNEXUS_NO_GITIGNORE = '1'; + const filter = await createIgnoreFilter(tmpDir); + + expect(filter.childrenIgnored(asPath('app/generated'))).toBe(false); + }); + + it('prunes nested-ignored files from a real repository walk', async () => { + await fs.mkdir(path.join(tmpDir, 'app', 'src'), { recursive: true }); + await fs.mkdir(path.join(tmpDir, 'app', 'public', 'generated'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, 'app', '.gitignore'), 'public/generated/\n'); + await fs.writeFile(path.join(tmpDir, 'app', 'src', 'index.ts'), 'export {};\n'); + await fs.writeFile(path.join(tmpDir, 'app', 'public', 'generated', 'bundle.js'), 'x;\n'); + + const { walkRepositoryPaths } = await import('../../src/core/ingestion/filesystem-walker.js'); + const scanned = (await walkRepositoryPaths(tmpDir)).map((f) => f.path); + + expect(scanned).toContain('app/src/index.ts'); + expect(scanned).not.toContain('app/public/generated/bundle.js'); + }); + + it('follows git when a nested file negates a path inside an ignored directory', async () => { + // git cannot re-include a file whose parent directory is excluded, so + // `gen/` + `!gen/keep.ts` leaves keep.ts out, while `gen/*` excludes only + // the contents and lets the negation bring keep.ts back. Root rules behave + // the same way. (`gen`, not `build`: `build` is a hardcoded default.) + const { walkRepositoryPaths } = await import('../../src/core/ingestion/filesystem-walker.js'); + for (const [pkg, rules] of [ + ['dir', 'gen/\n!gen/keep.ts\n'], + ['star', 'gen/*\n!gen/keep.ts\n'], + ]) { + await fs.mkdir(path.join(tmpDir, pkg, 'gen'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, pkg, '.gitignore'), rules); + await fs.writeFile(path.join(tmpDir, pkg, 'gen', 'keep.ts'), 'export {};\n'); + await fs.writeFile(path.join(tmpDir, pkg, 'gen', 'drop.ts'), 'export {};\n'); + } + + const scanned = (await walkRepositoryPaths(tmpDir)).map((f) => f.path); + + expect(scanned).not.toContain('dir/gen/keep.ts'); + expect(scanned).not.toContain('dir/gen/drop.ts'); + expect(scanned).toContain('star/gen/keep.ts'); + expect(scanned).not.toContain('star/gen/drop.ts'); + }); + + it.skipIf(process.platform === 'win32')( + 'does not hang on a nested .gitignore that is a FIFO', + async () => { + // Opening a FIFO for reading blocks until a writer appears. Without + // O_NONBLOCK the walk would stop in open(2), before the regular-file + // check, and never return. + await fs.mkdir(path.join(tmpDir, 'pkg', 'src'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, 'pkg', 'src', 'index.ts'), 'export {};\n'); + execFileSync('mkfifo', [path.join(tmpDir, 'pkg', '.gitignore')]); + + // A timer in this worker cannot interrupt a blocked synchronous open. + // Enforce the deadline outside the process that performs the walk. + const walkerUrl = new URL('../../src/core/ingestion/filesystem-walker.ts', import.meta.url) + .href; + const output = execFileSync( + process.execPath, + [ + '--import', + import.meta.resolve('tsx'), + '--input-type=module', + '--eval', + `import { walkRepositoryPaths } from ${JSON.stringify(walkerUrl)}; + const files = await walkRepositoryPaths(${JSON.stringify(tmpDir)}); + console.log(JSON.stringify(files.map((file) => file.path)));`, + ], + { + encoding: 'utf8', + timeout: 5_000, + killSignal: 'SIGKILL', + env: { ...process.env, GITNEXUS_NO_GLOBAL_IGNORE: '1' }, + }, + ); + + expect(JSON.parse(output)).toContain('pkg/src/index.ts'); + }, + 10_000, + ); +}); + describe('loadIgnoreRules — error handling', () => { let tmpDir: string; diff --git a/gitnexus/test/unit/impact-batching-grouping.test.ts b/gitnexus/test/unit/impact-batching-grouping.test.ts index e068870f9..7f4d18bd8 100644 --- a/gitnexus/test/unit/impact-batching-grouping.test.ts +++ b/gitnexus/test/unit/impact-batching-grouping.test.ts @@ -79,6 +79,8 @@ describe('impact: batching and grouping', () => { // Handle parameterized calls (including chunked STEP_IN_PROCESS queries) executeParameterizedMock.mockImplementation(async (...args: any[]) => { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('RETURN h.id AS hid')) return []; + if (query.includes('RETURN s.id AS sid')) return []; const params = args[2] || {}; // Match only the aggregation chunk (which uses COUNT(DISTINCT s.id)), // not the per-symbol enrichment pass added by impact byDepth processes @@ -91,6 +93,7 @@ describe('impact: batching and grouping', () => { const idx = chunkCallIndex++; return [ { + pId: 'proc-' + idx, entryPointId: `ep-${Math.floor(idx)}`, epName: `epName-${idx}`, epType: 'Function', @@ -148,6 +151,8 @@ describe('impact: batching and grouping', () => { executeParameterizedMock.mockImplementation(async (...args: any[]) => { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('RETURN h.id AS hid')) return []; + if (query.includes('RETURN s.id AS sid')) return []; // BFS frontier query (parameterized #1907 U3): return 6 impacted nodes. if (query.includes('r.type IN') && !query.includes('STEP_IN_PROCESS')) { const res: any[] = []; @@ -166,6 +171,7 @@ describe('impact: batching and grouping', () => { // For STEP_IN_PROCESS in this test, return grouping rows return [ { + pId: 'proc-1a', entryPointId: 'ep-1', epName: 'EP1', epType: 'Function', @@ -174,6 +180,7 @@ describe('impact: batching and grouping', () => { minStep: 1, }, { + pId: 'proc-2', entryPointId: 'ep-2', epName: 'EP2', epType: 'Function', @@ -182,6 +189,7 @@ describe('impact: batching and grouping', () => { minStep: 2, }, { + pId: 'proc-1b', entryPointId: 'ep-1', epName: 'EP1', epType: 'Function', @@ -190,6 +198,7 @@ describe('impact: batching and grouping', () => { minStep: 3, }, { + pId: 'proc-3', entryPointId: 'ep-3', epName: 'EP3', epType: 'Function', @@ -245,6 +254,8 @@ describe('impact: batching and grouping', () => { executeParameterizedMock.mockImplementation(async (...args: any[]) => { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('RETURN h.id AS hid')) return []; + if (query.includes('RETURN s.id AS sid')) return []; const params = args[2] || {}; // Match only the aggregation chunk (which uses COUNT(DISTINCT s.id)), // not the per-symbol enrichment pass added by impact byDepth processes @@ -254,6 +265,7 @@ describe('impact: batching and grouping', () => { chunkSizes.push(ids.length); return [ { + pId: 'proc-x-' + chunkSizes.length, entryPointId: 'ep-x', epName: 'EPX', epType: 'Function', @@ -351,6 +363,7 @@ describe('impact: batching and grouping', () => { executeQueryMock.mockImplementation(async () => []); executeParameterizedMock.mockImplementation(async (...args: any[]) => { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('RETURN h.id AS hid')) return []; if (query.includes('r.type IN') && !query.includes('STEP_IN_PROCESS')) { return [ { @@ -394,6 +407,7 @@ describe('impact: batching and grouping', () => { executeQueryMock.mockImplementation(async () => []); executeParameterizedMock.mockImplementation(async (...args: any[]) => { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('RETURN h.id AS hid')) return []; if (query.includes('STEP_IN_PROCESS')) { throw new Error('process chunk failed'); } @@ -443,6 +457,8 @@ describe('impact: batching and grouping', () => { executeQueryMock.mockImplementation(async () => []); executeParameterizedMock.mockImplementation(async (...args: any[]) => { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('RETURN h.id AS hid')) return []; + if (query.includes('RETURN s.id AS sid')) return []; if (query.includes('MEMBER_OF')) throw new Error('module chunk failed'); if (query.includes('STEP_IN_PROCESS') && query.includes('COUNT(DISTINCT s.id)')) { return [ @@ -500,6 +516,8 @@ describe('impact: batching and grouping', () => { executeQueryMock.mockImplementation(async () => []); executeParameterizedMock.mockImplementation(async (...args: any[]) => { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('RETURN h.id AS hid')) return []; + if (query.includes('RETURN s.id AS sid')) return []; if (query.includes('MIN(r.step) AS minStep') && !query.includes('COUNT(DISTINCT s.id)')) { throw new Error('minStep backfill failed'); } @@ -559,6 +577,8 @@ describe('impact: batching and grouping', () => { let processChunk = 0; executeParameterizedMock.mockImplementation(async (...args: any[]) => { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('RETURN h.id AS hid')) return []; + if (query.includes('RETURN s.id AS sid')) return []; if (query.includes('STEP_IN_PROCESS') && query.includes('COUNT(DISTINCT s.id)')) { processChunk += 1; if (processChunk === 2) throw new Error('later process chunk failed'); @@ -615,6 +635,7 @@ describe('impact: batching and grouping', () => { executeQueryMock.mockImplementation(async () => []); executeParameterizedMock.mockImplementation(async (...args: any[]) => { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); + if (query.includes('RETURN h.id AS hid')) return []; if (query.includes('MEMBER_OF') && query.includes('RETURN DISTINCT c.heuristicLabel')) { throw new Error('module classification failed'); } diff --git a/gitnexus/test/unit/impact-context-integrity.test.ts b/gitnexus/test/unit/impact-context-integrity.test.ts new file mode 100644 index 000000000..5ad29172b --- /dev/null +++ b/gitnexus/test/unit/impact-context-integrity.test.ts @@ -0,0 +1,847 @@ +/** Corrupt detail rows must not become usable context/impact answers (#3354). */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { db, aop } = vi.hoisted(() => ({ + db: { + initLbug: vi.fn().mockResolvedValue(undefined), + executeQuery: vi.fn().mockResolvedValue([]), + executeParameterized: vi.fn().mockResolvedValue([]), + closeLbug: vi.fn().mockResolvedValue(undefined), + isLbugReady: vi.fn().mockReturnValue(true), + }, + aop: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock('../../src/core/lbug/pool-adapter.js', async (importOriginal) => ({ + ...(await importOriginal()), + ...db, +})); +vi.mock('../../src/mcp/core/lbug-adapter.js', async (importOriginal) => ({ + ...(await importOriginal()), + ...db, +})); +vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => ({ + ...(await importOriginal()), + listRegisteredRepos: vi.fn().mockResolvedValue([ + { + name: 'integrity-fixture', + path: '/tmp/integrity-fixture', + storagePath: '/tmp/integrity-fixture/.gitnexus', + indexedAt: '2026-10-03T12:00:00Z', + lastCommit: 'fixture', + stats: { files: 2, nodes: 2, edges: 1, communities: 0, processes: 1 }, + }, + ]), + cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }), + findSiblingClones: vi.fn().mockResolvedValue([]), + loadMeta: vi.fn().mockResolvedValue({ + pdg: { maxCdgEdgesPerFunction: 0, maxReachingDefEdgesPerFunction: 0 }, + }), +})); +vi.mock('../../src/core/git-staleness.js', () => ({ + checkStalenessAsync: vi.fn().mockResolvedValue({ isStale: false, commitsBehind: 0 }), + checkStaleness: vi.fn().mockReturnValue({ isStale: false, commitsBehind: 0 }), + checkCwdMatch: vi.fn().mockResolvedValue({ match: 'none' }), +})); +vi.mock('../../src/storage/git.js', async (importOriginal) => ({ + ...(await importOriginal()), + getGitRoot: vi.fn().mockReturnValue(null), +})); +vi.mock('../../src/mcp/local/aop-metadata.js', () => ({ querySpringAopMetadata: aop })); + +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { SymbolIdentityError } from '../../src/mcp/local/query-result-integrity.js'; + +const TARGET = { + id: 'func:target', + name: 'target', + type: 'Function', + filePath: 'src/target.ts', + startLine: 1, + endLine: 4, +}; +const REF = { + relType: 'CALLS', + uid: 'func:caller', + name: 'caller', + filePath: 'src/caller.ts', + kind: 'Function', +}; +const EDGE = { + sourceId: TARGET.id, + id: REF.uid, + name: REF.name, + type: REF.kind, + filePath: REF.filePath, + relType: 'CALLS', + confidence: 1, +}; +const PROCESS = { + pId: 'proc:caller', + name: 'Caller flow', + processType: 'intra_community', + entryPointId: REF.uid, + hits: 1, + minStep: 0, + stepCount: 2, + epName: REF.name, + epType: REF.kind, + epFilePath: REF.filePath, +}; +const BAD = 'corrupt\0persisted-value'; + +type Seam = + | 'target' + | 'targetLabels' + | 'aopRows' + | 'pdgSeed' + | 'pdgNeighbor' + | 'pdgOwner' + | 'pdgStatement' + | 'pdgSelf' + | 'pdgCalleeBlocks' + | 'pdgSummary' + | 'pdgSpans' + | 'incoming' + | 'classIncoming' + | 'outgoing' + | 'typedProperties' + | 'contextProcess' + | 'contextRoute' + | 'interfaceBoundary' + | 'interfaceCount' + | 'chain' + | 'seeds' + | 'members' + | 'frontier' + | 'process' + | 'backfill' + | 'membership' + | 'modules' + | 'impactRoute' + | 'metadata'; +let backend: LocalBackend; +let rows: Partial>; +let failedSeam: Seam | undefined; + +function fixture(seam: Seam): unknown[] { + if (failedSeam === seam) throw new Error('ordinary unavailable query'); + return rows[seam] ?? []; +} + +function querySeam(query: string, params: Record | undefined): Seam | undefined { + if (params?.symName || params?.uid) return 'target'; + if (query.includes("RETURN n.id AS id, 'Class' AS label")) return 'targetLabels'; + if (query.includes("r.reason STARTS WITH 'spring-aop:v1:'")) return 'aopRows'; + if (query.includes('RETURN s.id AS id, s.name AS name')) return 'pdgOwner'; + if (query.includes('RETURN s.id AS id, s.filePath AS filePath')) return 'pdgSpans'; + if (query.includes('RETURN c.id AS id, r.reason AS reason')) return 'pdgSummary'; + if (query.includes('RETURN a.id AS id, r.reason AS reason')) return 'pdgSelf'; + if (query.includes('RETURN a.id AS id ORDER BY a.startLine')) return 'pdgSeed'; + if (query.includes('RETURN b.id AS id')) { + if (query.includes('b.calleeIds AS calleeIds')) return 'pdgCalleeBlocks'; + if (query.includes('b.text AS text')) return 'pdgStatement'; + return 'pdgSeed'; + } + if (query.includes('BasicBlock') && query.includes('RETURN DISTINCT')) return 'pdgNeighbor'; + if (query.includes('WITH DISTINCT caller') || query.includes('WITH DISTINCT target')) + return 'chain'; + if (query.includes('caller.id AS uid')) + return query.includes('(ctor:Constructor)') ? 'classIncoming' : 'incoming'; + if (query.includes('target.id AS uid')) return 'outgoing'; + if (query.includes('RETURN p.id AS uid')) return 'typedProperties'; + if (query.includes('RETURN p.id AS pid, p.heuristicLabel AS label')) return 'contextProcess'; + if (query.includes('RETURN route.name AS url')) return 'contextRoute'; + if (query.includes('RETURN DISTINCT iface.id AS id')) return 'interfaceBoundary'; + if (query.includes('RETURN COUNT(DISTINCT other.id) AS cnt')) return 'interfaceCount'; + if ( + query.includes('RETURN c.id AS id') || + query.includes('RETURN f.id AS id') || + query.includes('RETURN p.id AS id') + ) + return 'seeds'; + if (query.includes('RETURN DISTINCT member.id AS id')) return 'members'; + if (query.includes('AS sourceId')) return 'frontier'; + if (query.includes('RETURN p.id AS pId')) return 'process'; + if (query.includes('RETURN p.id AS pid, MIN(r.step) AS minStep')) return 'backfill'; + if (query.includes('RETURN s.id AS sid')) return 'membership'; + if (query.includes('c.heuristicLabel AS name')) return 'modules'; + if (query.includes('RETURN h.id AS hid')) return 'impactRoute'; + if (query.includes('n.visibility AS visibility')) return 'metadata'; + return undefined; +} + +async function context(extra = {}) { + return backend.callTool('context', { name: TARGET.name, ...extra }); +} +async function impact(extra = {}) { + return backend.callTool('impact', { + target: TARGET.name, + direction: 'upstream', + maxDepth: 1, + ...extra, + }); +} +function expectIntegrityError(result: any, isImpact = false) { + expect(result.error).toMatch(/invalid symbol identity/i); + expect(result.recoverySuggestion).toMatch(/analyze.*--force/); + expect(JSON.stringify(result)).not.toContain('persisted-value'); + expect(result).not.toHaveProperty('symbol'); + expect(result).not.toHaveProperty('incoming'); + if (isImpact) { + expect(result.risk).toBe('UNKNOWN'); + expect(result.impactedCount).toBeNull(); + expect(result.epistemic).not.toBe('exact'); + } +} +function tuple(value: Record, keys: string[]): unknown[] { + return keys.map((key) => value[key]); +} + +beforeEach(async () => { + vi.clearAllMocks(); + aop.mockResolvedValue(undefined); + failedSeam = undefined; + rows = { target: [{ ...TARGET }], frontier: [{ ...EDGE }] }; + db.executeParameterized.mockImplementation(async (_db, query, params) => { + const seam = querySeam(query, params); + return seam ? fixture(seam) : []; + }); + backend = new LocalBackend(); + await backend.init(); + vi.spyOn(backend as any, 'ensureInitialized').mockResolvedValue(undefined); + vi.spyOn(backend as any, 'computeEpistemicBoundary').mockResolvedValue({ epistemic: 'exact' }); +}); + +describe('identity corruption before target selection', () => { + for (const corrupt of [true, false]) { + it('distinguishes corrupt and ordinary ambiguous candidate failures: ' + corrupt, async () => { + rows.target = [ + { ...TARGET, id: 'func:one', filePath: 'src/one.ts' }, + { ...TARGET, id: 'func:two', filePath: 'src/two.ts' }, + ]; + vi.spyOn(backend as any, '_runImpactBFS').mockRejectedValue( + corrupt ? new SymbolIdentityError() : new Error('ordinary candidate failure'), + ); + const result = await impact(); + if (corrupt) { + expectIntegrityError(result, true); + } else { + expect(result.error).toBeUndefined(); + expect(result.status).toBe('ambiguous'); + expect(result.partialProbe).toBe(true); + } + }); + } + for (const tool of ['context', 'impact']) { + for (const badRow of [ + { id: BAD, label: 'Class' }, + { id: '', label: 'Class' }, + { id: 42, label: 'Class' }, + { id: TARGET.id, label: BAD }, + ]) { + it('rejects corrupt label enrichment for ' + tool + JSON.stringify(badRow), async () => { + rows.target = [{ ...TARGET, type: '' }]; + rows.targetLabels = [badRow, { id: TARGET.id, label: 'Class' }]; + expectIntegrityError( + tool === 'context' ? await context() : await impact(), + tool === 'impact', + ); + }); + } + } + for (const shape of ['object', 'tuple']) { + for (const field of ['name', 'filePath']) { + for (const tool of ['context', 'impact', 'pdg impact']) { + it('rejects NUL in target ' + field + ' from ' + shape + ' rows for ' + tool, async () => { + const target = { ...TARGET, [field]: BAD }; + rows.target = [ + shape === 'tuple' + ? tuple(target, ['id', 'name', 'type', 'filePath', 'startLine', 'endLine']) + : target, + ]; + expectIntegrityError( + tool === 'context' + ? await context() + : await impact(tool === 'pdg impact' ? { mode: 'pdg' } : {}), + tool !== 'context', + ); + }); + } + } + } + it('rejects corrupt exact-UID metadata before expansion', async () => { + rows.target = [{ ...TARGET, filePath: BAD }]; + expectIntegrityError(await context({ uid: TARGET.id })); + expectIntegrityError(await impact({ target_uid: TARGET.id }), true); + }); + for (const field of ['name', 'filePath']) { + it('rejects non-string target ' + field, async () => { + rows.target = [{ ...TARGET, [field]: 42 }]; + expectIntegrityError(await context()); + }); + } + it('validates every candidate before exact File narrowing', async () => { + rows.target = [ + { ...TARGET, id: 'File:src/target.ts', name: 'target.ts', type: 'File' }, + { ...TARGET, id: 'func:other', filePath: BAD }, + ]; + expectIntegrityError(await context({ name: TARGET.filePath })); + }); +}); + +describe('context detail row integrity', () => { + for (const seam of ['incoming', 'outgoing'] as const) { + for (const shape of ['object', 'tuple']) { + for (const field of ['uid', 'name', 'filePath']) { + it('rejects NUL in ' + seam + ' ' + field + ' from ' + shape + ' rows', async () => { + const ref = { ...REF, [field]: BAD }; + rows[seam] = [ + shape === 'tuple' ? tuple(ref, ['relType', 'uid', 'name', 'filePath', 'kind']) : ref, + ]; + expectIntegrityError(await context()); + }); + } + for (const badRow of [null, {}, [], { ...REF, uid: '' }, { ...REF, relType: '' }]) { + it( + 'rejects incomplete ' + + seam + + ' row ' + + JSON.stringify(badRow) + + ' in ' + + shape + + ' response', + async () => { + rows[seam] = [ + shape === 'tuple' && badRow !== null + ? tuple(badRow, ['relType', 'uid', 'name', 'filePath', 'kind']) + : badRow, + ]; + expectIntegrityError(await context()); + }, + ); + } + } + } + for (const badRow of [ + null, + {}, + [''], + { pid: '' }, + { pid: 'proc:target', label: BAD }, + ['proc:target', BAD, 0, 0], + ]) { + it('rejects corrupt context process ' + JSON.stringify(badRow), async () => { + rows.contextProcess = [badRow]; + expectIntegrityError(await context()); + }); + } + it('does not swallow corrupt class expansion or typed property rows', async () => { + rows.target = [{ ...TARGET, type: 'Class' }]; + rows.typedProperties = [{ uid: 'prop:target', name: 'prop', filePath: BAD, kind: 'Property' }]; + expectIntegrityError(await context()); + }); + it('rejects corrupt class refs before deduplication can discard them', async () => { + rows.target = [{ ...TARGET, type: 'Class' }]; + rows.incoming = [REF]; + rows.classIncoming = [{ ...REF, filePath: BAD }]; + expectIntegrityError(await context()); + }); + for (const badRow of [{}, { ...REF, filePath: BAD }, { ...REF, uid: '' }]) { + it('does not swallow corrupt chain rows ' + JSON.stringify(badRow), async () => { + rows.chain = [badRow]; + expectIntegrityError(await context({ chain_depth: 1 })); + }); + } + it('rejects NUL in route names', async () => { + rows.contextRoute = [{ url: BAD, method: 'GET' }]; + expectIntegrityError(await context()); + }); + it('rejects nested AOP identity fields while keeping the shared error envelope', async () => { + aop.mockResolvedValue({ + framework: 'spring', + advices: [{ adviceId: 'advice:1', adviceName: BAD }], + }); + expectIntegrityError(await context()); + }); +}); + +describe('epistemic boundary row integrity', () => { + const iface = { id: 'iface:target', name: 'Target contract', label: 'Interface' }; + + function prepareBoundary() { + vi.mocked((backend as any).computeEpistemicBoundary).mockRestore(); + rows.interfaceBoundary = [iface]; + rows.interfaceCount = [{ cnt: 2 }]; + } + + for (const tool of ['context', 'impact']) { + for (const shape of ['object', 'tuple']) { + for (const badRow of [ + { ...iface, id: undefined }, + { ...iface, id: '' }, + { ...iface, id: BAD }, + { ...iface, id: 42 }, + { ...iface, name: BAD }, + { ...iface, name: 42 }, + { ...iface, label: BAD }, + { ...iface, label: 42 }, + ]) { + it( + 'rejects corrupt ' + + tool + + ' boundary before deduplication: ' + + shape + + JSON.stringify(badRow), + async () => { + prepareBoundary(); + rows.interfaceBoundary = [iface, badRow].map((row) => + shape === 'tuple' ? tuple(row, ['id', 'name', 'label']) : row, + ); + expectIntegrityError( + tool === 'context' ? await context() : await impact(), + tool === 'impact', + ); + }, + ); + } + } + for (const seam of ['interfaceBoundary', 'interfaceCount'] as const) { + for (const corrupt of [true, false]) { + it( + 'distinguishes ' + tool + ' boundary query failure: ' + seam + ' ' + corrupt, + async () => { + prepareBoundary(); + db.executeParameterized.mockImplementation(async (_db, query, params) => { + const currentSeam = querySeam(query, params); + if (currentSeam === seam) { + throw corrupt ? new SymbolIdentityError() : new Error('ordinary boundary failure'); + } + return currentSeam ? fixture(currentSeam) : []; + }); + const result = tool === 'context' ? await context() : await impact(); + if (corrupt) { + expectIntegrityError(result, tool === 'impact'); + } else { + expect(result.error).toBeUndefined(); + expect(result.recoverySuggestion).toBeUndefined(); + expect(result.epistemic).toBe('lower-bound'); + if (tool === 'impact') expect(result.impactedCount).toBe(1); + } + }, + ); + } + } + it('preserves healthy ' + tool + ' boundary descriptions', async () => { + prepareBoundary(); + const result = tool === 'context' ? await context() : await impact(); + expect(result.error).toBeUndefined(); + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries).toContainEqual( + expect.stringContaining('Target contract is an interface'), + ); + expect(result.causes.dispatchBoundary).toBe(4); + }); + } +}); + +describe('impact detail row integrity', () => { + for (const seam of ['frontier', 'process'] as const) { + it( + 'PDG detail integrity rejects corrupt ' + seam + ' rows through the outer envelope', + async () => { + rows[seam] = [ + seam === 'frontier' ? { ...EDGE, filePath: BAD } : { ...PROCESS, epName: BAD }, + ]; + expectIntegrityError(await impact({ mode: 'pdg' }), true); + }, + ); + } + for (const shape of ['object', 'tuple']) { + for (const field of ['id', 'name', 'filePath', 'sourceId']) { + it('rejects NUL in frontier ' + field + ' from ' + shape + ' rows', async () => { + const edge = { ...EDGE, [field]: BAD }; + rows.frontier = [ + shape === 'tuple' + ? tuple(edge, [ + 'sourceId', + 'id', + 'name', + 'type', + 'filePath', + 'relType', + 'confidence', + 'staticGated', + ]) + : edge, + ]; + expectIntegrityError(await impact(), true); + }); + } + } + for (const badRow of [null, {}, [], { ...EDGE, id: '' }]) { + it('rejects incomplete frontier rows ' + JSON.stringify(badRow), async () => { + rows.frontier = [badRow]; + expectIntegrityError(await impact(), true); + }); + } + it('validates corrupt rows before test filtering', async () => { + rows.frontier = [{ ...EDGE, filePath: 'test/corrupt\0.test.ts' }]; + expectIntegrityError(await impact({ includeTests: false }), true); + }); + for (const shape of ['object', 'tuple']) { + for (const field of ['pId', 'name', 'entryPointId', 'epName', 'epFilePath']) { + it('rejects NUL in process ' + field + ' from ' + shape + ' rows', async () => { + const process = { ...PROCESS, [field]: BAD }; + rows.process = [ + shape === 'tuple' + ? tuple(process, [ + 'pId', + 'name', + 'processType', + 'entryPointId', + 'hits', + 'minStep', + 'stepCount', + 'epName', + 'epType', + 'epFilePath', + ]) + : process, + ]; + expectIntegrityError(await impact({ summaryOnly: true }), true); + }); + } + } + for (const badRow of [null, {}, [], { ...PROCESS, pId: '' }]) { + it('does not aggregate incomplete processes ' + JSON.stringify(badRow), async () => { + rows.process = [badRow]; + expectIntegrityError(await impact(), true); + }); + } + for (const badRow of [{}, { pid: BAD, minStep: 1 }]) { + it( + 'does not swallow corrupt backfill process identities ' + JSON.stringify(badRow), + async () => { + rows.process = [{ ...PROCESS, minStep: null }]; + rows.backfill = [badRow]; + expectIntegrityError(await impact(), true); + }, + ); + } + for (const badRow of [ + {}, + { sid: REF.uid, pid: '' }, + { sid: REF.uid, pid: 'proc:caller', pName: BAD }, + ]) { + it( + 'does not swallow corrupt per-symbol process identities ' + JSON.stringify(badRow), + async () => { + rows.process = [PROCESS]; + rows.membership = [badRow]; + expectIntegrityError(await impact(), true); + }, + ); + } + for (const type of ['Class', 'Const']) { + for (const badRow of [{}, { ...TARGET, id: 'seed:1', filePath: BAD }]) { + it('rejects corrupt ' + type + ' seeds ' + JSON.stringify(badRow), async () => { + rows.target = [{ ...TARGET, type }]; + rows[type === 'Class' ? 'seeds' : 'members'] = [badRow]; + expectIntegrityError(await impact({ direction: 'downstream' }), true); + }); + } + } + it('rejects NUL in module names before aggregation', async () => { + rows.modules = [{ name: BAD, hits: 1 }]; + expectIntegrityError(await impact(), true); + }); + it('rejects NUL in route names', async () => { + rows.impactRoute = [{ hid: REF.uid, url: BAD }]; + expectIntegrityError(await impact(), true); + }); + for (const shape of ['object', 'tuple']) { + for (const hid of [undefined, null, '', ' ', BAD, 42, {}]) { + it( + 'rejects corrupt route handler IDs from ' + shape + ' rows: ' + JSON.stringify(hid), + async () => { + const route = { hid, url: '/target', method: 'GET' }; + rows.impactRoute = [shape === 'tuple' ? tuple(route, ['hid', 'url', 'method']) : route]; + expectIntegrityError(await impact(), true); + }, + ); + } + } + it('rejects nested AOP paths', async () => { + aop.mockResolvedValue({ + framework: 'spring', + advices: [{ adviceId: 'advice:1', adviceFilePath: BAD }], + }); + expectIntegrityError(await impact(), true); + }); +}); + +describe('healthy and ordinary-failure compatibility', () => { + it('preserves Unicode, opaque IDs, optional NULL metadata and source NUL', async () => { + const content = 'const value = "actual\0source";'; + rows.target = [{ ...TARGET, name: 'café�', filePath: '源/café�.ts', content }]; + rows.incoming = [{ ...REF, name: '呼び出し�', filePath: null, kind: '' }]; + rows.contextProcess = [ + { pid: 'legacy:proc ', label: '', step: 0, stepCount: 0, entryPointId: null }, + ]; + rows.metadata = [{ annotations: ['@Text("source\0value")'], parameterTypes: null }]; + const result = await context({ include_content: true }); + expect(result.error).toBeUndefined(); + expect(result.symbol).toMatchObject({ + uid: TARGET.id, + name: 'café�', + filePath: '源/café�.ts', + content, + }); + expect(result.symbol.methodMetadata).toEqual({ annotations: ['@Text("source\0value")'] }); + expect(result.incoming.calls[0]).toMatchObject({ uid: REF.uid, name: '呼び出し�' }); + expect(result.processes).toEqual([ + { id: 'legacy:proc ', name: undefined, step_index: 0, step_count: 0 }, + ]); + }); + it('preserves tuple zero steps and empty process labels', async () => { + rows.contextProcess = [['proc:0', '', 0, 0, null]]; + expect((await context()).processes).toEqual([ + { id: 'proc:0', name: '', step_index: 0, step_count: 0 }, + ]); + }); + it('allows absent OPTIONAL MATCH entry-point fields and missing legacy sourceId', async () => { + rows.frontier = [{ ...EDGE, sourceId: undefined }]; + rows.process = [ + { ...PROCESS, name: '', entryPointId: null, epName: null, epType: null, epFilePath: null }, + ]; + const result = await impact(); + expect(result.error).toBeUndefined(); + expect(result.impactedCount).toBe(1); + expect(result.affected_processes).toEqual([ + { + name: 'unknown', + type: 'Function', + filePath: '', + affected_process_count: 1, + total_hits: 1, + earliest_broken_step: 0, + }, + ]); + }); + it('keeps missing optional route fields as ordinary skipped enrichment', async () => { + rows.contextRoute = [{}]; + rows.impactRoute = [{ hid: REF.uid }]; + expect((await context()).error).toBeUndefined(); + expect((await impact()).error).toBeUndefined(); + }); + it('does not misdiagnose an unmatched user-supplied NUL as index corruption', async () => { + rows.target = []; + const contextResult = await context({ name: 'client\0input' }); + const impactResult = await impact({ target: 'client\0input' }); + expect(contextResult.error).toContain('not found'); + expect(impactResult.error).toContain('not found'); + expect(contextResult.recoverySuggestion).toBeUndefined(); + expect(impactResult.recoverySuggestion).toBeUndefined(); + }); + it('keeps ordinary process query failures degraded rather than integrity errors', async () => { + failedSeam = 'process'; + const result = await impact(); + expect(result.error).toBeUndefined(); + expect(result.partial).toBe(true); + expect(result.impactedCount).toBe(1); + expect(result.affected_processes).toEqual([]); + }); + it('keeps ordinary PDG interprocedural failures as degraded results', async () => { + vi.spyOn(backend as any, '_runImpactBFS').mockRejectedValue( + new Error('ordinary bridge failure'), + ); + const result = await impact({ mode: 'pdg' }); + expect(result.error).toBeUndefined(); + expect(result.partial).toBe(true); + expect(result.interproceduralError).toBe('ordinary bridge failure'); + expect(result.recoverySuggestion).toBeUndefined(); + }); + it('keeps ordinary context process query failures as unavailable enrichment', async () => { + failedSeam = 'contextProcess'; + const result = await context(); + expect(result.error).toBeUndefined(); + expect(result.processes).toEqual([]); + }); +}); + +describe('raw PDG identities before coercion, caps, and projection', () => { + const seed = 'BasicBlock:src/target.ts:2:0:0'; + const reached = 'BasicBlock:src/caller.ts:1:0:0'; + + function preparePdg() { + rows.pdgSeed = [{ id: seed }]; + rows.pdgNeighbor = [{ id: reached }]; + rows.pdgOwner = [{ id: REF.uid, name: REF.name, label: 'Function', startLine: 0 }]; + rows.pdgStatement = [{ id: reached, line: 1, endLine: 1, text: 'value = 1;' }]; + } + + for (const seam of ['pdgSeed', 'pdgNeighbor', 'pdgOwner', 'pdgStatement'] as const) { + for (const bad of [BAD, '', null, 42]) { + it('rejects raw ' + seam + ' identity ' + JSON.stringify(bad), async () => { + preparePdg(); + rows[seam] = [{ id: bad, name: 'caller', label: 'Function', line: 1, startLine: 0 }]; + expectIntegrityError( + await impact({ mode: 'pdg', ...(seam === 'pdgStatement' ? { line: 2 } : {}) }), + true, + ); + }); + } + } + for (const seam of ['pdgSeed', 'pdgNeighbor'] as const) { + it('validates ' + seam + ' cap probe rows', async () => { + preparePdg(); + rows[seam] = [{ id: seam === 'pdgSeed' ? seed : reached }, { id: BAD }]; + expectIntegrityError(await impact({ mode: 'pdg', limit: 1 }), true); + }); + } + for (const field of ['name', 'label']) { + it('rejects raw owner ' + field + ' before String coercion', async () => { + preparePdg(); + rows.pdgOwner = [{ id: REF.uid, name: 'caller', label: 'Function', [field]: BAD }]; + expectIntegrityError(await impact({ mode: 'pdg' }), true); + }); + } + for (const field of ['seedBlocks', 'reachableBlocks', 'intraReachableBlocks']) { + for (const bad of [BAD, '', null, 42]) { + it('rejects malformed final ' + field + ' members ' + JSON.stringify(bad), async () => { + const healthy = await impact({ mode: 'pdg' }); + vi.spyOn(backend as any, '_runImpactPDG').mockResolvedValue({ + ...healthy, + [field]: [bad], + }); + expectIntegrityError(await impact({ mode: 'pdg' }), true); + }); + } + } + it('allows a generated unresolved owner marker', async () => { + preparePdg(); + rows.pdgOwner = []; + const result = await impact({ mode: 'pdg' }); + expect(result.error).toBeUndefined(); + expect(result.unresolvedBlockCount).toBe(1); + }); + it('preserves Unicode owner tuples and source NUL', async () => { + preparePdg(); + rows.pdgOwner = [[REF.uid, '呼び出し�', 'Function', 0]]; + expect((await impact({ mode: 'pdg' })).error).toBeUndefined(); + rows.pdgStatement = [{ id: reached, line: 1, endLine: 1, text: 'value = "source\0text";' }]; + const result = await impact({ mode: 'pdg', line: 2 }); + expect(result.error).toBeUndefined(); + expect(result.affectedStatements.some((s: any) => s.text.includes('\0'))).toBe(true); + }); + for (const field of ['callees', 'calleeIds']) { + it('does not swallow a corrupt statement bridge ' + field, async () => { + preparePdg(); + const original = db.executeParameterized.getMockImplementation()!; + db.executeParameterized.mockImplementation(async (...args) => { + if (args[1].includes('RETURN b.' + field + ' AS ' + field)) { + return [{ [field]: BAD }]; + } + return original(...args); + }); + expectIntegrityError(await impact({ mode: 'pdg', direction: 'downstream' }), true); + }); + } +}); + +describe('real AOP helper identities before deduplication', () => { + const reason = + 'spring-aop:v1:' + + JSON.stringify({ + kind: 'advice', + annotation: 'org.aspectj.lang.annotation.Around', + advice: 'around', + pointcut: 'execution(*)', + match: 'static', + activation: 'unknown', + proxy: 'possible', + }); + const advice = { + sourceId: TARGET.id, + sourceName: 'target', + sourceFilePath: TARGET.filePath, + targetId: 'advice:1', + targetName: 'audit', + targetFilePath: 'src/audit.ts', + reason, + }; + async function useRealAop() { + const actual = await vi.importActual( + '../../src/mcp/local/aop-metadata.js', + ); + aop.mockImplementation(actual.querySpringAopMetadata); + rows.target = [{ ...TARGET, type: 'Method' }]; + } + for (const tool of ['context', 'impact']) { + for (const field of ['sourceId', 'targetId']) { + for (const bad of [BAD, '', null, 42]) { + it('rejects raw AOP ' + field + ' for ' + tool + JSON.stringify(bad), async () => { + await useRealAop(); + rows.aopRows = [{ ...advice, [field]: bad }, advice]; + expectIntegrityError( + tool === 'context' ? await context() : await impact(), + tool === 'impact', + ); + }); + } + } + for (const field of ['sourceName', 'sourceFilePath', 'targetName', 'targetFilePath']) { + it('rejects corrupt duplicate AOP ' + field + ' for ' + tool, async () => { + await useRealAop(); + rows.aopRows = [{ ...advice, [field]: BAD }, advice]; + expectIntegrityError( + tool === 'context' ? await context() : await impact(), + tool === 'impact', + ); + }); + } + } + it('validates the AOP cap-probe row', async () => { + await useRealAop(); + rows.aopRows = [...Array.from({ length: 1000 }, () => advice), { ...advice, targetId: BAD }]; + expectIntegrityError(await context()); + }); + it('preserves Unicode and optional NULL metadata', async () => { + await useRealAop(); + rows.aopRows = [ + { ...advice, sourceName: '源�', sourceFilePath: null, targetName: '', targetFilePath: null }, + ]; + const result = await context(); + expect(result.error).toBeUndefined(); + expect(result.symbol.aop.advices[0]).toMatchObject({ + adviceId: advice.targetId, + advisedId: advice.sourceId, + advisedName: '源�', + }); + }); + it('keeps ordinary AOP query failures fail-soft', async () => { + await useRealAop(); + failedSeam = 'aopRows'; + expect((await context()).error).toBeUndefined(); + expect((await impact()).error).toBeUndefined(); + }); + it('handles early AOP rejection while the frontier is pending', async () => { + const unhandled = vi.fn(); + process.on('unhandledRejection', unhandled); + aop.mockRejectedValue(new SymbolIdentityError()); + const original = db.executeParameterized.getMockImplementation()!; + db.executeParameterized.mockImplementation(async (...args) => { + if (querySeam(args[1], args[2]) === 'frontier') { + await new Promise((resolve) => setTimeout(resolve, 30)); + } + return original(...args); + }); + try { + expectIntegrityError(await impact(), true); + expect(unhandled).not.toHaveBeenCalled(); + } finally { + process.off('unhandledRejection', unhandled); + } + }); +}); diff --git a/gitnexus/test/unit/impact-pagination.test.ts b/gitnexus/test/unit/impact-pagination.test.ts index 9b8aa8e21..0a710a71e 100644 --- a/gitnexus/test/unit/impact-pagination.test.ts +++ b/gitnexus/test/unit/impact-pagination.test.ts @@ -56,6 +56,7 @@ function setupMultiDepthHub(d1Count: number, d2Count: number) { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); if (query.includes('STEP_IN_PROCESS')) return []; if (query.includes('MEMBER_OF')) return []; + if (query.includes('RETURN h.id AS hid')) return []; // The #1858 epistemic-boundary probe (computeEpistemicBoundary) runs // concurrently with the BFS and also matches `r.type IN`, but targets the // `iface` alias. Return empty so it stays `epistemic: 'exact'` and does not @@ -99,6 +100,7 @@ function setupHubSymbol(count: number) { const query = typeof args[1] === 'string' ? args[1] : String(args[0] ?? ''); if (query.includes('STEP_IN_PROCESS')) return []; if (query.includes('MEMBER_OF')) return []; + if (query.includes('RETURN h.id AS hid')) return []; // See setupMultiDepthHub — keep the #1858 epistemic probe from matching the // `r.type IN` caller branch below. if (query.includes('iface')) return []; diff --git a/gitnexus/test/unit/impact-route-enrichment.test.ts b/gitnexus/test/unit/impact-route-enrichment.test.ts index 9e9cc8791..d2b53031c 100644 --- a/gitnexus/test/unit/impact-route-enrichment.test.ts +++ b/gitnexus/test/unit/impact-route-enrichment.test.ts @@ -80,7 +80,10 @@ async function runImpact(routeRows: readonly RouteRow[], routeQueryFails = false : []; } if (query.includes('STEP_IN_PROCESS') || query.includes('MEMBER_OF')) return []; - return [{ id: 'svc', name: 'UnfinalizeRound', filePath: 'svc.go', type: 'Method' }]; + if (query.includes('n.id AS id')) { + return [{ id: 'svc', name: 'UnfinalizeRound', filePath: 'svc.go', type: 'Method' }]; + } + return []; }); const backend = new LocalBackend(); diff --git a/gitnexus/test/unit/incremental-parse-cache.test.ts b/gitnexus/test/unit/incremental-parse-cache.test.ts index b86c19926..fb2916ab4 100644 --- a/gitnexus/test/unit/incremental-parse-cache.test.ts +++ b/gitnexus/test/unit/incremental-parse-cache.test.ts @@ -302,8 +302,11 @@ describe('PARSE_CACHE_VERSION', () => { // Moved 121 -> 122 for #3414 restoring helper calls. // Moved 122 -> 123 for #3408 FastAPI nested router-prefix capture fields. // Moved 123 -> 124 for #3402 Go gin/echo decorator routes. - it('pins SCHEMA_BUMP to 125 so concurrent bumps cannot silently collide (#2766, #3015, #3088, #2885, #3128, #2865, #3130, #1432, #3161, #3179, #3219, #3190, #3253, #3273, #3339, #3354, #3371, #2965, #3390, #3398, #3396, #3394, #3399, #3414, #3408, #3402)', () => { - expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(125); + // Moved 125 -> 126 for #3446: SDK positional tool definitions and attribution. + // Moved 126 -> 127 for #3450: reject destructured SDK registration-method writes. + // Moved 127 -> 128 for #3450: recognize SDK namespace imports. + it('pins SCHEMA_BUMP to 128 so concurrent bumps cannot silently collide (#2766, #3015, #3088, #2885, #3128, #2865, #3130, #1432, #3161, #3179, #3219, #3190, #3253, #3273, #3339, #3354, #3371, #2965, #3390, #3398, #3396, #3394, #3399, #3414, #3408, #3402, #3446, #3450)', () => { + expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).toBe(128); expect(PARSE_CACHE_BUCKET_COUNT).toBe(128); // The PREVIOUS version must fail the reuse gate, not merely differ from the // current one — a hardcoded number outside the conflict hunk rebases cleanly @@ -313,7 +316,7 @@ describe('PARSE_CACHE_VERSION', () => { 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, - 123, 124, + 123, 124, 125, 126, 127, ]) { expect(Number(PARSE_CACHE_VERSION.split('+', 1)[0])).not.toBe(taken); } diff --git a/gitnexus/test/unit/incremental-write-benchmark-verdict.test.ts b/gitnexus/test/unit/incremental-write-benchmark-verdict.test.ts new file mode 100644 index 000000000..e1c3b0c35 --- /dev/null +++ b/gitnexus/test/unit/incremental-write-benchmark-verdict.test.ts @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import vm from 'node:vm'; +import { parse } from '@babel/parser'; +import { isVariableDeclarator, traverseFast } from '@babel/types'; +import { expect, it } from 'vitest'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { reconcileGraphNodeIdentities } from '../../src/core/incremental/write-reconciliation.js'; + +// Execute the standalone benchmark's actual audit closure, without its timed +// workload, against the production oracle and controlled native scan results. +const source = readFileSync( + new URL('../../bench/incremental-write-integrity/measure.cjs', import.meta.url), + 'utf8', +); +let auditSource: string | undefined; +traverseFast(parse(source, { sourceType: 'script' }), (node) => { + if ( + isVariableDeclarator(node) && + node.id.type === 'Identifier' && + node.id.name === 'audit' && + node.init?.start != null && + node.init.end != null + ) { + auditSource = source.slice(node.init.start, node.init.end); + } +}); +if (!auditSource) throw new Error('Benchmark audit closure not found'); + +const row = { id: 'Function:a.ts:f', name: 'f', filePath: 'a.ts', startLine: 0, endLine: 1 }; +it.each([ + { name: 'healthy scan', rows: [row], verdict: 'certified' }, + { name: 'complete tuple set plus duplicate row', rows: [row, row], verdict: 'rejected' }, + { + name: 'complete tuple set plus unexpected row', + rows: [row, { ...row, id: 'Function:a.ts:extra' }], + verdict: 'rejected', + }, + { name: 'missing tuple', rows: [], verdict: 'rejected' }, + { name: 'inconsistent range', rows: [{ ...row, startLine: 99 }], verdict: 'rejected' }, +])('reports the production rejection verdict for $name', async ({ rows, verdict }) => { + const graph = createKnowledgeGraph(); + graph.addNode({ id: row.id, label: 'Function', properties: { ...row } }); + const wanted = new Set([ + JSON.stringify([row.id, row.name, row.filePath, row.startLine, row.endLine]), + ]); + const nativePhases: { verdict: string }[] = []; + const audit = vm.runInNewContext(`(${auditSource})`, { + assert, + graph, + wanted, + nativePhases, + size: { nodes: 1 }, + reconcileGraphNodeIdentities, + adapter: { executeQuery: async (query: string) => (query.includes('Function') ? rows : []) }, + }) as (phase: string) => Promise; + await audit('controlled-scan'); + expect(nativePhases).toEqual([expect.objectContaining({ verdict, rows: rows.length })]); +}); diff --git a/gitnexus/test/unit/incremental-write-integrity.test.ts b/gitnexus/test/unit/incremental-write-integrity.test.ts new file mode 100644 index 000000000..14ac9fca9 --- /dev/null +++ b/gitnexus/test/unit/incremental-write-integrity.test.ts @@ -0,0 +1,267 @@ +import { appendFile } from 'node:fs/promises'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { setupMiniRepo } from '../helpers/mini-repo.js'; +import { getStoragePaths, loadMeta, readRegistry } from '../../src/storage/repo-manager.js'; +import * as adapter from '../../src/core/lbug/lbug-adapter.js'; +import * as fts from '../../src/core/search/fts-indexes.js'; +import * as analyzerIdentity from '../../src/core/analyzer-identity.js'; +import * as checkpoints from '../../src/core/lbug/wal-checkpoint-driver.js'; +import { commitAll, initGitRepo } from '../helpers/temp-git-repo.js'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { reconcileGraphNodeIdentities } from '../../src/core/incremental/write-reconciliation.js'; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + +const options = { skipAgentsMd: true, skipSkills: true }; +const callbacks = { onProgress: () => {} }; + +async function touchHandler(repoPath: string): Promise { + initGitRepo(repoPath); + await appendFile(path.join(repoPath, 'src/handler.ts'), '\n// integrity probe\n'); + commitAll(repoPath, 'touch handler'); +} + +describe('incremental graph identity before publication', () => { + it('chooses a full write before selective mutation when manual checkpoints are disabled', async () => { + const repo = await setupMiniRepo('gitnexus-test-checkpoint-opt-out-'); + const { storagePath } = getStoragePaths(repo.dbPath); + try { + await runFullAnalysis(repo.dbPath, options, callbacks); + await touchHandler(repo.dbPath); + vi.stubEnv('GITNEXUS_WAL_MANUAL_CHECKPOINT', '0'); + const deleteNodes = vi.spyOn(adapter, 'deleteNodesForFiles'); + const flush = vi.spyOn(adapter, 'tryFlushWAL'); + const logs: string[] = []; + const result = await runFullAnalysis(repo.dbPath, options, { + ...callbacks, + onLog: (line) => logs.push(line), + }); + expect(deleteNodes).not.toHaveBeenCalled(); + expect(flush).not.toHaveBeenCalled(); + expect(result.incrementalStats).toBeUndefined(); + expect(logs).toContain( + 'Manual WAL checkpoints are disabled; switching to a full DB write before mutation.', + ); + expect((await loadMeta(storagePath))?.incrementalInProgress).toBeUndefined(); + } finally { + await adapter.closeLbug(); + await repo.cleanup(); + } + }); + + it('retries a transient checkpoint I/O failure at the final publication gate', async () => { + const repo = await setupMiniRepo('gitnexus-test-final-checkpoint-retry-'); + const { storagePath } = getStoragePaths(repo.dbPath); + try { + await runFullAnalysis(repo.dbPath, options, callbacks); + await touchHandler(repo.dbPath); + // Isolate the final gate from the periodic driver's independent cadence. + vi.spyOn(checkpoints, 'startWalCheckpointDriver').mockReturnValue({ stop: async () => {} }); + const flush = adapter.tryFlushWAL; + const build = fts.buildSearchIndexesOrDegrade; + let attempts = 0; + vi.spyOn(fts, 'buildSearchIndexesOrDegrade').mockImplementation(async (...args) => { + const result = await build(...args); + vi.spyOn(adapter, 'tryFlushWAL').mockImplementation(async () => { + if (++attempts === 1) { + throw new Error( + 'Runtime exception: IO exception: Error renaming file db.wal to db.wal.checkpoint', + ); + } + return flush(); + }); + return result; + }); + const result = await runFullAnalysis(repo.dbPath, options, callbacks); + expect(attempts).toBe(2); + expect(result.incrementalStats?.writeMode).toBe('incremental'); + expect((await loadMeta(storagePath))?.incrementalInProgress).toBeUndefined(); + } finally { + await adapter.closeLbug(); + await repo.cleanup(); + } + }); + it('matches native CSV normalization for nullable, absent-range and Unicode identity fields', async () => { + const repo = await setupMiniRepo('gitnexus-test-identity-parity-'); + const { storagePath, lbugPath } = getStoragePaths(repo.dbPath); + const graph = createKnowledgeGraph(); + graph.addNode({ + id: 'File:src/Å.ts', + label: 'File', + properties: { name: 'Å.ts', filePath: 'src/Å.ts' }, + }); + graph.addNode({ + id: 'Route:/王', + label: 'Route', + properties: { name: '/王', filePath: 'src/Å.ts' }, + }); + graph.addNode({ + id: 'Tool:王', + label: 'Tool', + properties: { name: '王', filePath: 'src/Å.ts' }, + }); + graph.addNode({ id: 'Destination:topic', label: 'Destination', properties: { name: 'topic' } }); + graph.addNode({ + id: 'Function:src/Å.ts:王\u0000\uD800', + label: 'Function', + properties: { name: '王\r\n\uD800', filePath: 'src/Å.ts' }, + }); + try { + await adapter.initLbug(lbugPath, { skipFts: true }); + await adapter.loadGraphToLbug( + graph, + repo.dbPath, + storagePath, + undefined, + undefined, + undefined, + 'none', + ); + await adapter.tryFlushWAL(); + await expect( + reconcileGraphNodeIdentities(graph, adapter.executeQuery, 'native CSV parity'), + ).resolves.toMatchObject({ nodes: 5 }); + } finally { + await adapter.closeLbug(); + await repo.cleanup(); + } + }); + + it.each([ + { phase: 'after-copy', atomicIncremental: false }, + { phase: 'after-fts', atomicIncremental: false }, + { phase: 'after-fts', atomicIncremental: true }, + { phase: 'after-fts-finalization', atomicIncremental: false }, + { phase: 'checkpoint-false', atomicIncremental: false }, + { phase: 'checkpoint-throw', atomicIncremental: true }, + ])( + 'refuses freshness after $phase failure (atomic=$atomicIncremental)', + async ({ phase, atomicIncremental }) => { + const repo = await setupMiniRepo('gitnexus-test-write-integrity-'); + const { storagePath, lbugPath } = getStoragePaths(repo.dbPath); + try { + const healthy = await runFullAnalysis(repo.dbPath, options, callbacks); + const before = await loadMeta(storagePath); + const registeredBefore = (await readRegistry()).find((entry) => entry.path === repo.dbPath); + const logs: string[] = []; + const victim = [...healthy.pipelineResult.graph.iterNodes()].find( + (node) => node.label === 'Function' && node.properties.filePath === 'src/validator.ts', + ); + expect(victim?.id).toBeTruthy(); + + await touchHandler(repo.dbPath); + const damage = async () => { + await adapter.executePrepared('MATCH (n:Function {id: $id}) DETACH DELETE n', { + id: victim.id, + }); + }; + if (phase === 'after-copy') { + const load = adapter.loadGraphToLbug; + vi.spyOn(adapter, 'loadGraphToLbug').mockImplementation(async (...args) => { + const result = await load(...args); + await damage(); + return result; + }); + } else { + const build = fts.buildSearchIndexesOrDegrade; + vi.spyOn(fts, 'buildSearchIndexesOrDegrade').mockImplementation(async (...args) => { + const result = await build(...args); + if (phase === 'checkpoint-false') { + vi.spyOn(adapter, 'tryFlushWAL').mockResolvedValue(false); + } else if (phase === 'checkpoint-throw') { + vi.spyOn(adapter, 'tryFlushWAL').mockRejectedValue( + new Error('injected final checkpoint failure'), + ); + } else { + await damage(); + if (phase === 'after-fts-finalization') { + vi.spyOn(analyzerIdentity, 'finalizeAnalyzerRunnerIdentity').mockImplementation( + () => { + throw new Error('injected analyzer finalization failure'); + }, + ); + } + } + return result; + }); + } + + await expect( + runFullAnalysis( + repo.dbPath, + { ...options, atomicIncremental }, + { + ...callbacks, + onLog: (line) => logs.push(line), + }, + ), + ).rejects.toThrow( + phase.startsWith('checkpoint') + ? /checkpoint/i + : phase === 'after-fts-finalization' + ? /injected analyzer finalization failure/ + : /graph.*reconciliation/i, + ); + const after = await loadMeta(storagePath); + expect(after?.lastCommit).toBe(before?.lastCommit); + expect(after?.indexedAt).toBe(before?.indexedAt); + expect(logs.filter((line) => line.includes('atomic-incremental'))).toEqual( + atomicIncremental ? [expect.stringContaining('staged')] : [], + ); + expect(Boolean(after?.incrementalInProgress)).toBe(!atomicIncremental); + const registeredAfter = (await readRegistry()).find((entry) => entry.path === repo.dbPath); + expect(registeredAfter?.indexedAt).toBe(registeredBefore?.indexedAt); + expect(registeredAfter?.lastCommit).toBe(registeredBefore?.lastCommit); + vi.restoreAllMocks(); + if (atomicIncremental) { + await adapter.initLbug(lbugPath); + const retained = await adapter.executePrepared( + 'MATCH (n:Function {id: $id}) RETURN n.id AS id', + { + id: victim.id, + }, + ); + expect(retained).toEqual([{ id: victim.id }]); + await adapter.closeLbug(); + } else { + expect(after?.incrementalInProgress).toMatchObject({ + phase: 'graph-reconciliation', + checkpointSucceeded: false, + }); + await expect( + runFullAnalysis(repo.dbPath, { ...options, repairFts: true }, callbacks), + ).rejects.toThrow(/mid-incremental-recovery/); + expect((await loadMeta(storagePath))?.incrementalInProgress?.phase).toBe( + 'graph-reconciliation', + ); + // The dirty marker drives recovery without a manual --force, and a + // no-op rerun after recovery must stop rebuilding. + const recovered = await runFullAnalysis(repo.dbPath, options, callbacks); + expect(recovered.incrementalStats).toBeUndefined(); + expect((await loadMeta(storagePath))?.incrementalInProgress).toBeUndefined(); + await adapter.initLbug(lbugPath, { readOnly: true }); + await expect( + reconcileGraphNodeIdentities( + recovered.pipelineResult.graph, + adapter.executeQuery, + 'recovered/reopened', + ), + ).resolves.toMatchObject({ nodes: expect.any(Number) }); + await adapter.closeLbug(); + expect((await runFullAnalysis(repo.dbPath, options, callbacks)).alreadyUpToDate).toBe( + true, + ); + } + } finally { + await adapter.closeLbug(); + await repo.cleanup(); + } + }, + 120_000, + ); +}); diff --git a/gitnexus/test/unit/incremental-write-reproducer-cleanup.test.ts b/gitnexus/test/unit/incremental-write-reproducer-cleanup.test.ts new file mode 100644 index 000000000..ed3f2dbfe --- /dev/null +++ b/gitnexus/test/unit/incremental-write-reproducer-cleanup.test.ts @@ -0,0 +1,265 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import vm from 'node:vm'; +import { expect, it } from 'vitest'; + +const source = readFileSync( + new URL('../../bench/incremental-write-integrity/reproduce.cjs', import.meta.url), + 'utf8', +); + +type FailurePoint = + | 'write:1' + | 'write:2' + | 'database:1' + | 'database:2' + | 'connection:1' + | 'connection:2' + | 'connection-close:1' + | 'connection-close:2' + | 'database-close:1' + | 'database-close:2' + | 'remove'; + +// Execute the actual entry point with controlled native and file APIs. Healthy +// scan tuples allow constructor and cleanup faults in both native sessions. +async function runReproducer( + failures: FailurePoint[] = [], + keep = false, + options: { + requireCorruption?: boolean; + copiedScan?: 'missing' | 'duplicate' | 'wrong-field'; + } = {}, +) { + const directory = path.join(os.tmpdir(), 'ladybug-copy-identity-controlled'); + const events: string[] = []; + const diagnostics: unknown[] = []; + const injected = new Map(failures.map((point) => [point, new Error(point)])); + let databases = 0; + let connections = 0; + let writes = 0; + let scans = 0; + let indices = Array.from({ length: 8192 }, (_, i) => i); + const output: string[] = []; + const processDouble = { + argv: [ + 'node', + 'reproduce.cjs', + ...(keep ? ['--keep'] : []), + ...(options.requireCorruption ? ['--require-corruption'] : []), + ], + version: process.version, + exitCode: 0, + stdout: { write: (value: string) => output.push(value) }, + }; + const fail = (point: FailurePoint) => { + const error = injected.get(point); + if (error) throw error; + }; + class Database { + readonly number = ++databases; + constructor() { + events.push(`database:${this.number}`); + fail(`database:${this.number}` as FailurePoint); + } + async close() { + events.push(`database-close:${this.number}`); + fail(`database-close:${this.number}` as FailurePoint); + } + } + class Connection { + readonly number = ++connections; + constructor(database: Database) { + events.push(`connection:${this.number}`); + expect(database.number).toBe(this.number); + fail(`connection:${this.number}` as FailurePoint); + } + async query(cypher: string) { + if (cypher.includes('DETACH DELETE')) { + indices = indices.filter((i) => i >= 32 && i < 8192 - 32); + } else if (cypher.startsWith('COPY')) { + indices = Array.from({ length: 8192 }, (_, i) => i); + } + const rows = cypher.includes('RETURN id(n)') + ? indices.map((i) => ({ + internalID: { offset: i }, + id: `Function:src/owner${Math.floor(i / 32)}.ts:fn${i}`, + name: `fn${i}`, + filePath: `src/owner${Math.floor(i / 32)}.ts`, + startLine: (i % 32) * 4, + endLine: (i % 32) * 4 + 2, + })) + : []; + if (cypher.includes('RETURN id(n)') && ++scans === 3) { + if (options.copiedScan === 'missing') rows.pop(); + if (options.copiedScan === 'duplicate') rows.push({ ...rows[0] }); + if (options.copiedScan === 'wrong-field') rows[0].name = 'wrong-function'; + } + return { getAll: async () => rows, close: async () => {} }; + } + async close() { + events.push(`connection-close:${this.number}`); + fail(`connection-close:${this.number}` as FailurePoint); + } + } + const fsDouble = { + mkdtemp: async () => directory, + writeFile: async () => { + const point = `write:${++writes}` as FailurePoint; + events.push(point); + fail(point); + }, + rm: async (target: string, options: { recursive: boolean; force: boolean }) => { + expect(target).toBe(directory); + expect(options).toEqual({ recursive: true, force: true }); + events.push('remove'); + fail('remove'); + }, + }; + const modules: Record = { + 'node:assert/strict': assert, + 'node:fs/promises': fsDouble, + 'node:os': os, + 'node:path': path, + '@ladybugdb/core': { Database, Connection, VERSION: 'controlled' }, + }; + await vm.runInNewContext(source, { + require: (name: string) => { + if (!(name in modules)) throw new Error(`Unexpected require: ${name}`); + return modules[name]; + }, + process: processDouble, + console: { error: (error: unknown) => diagnostics.push(error) }, + AggregateError, + }); + return { events, diagnostics, injected, output, exitCode: processDouble.exitCode, directory }; +} + +it.each([ + { point: 'write:1', closed: [] }, + { point: 'write:2', closed: [] }, + { point: 'database:1', closed: [] }, + { point: 'connection:1', closed: ['database-close:1'] }, + { point: 'database:2', closed: ['connection-close:1', 'database-close:1'] }, + { + point: 'connection:2', + closed: ['connection-close:1', 'database-close:1', 'database-close:2'], + }, + { point: 'connection-close:1', closed: ['connection-close:1', 'database-close:1'] }, + { point: 'database-close:1', closed: ['connection-close:1', 'database-close:1'] }, + { + point: 'connection-close:2', + closed: ['connection-close:1', 'database-close:1', 'connection-close:2', 'database-close:2'], + }, + { + point: 'database-close:2', + closed: ['connection-close:1', 'database-close:1', 'connection-close:2', 'database-close:2'], + }, +] satisfies { point: FailurePoint; closed: string[] }[])( + 'closes acquired resources and removes the directory when $point fails', + async ({ point, closed }) => { + const result = await runReproducer([point]); + expect(result.exitCode).toBe(1); + expect(result.diagnostics).toEqual([result.injected.get(point)]); + expect(result.events.filter((event) => event.includes('-close:'))).toEqual(closed); + expect(result.events.at(-1)).toBe('remove'); + expect(result.events.filter((event) => event === 'remove')).toHaveLength(1); + }, +); + +it('closes both native sessions and removes the directory after a healthy run', async () => { + const result = await runReproducer(); + expect(result.exitCode).toBe(0); + expect(result.diagnostics).toEqual([]); + expect(result.events.slice(-5)).toEqual([ + 'database:2', + 'connection:2', + 'connection-close:2', + 'database-close:2', + 'remove', + ]); + const report = JSON.parse(result.output.join('')); + expect(report.phases).toEqual([ + { phase: 'baseline', rows: 8192, wrong_tuples: 0, missing_tuples: 0, examples: [] }, + { phase: 'after-delete', rows: 8128, wrong_tuples: 0, missing_tuples: 64, examples: [] }, + { phase: 'after-copy', rows: 8192, wrong_tuples: 0, missing_tuples: 0, examples: [] }, + { phase: 'after-checkpoint', rows: 8192, wrong_tuples: 0, missing_tuples: 0, examples: [] }, + { phase: 'reopened', rows: 8192, wrong_tuples: 0, missing_tuples: 0, examples: [] }, + ]); + expect(report.directory).toBeUndefined(); +}); + +it.each([ + { copiedScan: 'missing', rows: 8191, wrong_tuples: 0, missing_tuples: 1 }, + { copiedScan: 'duplicate', rows: 8193, wrong_tuples: 0, missing_tuples: 0 }, + { copiedScan: 'wrong-field', rows: 8192, wrong_tuples: 1, missing_tuples: 1 }, +] as const)( + 'accepts a $copiedScan scan discrepancy as reproduced corruption', + async ({ copiedScan, ...expected }) => { + const result = await runReproducer([], false, { requireCorruption: true, copiedScan }); + const report = JSON.parse(result.output.join('')); + expect( + report.phases.find((phase: { phase: string }) => phase.phase === 'after-copy'), + ).toMatchObject(expected); + expect(result.exitCode).toBe(0); + expect(result.diagnostics).toEqual([]); + expect(result.events.at(-1)).toBe('remove'); + }, +); + +it('rejects --require-corruption when the native scan is healthy', async () => { + const result = await runReproducer([], false, { requireCorruption: true }); + expect(result.exitCode).toBe(1); + expect(result.diagnostics).toHaveLength(1); + expect((result.diagnostics[0] as Error).message).toContain('Native failure did not reproduce'); + expect(result.events.at(-1)).toBe('remove'); +}); + +it('keeps the directory deliberately without retaining native handles for --keep', async () => { + const result = await runReproducer([], true); + expect(result.exitCode).toBe(0); + expect(result.events).not.toContain('remove'); + expect(result.events.slice(-2)).toEqual(['connection-close:2', 'database-close:2']); + expect(JSON.parse(result.output.join('')).directory).toBe(result.directory); +}); + +it('honors --keep when connection construction fails', async () => { + const result = await runReproducer(['connection:1'], true); + expect(result.exitCode).toBe(1); + expect(result.diagnostics).toEqual([result.injected.get('connection:1')]); + expect(result.events.at(-1)).toBe('database-close:1'); + expect(result.events).not.toContain('remove'); +}); + +it('retains both close failures and still removes the directory', async () => { + const result = await runReproducer(['connection-close:1', 'database-close:1']); + expect(result.exitCode).toBe(1); + const error = result.diagnostics[0] as AggregateError; + expect(error).toBeInstanceOf(AggregateError); + expect(error.errors).toEqual([ + result.injected.get('connection-close:1'), + result.injected.get('database-close:1'), + ]); + expect(result.events.at(-1)).toBe('remove'); +}); + +it('retains the original construction error when database cleanup also fails', async () => { + const result = await runReproducer(['connection:1', 'database-close:1']); + expect(result.exitCode).toBe(1); + const error = result.diagnostics[0] as AggregateError; + expect(error).toBeInstanceOf(AggregateError); + expect(error.errors).toEqual([ + result.injected.get('connection:1'), + result.injected.get('database-close:1'), + ]); + expect(result.events.at(-1)).toBe('remove'); +}); + +it('reports directory-removal failures after closing the native handles', async () => { + const result = await runReproducer(['remove']); + expect(result.exitCode).toBe(1); + expect(result.diagnostics).toEqual([result.injected.get('remove')]); + expect(result.events.slice(-3)).toEqual(['connection-close:2', 'database-close:2', 'remove']); +}); diff --git a/gitnexus/test/unit/index-lock.test.ts b/gitnexus/test/unit/index-lock.test.ts index 1a6846cc0..120e09727 100644 --- a/gitnexus/test/unit/index-lock.test.ts +++ b/gitnexus/test/unit/index-lock.test.ts @@ -17,6 +17,7 @@ import { existsSync, chmodSync, symlinkSync, + mkdirSync, } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -178,6 +179,109 @@ describe('release', () => { }); describe('sweepStagingArtifacts', () => { + const recoveryStage = 'lbug.staging.6e34c761-bf58-46cc-8b54-78d11607bc46'; + const seedRecovery = (stagingFile = recoveryStage): void => { + writeFileSync( + path.join(dir, 'gitnexus.json'), + JSON.stringify({ + embeddingCheckpoint: { + kind: 'interrupted', + at: '2026-10-03T12:00:00.000Z', + nodesProcessed: 1, + totalNodes: 2, + chunksProcessed: 1, + model: 'test-model', + dimensions: 2, + provider: 'local', + pendingNodeIds: ['node-2'], + recovery: { + stagingFile, + schemaFingerprint: 'schema-v1', + unsafeNodeIds: ['node-2'], + }, + }, + }), + ); + }; + + it('retains the checkpoint-referenced family while reclaiming unrelated orphans', () => { + const family = [ + '', + '.wal', + '.shadow', + '.wal.checkpoint', + '.lock', + '.checkpoint.intent.lock', + '.checkpoint.apply.lock', + ].map((suffix) => recoveryStage + suffix); + for (const name of [...family, `${recoveryStage}.unexpected`, 'lbug.staging.orphan']) { + writeFileSync(path.join(dir, name), 'x'); + } + seedRecovery(); + + sweepStagingArtifacts(dir); + + for (const name of family) expect(existsSync(path.join(dir, name))).toBe(true); + expect(existsSync(path.join(dir, `${recoveryStage}.unexpected`))).toBe(false); + expect(existsSync(path.join(dir, 'lbug.staging.orphan'))).toBe(false); + }); + + it('preserves a referenced generation when a shared caller acquires the lock', async () => { + writeFileSync(path.join(dir, recoveryStage), 'x'); + seedRecovery(); + const lock = await acquireIndexLock(dir); + try { + expect(existsSync(path.join(dir, recoveryStage))).toBe(true); + } finally { + lock.release(); + } + }); + + it('retains only the referenced family in a branch sub-slot', async () => { + const branchDir = path.join(dir, 'branches', 'feature'); + mkdirSync(branchDir, { recursive: true }); + seedRecovery(); + const metadata = JSON.parse(readFileSync(path.join(dir, 'gitnexus.json'), 'utf8')); + writeFileSync( + path.join(branchDir, 'gitnexus.json'), + JSON.stringify({ ...metadata, storagePath: dir }), + ); + writeFileSync(path.join(branchDir, recoveryStage), 'stage'); + writeFileSync(path.join(branchDir, 'lbug.staging.orphan'), 'orphan'); + writeFileSync(path.join(dir, 'lbug.staging.orphan'), 'other-slot'); + + const lock = await acquireIndexLock(branchDir); + try { + expect(existsSync(path.join(branchDir, recoveryStage))).toBe(true); + expect(existsSync(path.join(branchDir, 'lbug.staging.orphan'))).toBe(false); + expect(existsSync(path.join(dir, 'lbug.staging.orphan'))).toBe(true); + } finally { + lock.release(); + } + }); + + it('rejects a symlinked reference and never follows it while reclaiming the stage', () => { + const external = path.join(dir, 'foreign-database'); + writeFileSync(external, 'external'); + symlinkSync(external, path.join(dir, recoveryStage)); + seedRecovery(); + + sweepStagingArtifacts(dir); + + expect(existsSync(path.join(dir, recoveryStage))).toBe(false); + expect(readFileSync(external, 'utf8')).toBe('external'); + }); + + it('does not sweep any generation when acquisition explicitly defers cleanup', async () => { + writeFileSync(path.join(dir, 'lbug.staging.orphan'), 'x'); + const lock = await acquireIndexLock(dir, { sweep: false }); + try { + expect(existsSync(path.join(dir, 'lbug.staging.orphan'))).toBe(true); + } finally { + lock.release(); + } + }); + it('removes only staging files, never the live index or its sidecars', () => { const files = [ 'lbug', diff --git a/gitnexus/test/unit/parse-diff-hunks.test.ts b/gitnexus/test/unit/parse-diff-hunks.test.ts index 0057803f5..84ba9ad03 100644 --- a/gitnexus/test/unit/parse-diff-hunks.test.ts +++ b/gitnexus/test/unit/parse-diff-hunks.test.ts @@ -164,7 +164,30 @@ describe('parseDiffHunks', () => { 'rename from src/old.ts', 'rename to src/new.ts', ].join('\n'); - expect(parseDiffHunks(diff)).toEqual([{ filePath: 'src/new.ts', hunks: [] }]); + expect(parseDiffHunks(diff)).toEqual([ + { filePath: 'src/new.ts', oldFilePath: 'src/old.ts', hunks: [] }, + ]); + }); + + it('decodes the source of a C-quoted rename and preserves it through content headers', () => { + const diff = [ + 'diff --git "a/src/old\\040name.py" b/src/new.txt', + 'similarity index 80%', + 'rename from "src/old\\040name.py"', + 'rename to src/new.txt', + '--- "a/src/old\\040name.py"', + '+++ b/src/new.txt', + '@@ -1 +1 @@', + '-old', + '+new', + ].join('\n'); + expect(parseDiffHunks(diff)).toEqual([ + { + filePath: 'src/new.txt', + oldFilePath: 'src/old name.py', + hunks: [{ startLine: 1, endLine: 1 }], + }, + ]); }); it('keeps line ranges for whitespace-only hunks', () => { @@ -241,7 +264,9 @@ describe('parseDiffHunks', () => { 'rename from plain.ts', 'rename to foo b/plain.ts', ].join('\n'); - expect(parseDiffHunks(diff)).toEqual([{ filePath: 'foo b/plain.ts', hunks: [] }]); + expect(parseDiffHunks(diff)).toEqual([ + { filePath: 'foo b/plain.ts', oldFilePath: 'plain.ts', hunks: [] }, + ]); }); it('keeps one FileDiff when a content line repeats +++ b/', () => { @@ -278,7 +303,7 @@ describe('parseDiffHunks', () => { 'rename to new.ts', ].join('\n'); expect(parseDiffHunksResult(diff)).toEqual({ - files: [{ filePath: 'new.ts', hunks: [] }], + files: [{ filePath: 'new.ts', oldFilePath: 'old name.ts', hunks: [] }], unparsedGitHeaders: 0, }); }); @@ -291,7 +316,7 @@ describe('parseDiffHunks', () => { 'rename to new name.ts', ].join('\n'); expect(parseDiffHunksResult(diff)).toEqual({ - files: [{ filePath: 'new name.ts', hunks: [] }], + files: [{ filePath: 'new name.ts', oldFilePath: 'old.ts', hunks: [] }], unparsedGitHeaders: 0, }); }); diff --git a/gitnexus/test/unit/pdg-impact-engine.test.ts b/gitnexus/test/unit/pdg-impact-engine.test.ts index e163df4be..3ca021e0b 100644 --- a/gitnexus/test/unit/pdg-impact-engine.test.ts +++ b/gitnexus/test/unit/pdg-impact-engine.test.ts @@ -1,12 +1,95 @@ import { describe, expect, it } from 'vitest'; import { IMPACT_MAX_DEPTH } from '../../src/mcp/tools.js'; +import { CALLEES_TRUNCATED_SENTINEL } from '../../src/core/ingestion/cfg/callee-cell-format.js'; import { pdgLayerStatus, runImpactPDG, + splitCalleeIds, type RunPdgImpactDeps, } from '../../src/mcp/local/pdg-impact.js'; +import { SymbolIdentityError } from '../../src/mcp/local/query-result-integrity.js'; + +describe('splitCalleeIds', () => { + const firstId = 'Function:src/my dir/rené.cpp:unsigned char'; + const secondId = 'Function:src/my dir/rené.cpp:long double'; + + it.each([undefined, null, '', ' ', '\t', '\t \t'])( + 'preserves an entirely empty optional cell (%j)', + (cell) => { + expect(splitCalleeIds(cell)).toEqual([]); + }, + ); + + it('preserves spaces and Unicode within healthy callee identities', () => { + expect(splitCalleeIds(`${firstId}\t${secondId}`)).toEqual([firstId, secondId]); + }); + + it('drops the generated truncation sentinel without changing resolved identities', () => { + expect(splitCalleeIds(CALLEES_TRUNCATED_SENTINEL)).toEqual([]); + expect(splitCalleeIds(`${firstId}\t${CALLEES_TRUNCATED_SENTINEL}\t${secondId}`)).toEqual([ + firstId, + secondId, + ]); + }); + + it.each([ + ['leading', `\t${firstId}`], + ['interior', `${firstId}\t\t${secondId}`], + ['trailing', `${firstId}\t`], + ['whitespace-only token', `${firstId}\t \t${secondId}`], + ['before a sentinel', `\t${CALLEES_TRUNCATED_SENTINEL}`], + ['after a sentinel', `${CALLEES_TRUNCATED_SENTINEL}\t`], + ['mixed with a sentinel', `${firstId}\t\t${CALLEES_TRUNCATED_SENTINEL}\t${secondId}`], + ])('rejects a populated cell with an empty identity (%s)', (_case, cell) => { + expect(() => splitCalleeIds(cell)).toThrow(SymbolIdentityError); + }); +}); describe('runImpactPDG', () => { + it.each([ + ['leading', '\tFunction:src/hot.ts:callee'], + ['interior', 'Function:src/hot.ts:a\t\tFunction:src/hot.ts:b'], + ['trailing', 'Function:src/hot.ts:callee\t'], + ['sentinel-adjacent', `Function:src/hot.ts:callee\t${CALLEES_TRUNCATED_SENTINEL}\t`], + ])( + 'rejects an empty callee identity before interprocedural descent (%s)', + async (_case, cell) => { + const seed = 'BasicBlock:src/hot.ts:1:0:0'; + const queries: string[] = []; + const exec: RunPdgImpactDeps['executeParameterized'] = async (_repo, query) => { + queries.push(query); + if (query.includes('MATCH (a:BasicBlock) WHERE')) return [{ id: seed }]; + if (query.includes('RETURN b.id AS id, b.calleeIds AS calleeIds')) { + return [{ id: seed, calleeIds: cell, callees: 'callee' }]; + } + return []; + }; + + await expect( + runImpactPDG({ + repo: { lbugPath: 'repo' }, + sym: { + id: 'Function:src/hot.ts:hot', + name: 'hot', + filePath: 'src/hot.ts', + startLine: 0, + endLine: 3, + }, + symType: 'Function', + direction: 'downstream', + maxDepth: 2, + limit: 50, + line: 1, + executeParameterized: exec, + }), + ).rejects.toBeInstanceOf(SymbolIdentityError); + expect( + queries.some((query) => query.includes('RETURN b.id AS id, b.calleeIds AS calleeIds')), + ).toBe(true); + expect(queries.some((query) => query.includes("r.type = 'CALL_SUMMARY'"))).toBe(false); + }, + ); + it('clamps huge maxDepth values to the documented impact traversal cap', async () => { let bfsQueries = 0; const exec = async (_repo: string, query: string) => { diff --git a/gitnexus/test/unit/query-degraded-signal.test.ts b/gitnexus/test/unit/query-degraded-signal.test.ts index a7ed807e8..f4c9e763c 100644 --- a/gitnexus/test/unit/query-degraded-signal.test.ts +++ b/gitnexus/test/unit/query-degraded-signal.test.ts @@ -42,6 +42,7 @@ vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => { }); import { LocalBackend } from '../../src/mcp/local/local-backend'; +import { resetExtensionState } from '../../src/core/lbug/extension-loader.js'; // A backend whose hybrid search yields exactly one matched symbol, so the // enrichment chunk loop runs and can be made to fail. `ftsUsed` is parameterized @@ -415,3 +416,100 @@ describe('query: degraded-enrichment signal', () => { } }); }); + +describe('query: partial missing FTS indexes', () => { + beforeEach(() => { + vi.clearAllMocks(); + loadMetaMock.mockResolvedValue(null); + executeParameterizedMock.mockResolvedValue([]); + }); + afterEach(() => vi.unstubAllEnvs()); + + it('returns successful symbols with a repair hint and partial flag when another index is missing', async () => { + const b = makeBackend(true) as any; + const resultWithMissing = await b.backend.bm25Search(); + b.backend.bm25Search.mockResolvedValue({ + ...resultWithMissing, + missingIndexes: ['Function.function_fts'], + }); + + const result = await runQuery(b); + + expect(result.definitions.map((d: any) => d.id)).toContain('func:x'); + expect(result.partial).toBe(true); + expect(result.warning).toContain('Function.function_fts'); + expect(result.warning).toContain('repair-fts'); + }); + + it('composes missing indexes with real FTS errors and enrichment failures', async () => { + const b = makeBackend(true, ['connection reset']) as any; + const resultWithMissing = await b.backend.bm25Search(); + b.backend.bm25Search.mockResolvedValue({ + ...resultWithMissing, + missingIndexes: ['Function.function_fts'], + }); + executeParameterizedMock.mockImplementation(async (_repo: string, cypher: string) => { + if (cypher.includes('STEP_IN_PROCESS')) throw new Error('timed out'); + return []; + }); + + const result = await runQuery(b); + + expect(result.partial).toBe(true); + expect(result.warning).toContain('Function.function_fts'); + expect(result.warning).toContain('FTS keyword search partially failed'); + expect(result.warning).toContain('enrichment'); + }); +}); + +it('propagates missing-index diagnostics through the real bm25Search helper into query', async () => { + vi.clearAllMocks(); + loadMetaMock.mockResolvedValue(null); + executeParameterizedMock.mockResolvedValue([]); + const search = await import('../../src/core/search/bm25-index.js'); + const spy = vi.spyOn(search, 'searchFTSFromLbug').mockResolvedValue({ + results: [], + ftsAvailable: true, + missingIndexes: ['Function.function_fts'], + }); + try { + const b = makeBackend(true) as any; + b.backend.bm25Search = (LocalBackend.prototype as any).bm25Search; + const result = await runQuery(b); + expect(result.partial).toBe(true); + expect(result.warning).toContain('Function.function_fts'); + } finally { + spy.mockRestore(); + } +}); + +it('reports missing indexes and real errors when every FTS query fails through the search boundary', async () => { + vi.clearAllMocks(); + resetExtensionState(); + loadMetaMock.mockResolvedValue(null); + executeParameterizedMock.mockImplementation(async (_repo: string, cypher: string) => { + if (cypher.includes("QUERY_FTS_INDEX('Function'")) { + throw new Error( + "Binder exception: Table Function doesn't have an index with name function_fts.", + ); + } + if (cypher.includes('QUERY_FTS_INDEX')) { + throw new Error('connection reset at /home/alice/private/index.lbug'); + } + return []; + }); + const b = makeBackend(false) as any; + b.backend.bm25Search = (LocalBackend.prototype as any).bm25Search; + + const result = await runQuery(b); + + expect(result.definitions).toEqual([]); + expect(result.warning).toContain('FTS keyword search failed'); + expect(result.warning).toContain('connection reset at '); + expect(result.warning).toContain('Function.function_fts'); + expect(result.warning).toContain('gitnexus analyze --repair-fts'); + expect(result.warning).toContain('resolved: repo1'); + expect(result.warning).not.toContain('not a missing-index'); + expect(result.warning).not.toContain('partially failed'); + expect(JSON.stringify(result)).not.toContain('/home/alice'); +}); diff --git a/gitnexus/test/unit/query-fts-missing-index.test.ts b/gitnexus/test/unit/query-fts-missing-index.test.ts new file mode 100644 index 000000000..aa50a50bf --- /dev/null +++ b/gitnexus/test/unit/query-fts-missing-index.test.ts @@ -0,0 +1,114 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +const native = vi.hoisted(() => ({ + error: undefined as string | undefined, + params: [] as unknown[], +})); + +// Control the native statement result, while exercising the real adapter's +// connection lock, prepared-query path and missing-index error handling. +vi.mock('@ladybugdb/core', () => { + const emptyResult = { + getAll: async () => [], + close: async () => {}, + isSuccess: () => true, + getErrorMessage: async () => '', + }; + class Database { + async init() {} + async close() {} + } + class Connection { + async query() { + return emptyResult; + } + async prepare() { + return { + isSuccess: () => native.error === undefined, + getErrorMessage: async () => native.error ?? '', + }; + } + async execute(_statement: unknown, params: unknown) { + native.params.push(params); + return emptyResult; + } + async close() {} + } + const mod = { Database, Connection }; + return { ...mod, default: mod, lbug: mod }; +}); + +import { closeLbug, queryFTS, withLbugDb } from '../../src/core/lbug/lbug-adapter.js'; + +const MISSING = "Binder exception: Table Function doesn't have an index with name function_fts."; + +describe('queryFTS missing-index diagnostics', () => { + let dir: string; + let dbPath: string; + beforeEach(async () => { + native.error = undefined; + native.params = []; + dir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-test-')); + dbPath = path.join(dir, 'lbug'); + await fs.writeFile(dbPath, ''); + }); + afterEach(async () => { + await closeLbug(); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it('retains the existing empty-result default for a missing index', async () => { + await withLbugDb( + dbPath, + async () => { + native.error = MISSING; + expect(await queryFTS('Function', 'function_fts', 'auth')).toEqual([]); + }, + { readOnly: true, skipFts: true }, + ); + }); + + it('can propagate a missing index so search does not count it as a successful zero-match query', async () => { + await withLbugDb( + dbPath, + async () => { + native.error = MISSING; + await expect( + queryFTS('Function', 'function_fts', 'auth', 5, false, 'throw'), + ).rejects.toThrow(MISSING); + }, + { readOnly: true, skipFts: true }, + ); + }); + + it('preserves prepared query binding and successful empty results in diagnostic mode', async () => { + await withLbugDb( + dbPath, + async () => { + const query = "auth' DELETE n"; + expect(await queryFTS('Function', 'function_fts', query, 5, false, 'throw')).toEqual([]); + expect(native.params).toEqual([{ query }]); + }, + { readOnly: true, skipFts: true }, + ); + }); + + it('still propagates real failures in either missing-index mode', async () => { + await withLbugDb( + dbPath, + async () => { + native.error = 'Runtime exception: connection reset'; + await expect(queryFTS('Function', 'function_fts', 'auth')).rejects.toThrow( + 'connection reset', + ); + await expect( + queryFTS('Function', 'function_fts', 'auth', 5, false, 'throw'), + ).rejects.toThrow('connection reset'); + }, + { readOnly: true, skipFts: true }, + ); + }); +}); diff --git a/gitnexus/test/unit/review-agent-workflow.test.ts b/gitnexus/test/unit/review-agent-workflow.test.ts index a882ebc5f..415535b69 100644 --- a/gitnexus/test/unit/review-agent-workflow.test.ts +++ b/gitnexus/test/unit/review-agent-workflow.test.ts @@ -686,12 +686,12 @@ describe('gitnexus review-agent workflow security contract', () => { it('pins every third-party action and the GitNexus analyzer exactly', () => { const expectedPins = [ - 'actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0', + 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1', 'actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3', 'actions/setup-node@820762786026740c76f36085b0efc47a31fe5020', 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a', 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c', - 'anthropics/claude-code-action/base-action@3553f84341b92da26052e28acf1aa898f9511f32', + 'anthropics/claude-code-action/base-action@e0cf66d1d257526b5d07f141838c338921cb8455', ]; for (const pin of expectedPins) { @@ -714,7 +714,7 @@ describe('gitnexus review-agent workflow security contract', () => { expect(workflow).toContain('.github/scripts/npm-ci-retry.sh'); expect(workflow).not.toContain('--package-lock=false'); expect(workflow).toContain( - 'actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0', + 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1', ); expect(workflow).toContain( 'actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0', diff --git a/gitnexus/test/unit/run-analyze-fts-crash-marker.test.ts b/gitnexus/test/unit/run-analyze-fts-crash-marker.test.ts index 6997a0604..b372fb8dd 100644 --- a/gitnexus/test/unit/run-analyze-fts-crash-marker.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-crash-marker.test.ts @@ -113,7 +113,13 @@ const mockLbugAdapter = async () => { initLbug: vi.fn(async () => undefined), loadGraphToLbug: vi.fn(async () => undefined), getLbugStats: vi.fn(async () => ({ nodes: 1, edges: 0, communities: 0, processes: 0 })), - executeQuery: vi.fn(async () => []), + // The policy fixture has one stored File row. Publication now reconciles + // that identity instead of trusting the synthetic node count alone. + executeQuery: vi.fn(async (query: string) => + query.startsWith('MATCH (n:`File`) RETURN n.id AS id') + ? [{ id: 'file:src/a.ts', name: '', filePath: REL_FILE }] + : [], + ), executeWithReusedStatement: vi.fn(async () => []), closeLbug: vi.fn(async () => undefined), wipeLbugDbFiles: vi.fn(async () => undefined), @@ -1011,9 +1017,12 @@ describe('runFullAnalysis FTS crash marker', () => { ); it('treats a boundary checkpoint failure as best-effort on an in-place plan', async () => { - const checkpointOnce = vi.fn(async () => { - throw new Error('checkpoint rename failed'); - }); + // Only the earlier boundary is best-effort. Publication still requires + // its own successful checkpoint through the same policy helper. + const checkpointOnce = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error('checkpoint rename failed')) + .mockResolvedValue(true); let stamped: RepoMeta['incrementalInProgress']; vi.doMock('../../src/core/lbug/wal-checkpoint-driver.js', async (importActual) => ({ ...(await importActual()), @@ -1060,6 +1069,7 @@ describe('runFullAnalysis FTS crash marker', () => { { onProgress: () => {}, onLog: () => {} }, ); expect(result.ftsSkipped).not.toBe(true); + expect(checkpointOnce).toHaveBeenCalledTimes(2); expect(stamped).toMatchObject({ phase: FTS_DIRTY_PHASE, writePlan: 'in-place', diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 2f0304ef4..b60cd60f1 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -1,13 +1,18 @@ import { execSync } from 'child_process'; import fs from 'fs/promises'; -import { afterEach, describe, expect, it, vi, type Mock } from 'vitest'; +import { basename } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; import { getStoragePaths, loadMeta, saveMeta, type RepoMeta, } from '../../src/storage/repo-manager.js'; -import { EMBEDDING_DIMS, STALE_HASH_SENTINEL } from '../../src/core/lbug/schema.js'; +import { + EMBEDDING_DIMS, + STALE_HASH_SENTINEL, + SCHEMA_FINGERPRINT, +} from '../../src/core/lbug/schema.js'; import { getIndexIncompleteReasons } from '../../src/core/index-freshness.js'; import type { EmbeddingPipelineOptions, @@ -2215,10 +2220,11 @@ describe('runFullAnalysis embedding-checkpoint meta write (#2790)', () => { vi.unstubAllEnvs(); }); - it('preserves lastCommit / fileHashes / the dirty flag, and never restates a stale count', async () => { + it('keeps staging counts out of published metadata until the index is swapped', async () => { const STALE_COMMIT = '1111111111111111111111111111111111111111'; const STALE_HASHES = { 'src/app.ts': 'stale-hash' }; const LIVE_EMBEDDING_COUNT = 42; + vi.stubEnv('GITNEXUS_ATOMIC_WINDOWS_SWAP', '1'); vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ initLbug: vi.fn(async () => undefined), @@ -2315,15 +2321,14 @@ describe('runFullAnalysis embedding-checkpoint meta write (#2790)', () => { nodeIds: ['node-1', 'node-2'], }); snapshots.windowStart = await loadMeta(storagePath); - // Post-window checkpoint — this one MEASURED the live count. + // Post-window checkpoint measures staging, not the published DB. await pipelineOptions.onCheckpoint?.({ nodesProcessed: 2, totalNodes: 4, chunksProcessed: 4, }); snapshots.postWindow = await loadMeta(storagePath); - // Window 2 — the old code restated the PREVIOUS run's count here and - // clobbered the live figure the post-window save had just written. + // Window 2 must retain the published count too. await pipelineOptions.onCheckpointWindowStart?.({ nodesProcessed: 2, totalNodes: 4, @@ -2375,19 +2380,27 @@ describe('runFullAnalysis embedding-checkpoint meta write (#2790)', () => { }); expect(snapshots.windowStart?.lastCommit).not.toBe(currentCommit); - // ── Post-window: the one save that legitimately measured the count ── + // ── Post-window: the staged count is not published yet ───────────── expect(snapshots.postWindow).toMatchObject({ lastCommit: STALE_COMMIT, fileHashes: STALE_HASHES, incrementalInProgress: { phase: 'full-rebuild' }, - stats: { embeddings: LIVE_EMBEDDING_COUNT }, + stats: { embeddings: 7 }, + }); + expect(snapshots.postWindow?.embeddingCheckpoint?.recovery).toMatchObject({ + stagingFile: expect.stringMatching(/^lbug\.staging\.[a-f0-9-]+$/), + schemaFingerprint: SCHEMA_FINGERPRINT, + unsafeNodeIds: [], }); - // ── Window 2: no stale restatement over the measured figure ──────── + // ── Window 2: the published count remains unchanged ──────────────── expect(snapshots.secondWindow).toMatchObject({ lastCommit: STALE_COMMIT, - stats: { embeddings: LIVE_EMBEDDING_COUNT }, - embeddingCheckpoint: { pendingNodeIds: ['node-3', 'node-4'] }, + stats: { embeddings: 7 }, + embeddingCheckpoint: { + pendingNodeIds: ['node-3', 'node-4'], + recovery: { unsafeNodeIds: ['node-3', 'node-4'] }, + }, }); // Only the finalize write — after the index is published — advances @@ -2395,7 +2408,7 @@ describe('runFullAnalysis embedding-checkpoint meta write (#2790)', () => { const finalMeta = JSON.parse( await fs.readFile(`${storagePath}/meta.json`, 'utf-8'), ) as RepoMeta; - expect(finalMeta).toMatchObject({ lastCommit: currentCommit }); + expect(finalMeta).toMatchObject({ lastCommit: currentCommit, stats: { embeddings: 42 } }); expect(finalMeta.embeddingCheckpoint).toBeUndefined(); expect(finalMeta.incrementalInProgress).toBeUndefined(); } finally { @@ -2427,6 +2440,14 @@ describe('runFullAnalysis embedding-checkpoint meta write (#2790)', () => { * NEXT run does with it). */ describe('runFullAnalysis embedding-checkpoint resilience (#2790 review)', () => { + const actualPlatformDescriptor = Object.getOwnPropertyDescriptor(process, 'platform'); + + beforeEach(() => { + // These mocked checkpoint tests exercise the atomic rebuild used on POSIX. + // Select that same path on Windows; the in-place case below opts out. + vi.stubEnv('GITNEXUS_ATOMIC_WINDOWS_SWAP', '1'); + }); + const RESILIENCE_NODE_ID = 'Function:src/app.ts:handler:1'; const stubNode = { id: RESILIENCE_NODE_ID, @@ -2461,7 +2482,7 @@ describe('runFullAnalysis embedding-checkpoint resilience (#2790 review)', () => const mockResilienceHarness = ( controls: ResilienceControls, - ): { runEmbeddingPipeline: Mock; loadCachedEmbeddings: Mock } => { + ): { runEmbeddingPipeline: Mock; loadCachedEmbeddings: Mock; batchInsertEmbeddings: Mock } => { const loadCachedEmbeddings = vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [], @@ -2530,11 +2551,13 @@ describe('runFullAnalysis embedding-checkpoint resilience (#2790 review)', () => pipelineOptions: EmbeddingPipelineOptions, ): Promise => controls.pipeline(pipelineOptions), ); + const batchInsertEmbeddings = vi.fn(async () => undefined); vi.doMock('../../src/core/embeddings/embedding-pipeline.js', () => ({ runEmbeddingPipeline, + batchInsertEmbeddings, buildVectorIndex: vi.fn(async () => false), })); - return { runEmbeddingPipeline, loadCachedEmbeddings }; + return { runEmbeddingPipeline, loadCachedEmbeddings, batchInsertEmbeddings }; }; /** A checkpoint shaped exactly as `RepoMeta` declares it. */ @@ -2583,12 +2606,17 @@ describe('runFullAnalysis embedding-checkpoint resilience (#2790 review)', () => }; afterEach(() => { + if (actualPlatformDescriptor) { + Object.defineProperty(process, 'platform', actualPlatformDescriptor); + } vi.doUnmock('../../src/core/lbug/lbug-adapter.js'); vi.doUnmock('../../src/core/search/fts-indexes.js'); vi.doUnmock('../../src/core/ingestion/pipeline.js'); vi.doUnmock('../../src/storage/repo-manager.js'); vi.doUnmock('../../src/core/embeddings/embedding-identity.js'); vi.doUnmock('../../src/core/embeddings/embedding-pipeline.js'); + vi.doUnmock('../../src/core/embeddings/staged-embedding-recovery.js'); + vi.restoreAllMocks(); vi.resetModules(); vi.clearAllMocks(); vi.unstubAllEnvs(); @@ -2699,6 +2727,7 @@ describe('runFullAnalysis embedding-checkpoint resilience (#2790 review)', () => */ it('carries the mid-run count forward, not the run-start snapshot, so --force still loads the cache', async () => { const MID_RUN_COUNT = 12; + vi.stubEnv('GITNEXUS_ATOMIC_WINDOWS_SWAP', '1'); const tmpRepo = await createTempDir('gitnexus-2790r-latest-meta-'); try { const { storagePath } = getStoragePaths(tmpRepo.dbPath); @@ -3039,4 +3068,625 @@ describe('runFullAnalysis embedding-checkpoint resilience (#2790 review)', () => await tmpRepo.cleanup(); } }); + + const mockStagedFiles = async (): Promise => { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + vi.mocked(adapter.initLbug).mockImplementation(async (dbPath) => { + if (dbPath.includes('.staging.')) await fs.writeFile(dbPath, 'staged fixture'); + }); + vi.mocked(adapter.wipeLbugDbFiles).mockImplementation(async (dbPath) => { + await fs.rm(dbPath, { force: true }); + }); + }; + + it.each([ + { mode: 'staged', atomicSwap: '1', checkpointCount: 7 }, + { mode: 'in-place', atomicSwap: '0', checkpointCount: 42 }, + ])( + 'keeps Windows $mode checkpoint counts consistent with the live index', + async ({ mode, atomicSwap, checkpointCount }) => { + const tmpRepo = await createTempDir('gitnexus-2790-checkpoint-meta-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await seedMeta(storagePath, tmpRepo.dbPath, { stats: { nodes: 2, embeddings: 7 } }); + await fs.writeFile(lbugPath, 'published fixture'); + const snapshots: Array = []; + mockResilienceHarness({ + count: [{ cnt: 42 }], + pipeline: async (options) => { + await options.onCheckpointWindowStart?.({ + nodesProcessed: 0, + totalNodes: 3, + chunksProcessed: 0, + nodeIds: [RESILIENCE_NODE_ID], + }); + snapshots.push(await loadMeta(storagePath)); + await options.onCheckpoint?.({ nodesProcessed: 3, totalNodes: 3, chunksProcessed: 3 }); + snapshots.push(await loadMeta(storagePath)); + return cleanResult(); + }, + }); + await mockStagedFiles(); + // Load modules on the actual host before changing only the platform + // branch exercised by runFullAnalysis; all native DB work is mocked. + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + vi.stubEnv('GITNEXUS_ATOMIC_WINDOWS_SWAP', atomicSwap); + Object.defineProperty(process, 'platform', { value: 'win32', configurable: true }); + await runFullAnalysis( + tmpRepo.dbPath, + { force: true, embeddings: true, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + + expect(snapshots[0]?.stats?.embeddings).toBe(7); + expect(snapshots[1]?.stats?.embeddings).toBe(checkpointCount); + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + const buildPath = vi.mocked(adapter.initLbug).mock.calls.at(-1)?.[0]; + if (mode === 'staged') { + expect(buildPath).toMatch(/\.staging\.[a-f0-9-]+$/); + expect(snapshots[1]?.embeddingCheckpoint?.recovery).toMatchObject({ + stagingFile: expect.stringMatching(/^lbug\.staging\.[a-f0-9-]+$/), + unsafeNodeIds: [], + }); + expect(await fs.readFile(lbugPath, 'utf8')).toBe('staged fixture'); + } else { + expect(buildPath).toBe(lbugPath); + expect(snapshots[0]?.embeddingCheckpoint?.recovery).toBeUndefined(); + expect(snapshots[1]?.embeddingCheckpoint?.recovery).toBeUndefined(); + } + const finalMeta = await loadMeta(storagePath); + expect(finalMeta?.stats?.embeddings).toBe(42); + expect(finalMeta?.embeddingCheckpoint).toBeUndefined(); + } finally { + if (actualPlatformDescriptor) { + Object.defineProperty(process, 'platform', actualPlatformDescriptor); + } + await tmpRepo.cleanup(); + } + }, + ); + + it.each(['close', 'rename'] as const)( + 'keeps the published count and database when %s fails before the staged publish', + async (failure) => { + const tmpRepo = await createTempDir('gitnexus-2790-checkpoint-meta-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await seedMeta(storagePath, tmpRepo.dbPath, { stats: { nodes: 2, embeddings: 7 } }); + await fs.writeFile(lbugPath, 'published fixture'); + const rename = fs.rename.bind(fs); + let checkpointMeta: RepoMeta | null = null; + let failedPublishRename: Mock | undefined; + mockResilienceHarness({ + count: [{ cnt: 42 }], + pipeline: async (options) => { + await options.onCheckpointWindowStart?.({ + nodesProcessed: 0, + totalNodes: 3, + chunksProcessed: 0, + nodeIds: [RESILIENCE_NODE_ID], + }); + await options.onCheckpoint?.({ nodesProcessed: 3, totalNodes: 3, chunksProcessed: 3 }); + checkpointMeta = await loadMeta(storagePath); + if (failure === 'close') { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + vi.mocked(adapter.closeLbug).mockRejectedValueOnce( + new Error('pre-publish close failed'), + ); + } else { + failedPublishRename = vi.fn(async () => { + throw Object.assign(new Error('staged publish rename failed'), { code: 'EIO' }); + }); + vi.spyOn(fs, 'rename').mockImplementation(async (source, destination) => { + if ( + String(source).startsWith(`${lbugPath}.staging.`) && + String(destination) === lbugPath + ) { + return failedPublishRename?.(); + } + return rename(source, destination); + }); + } + return cleanResult(); + }, + }); + await mockStagedFiles(); + expect( + await runAnalyze( + tmpRepo.dbPath, + { force: true, embeddings: true, skipAgentsMd: true, skipSkills: true }, + [], + ), + ).toMatchObject({ + message: + failure === 'close' ? 'pre-publish close failed' : 'staged publish rename failed', + }); + expect(checkpointMeta?.stats?.embeddings).toBe(7); + expect((await loadMeta(storagePath))?.stats?.embeddings).toBe(7); + expect(await fs.readFile(lbugPath, 'utf8')).toBe('published fixture'); + const recovery = (await loadMeta(storagePath))?.embeddingCheckpoint?.recovery; + if (!recovery) throw new Error('expected durable stage after failed publish'); + expect(await fs.readFile(`${storagePath}/${recovery.stagingFile}`, 'utf8')).toBe( + 'staged fixture', + ); + if (failure === 'rename') expect(failedPublishRename).toHaveBeenCalledTimes(1); + } finally { + vi.restoreAllMocks(); + await tmpRepo.cleanup(); + } + }, + ); + + const seedRecovery = async (storagePath: string, repoPath: string) => { + const stagingFile = 'lbug.staging.11111111-1111-4111-8111-111111111111'; + const checkpoint = checkpointFixture({ + kind: 'interrupted', + pendingNodeIds: [], + recovery: { stagingFile, schemaFingerprint: SCHEMA_FINGERPRINT, unsafeNodeIds: [] }, + }); + await seedMeta(storagePath, repoPath, { + stats: { nodes: 2, embeddings: 7 }, + embeddingCheckpoint: checkpoint, + }); + await fs.writeFile(`${storagePath}/${stagingFile}`, 'previous durable source'); + vi.doMock('../../src/core/embeddings/staged-embedding-recovery.js', () => ({ + recoverStagedEmbeddings: vi.fn(async () => ({ rows: [], embeddingNodeIds: new Set() })), + })); + return { checkpoint, stagingFile }; + }; + + it.each( + ['1', '0'].flatMap((manualCheckpoint) => + ( + ['window-start crash', 'post-window crash', 'final-metadata failure', 'success'] as const + ).map((outcome) => ({ manualCheckpoint, outcome })), + ), + )( + 'preserves the in-place recovery receipt with manual checkpoints=$manualCheckpoint through $outcome', + async ({ manualCheckpoint, outcome }) => { + vi.stubEnv('GITNEXUS_WAL_MANUAL_CHECKPOINT', manualCheckpoint); + vi.stubEnv('GITNEXUS_INDEX_LOCK_BACKEND', 'file'); + const tmpRepo = await createTempDir('gitnexus-3456-opt-out-source-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + const { checkpoint, stagingFile } = await seedRecovery(storagePath, tmpRepo.dbPath); + const original = { + ...checkpoint, + pendingNodeIds: ['original-pending'], + recovery: { + stagingFile, + schemaFingerprint: SCHEMA_FINGERPRINT, + unsafeNodeIds: ['original-pending', 'original-unsafe'], + }, + } satisfies NonNullable; + await seedMeta(storagePath, tmpRepo.dbPath, { + stats: { nodes: 2, embeddings: 7 }, + embeddingCheckpoint: original, + }); + const sourceFiles = [ + { filename: stagingFile, contents: 'previous durable source' }, + { filename: `${stagingFile}.wal`, contents: 'previous durable WAL' }, + { filename: `${stagingFile}.shadow`, contents: 'previous durable shadow' }, + ]; + for (const source of sourceFiles) { + await fs.writeFile(`${storagePath}/${source.filename}`, source.contents); + } + await fs.writeFile(lbugPath, 'previous published index'); + const { normalizeCachedEmbeddings } = + await import('../../src/core/embeddings/embedding-restore-spill.js'); + vi.doMock( + '../../src/core/embeddings/staged-embedding-recovery.js', + async (importActual) => ({ + ...(await importActual< + typeof import('../../src/core/embeddings/staged-embedding-recovery.js') + >()), + recoverStagedEmbeddings: vi.fn(async () => + normalizeCachedEmbeddings({ + embeddings: [ + { + nodeId: RESILIENCE_NODE_ID, + chunkIndex: 0, + startLine: 1, + endLine: 2, + contentHash: 'current-hash', + embedding: new Array(EMBEDDING_DIMS).fill(0), + }, + ], + }), + ), + }), + ); + const snapshots: Array = []; + const rename = fs.rename.bind(fs); + const { batchInsertEmbeddings } = mockResilienceHarness({ + count: [{ cnt: 9 }], + pipeline: async (options) => { + await options.onCheckpointWindowStart?.({ + nodesProcessed: 0, + totalNodes: 3, + chunksProcessed: 0, + nodeIds: ['current-window'], + }); + snapshots.push(await loadMeta(storagePath)); + if (outcome === 'window-start crash') throw new Error(outcome); + await options.onCheckpoint?.({ nodesProcessed: 3, totalNodes: 3, chunksProcessed: 9 }); + snapshots.push(await loadMeta(storagePath)); + if (outcome === 'post-window crash') throw new Error(outcome); + if (outcome === 'final-metadata failure') { + vi.spyOn(fs, 'rename').mockImplementation(async (source, destination) => { + if (basename(String(destination)) === 'gitnexus.json') throw new Error(outcome); + return rename(source, destination); + }); + } + return cleanResult(); + }, + }); + await mockStagedFiles(); + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + vi.mocked(adapter.loadGraphToLbug).mockImplementation(async () => { + await fs.writeFile(lbugPath, 'in-place replacement'); + }); + // Import on the host first, then choose the Windows default in-place + // branch. The native adapter is mocked; source files and lock cleanup are real. + await import('../../src/core/run-analyze.js'); + vi.stubEnv('GITNEXUS_ATOMIC_WINDOWS_SWAP', '0'); + Object.defineProperty(process, 'platform', { value: 'win32', configurable: true }); + const error = await runAnalyze( + tmpRepo.dbPath, + { force: true, embeddings: true, skipAgentsMd: true, skipSkills: true }, + [], + ); + if (actualPlatformDescriptor) { + Object.defineProperty(process, 'platform', actualPlatformDescriptor); + } + expect(batchInsertEmbeddings).toHaveBeenCalled(); + expect(vi.mocked(adapter.initLbug).mock.calls.at(-1)?.[0]).toBe(lbugPath); + const finalMeta = await loadMeta(storagePath); + // Reacquiring the real lock performs the next retry's orphan sweep. + // A lost receipt would delete every byte of the proven old generation here. + const { acquireIndexLock } = await import('../../src/storage/index-lock.js'); + const lock = await acquireIndexLock(storagePath, { timeoutMs: 1000 }); + try { + if (outcome === 'success') { + expect(error).toBeNull(); + expect(finalMeta?.embeddingCheckpoint).toBeUndefined(); + expect(finalMeta?.stats?.embeddings).toBe(9); + expect(await fs.readFile(lbugPath, 'utf8')).toBe('in-place replacement'); + for (const source of sourceFiles) { + await expect(fs.stat(`${storagePath}/${source.filename}`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + } + } else { + expect(error).toMatchObject({ message: outcome }); + for (const source of sourceFiles) { + expect(await fs.readFile(`${storagePath}/${source.filename}`, 'utf8')).toBe( + source.contents, + ); + } + expect(finalMeta?.embeddingCheckpoint).toEqual(original); + expect(finalMeta?.stats?.embeddings).toBe(outcome === 'window-start crash' ? 7 : 9); + } + expect(snapshots[0]?.embeddingCheckpoint).toEqual(original); + expect(snapshots[0]?.stats?.embeddings).toBe(7); + if (outcome !== 'window-start crash') { + expect(snapshots[1]?.embeddingCheckpoint).toEqual(original); + expect(snapshots[1]?.stats?.embeddings).toBe(9); + } + } finally { + lock.release(); + } + } finally { + if (actualPlatformDescriptor) { + Object.defineProperty(process, 'platform', actualPlatformDescriptor); + } + vi.restoreAllMocks(); + await tmpRepo.cleanup(); + } + }, + ); + + it('finishes a staged embedding run with manual checkpoints disabled', async () => { + vi.stubEnv('GITNEXUS_WAL_MANUAL_CHECKPOINT', '0'); + const tmpRepo = await createTempDir('gitnexus-3456-checkpoint-opt-out-'); + try { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + await seedMeta(storagePath, tmpRepo.dbPath, { stats: { embeddings: 7 } }); + mockResilienceHarness({ + count: [{ cnt: 9 }], + pipeline: async (options) => { + await options.onCheckpointWindowStart?.({ + nodesProcessed: 0, + totalNodes: 3, + chunksProcessed: 0, + nodeIds: ['active-node'], + }); + expect((await loadMeta(storagePath))?.embeddingCheckpoint?.recovery).toBeUndefined(); + await options.onCheckpoint?.({ nodesProcessed: 3, totalNodes: 3, chunksProcessed: 9 }); + const midRun = await loadMeta(storagePath); + expect(midRun?.embeddingCheckpoint?.recovery).toBeUndefined(); + expect(midRun?.stats?.embeddings).toBe(7); + return cleanResult(); + }, + }); + await mockStagedFiles(); + expect(await runAnalyze(tmpRepo.dbPath, { force: true, embeddings: true }, [])).toBeNull(); + expect((await loadMeta(storagePath))?.embeddingCheckpoint).toBeUndefined(); + expect((await loadMeta(storagePath))?.stats?.embeddings).toBe(9); + expect(await fs.readFile(getStoragePaths(tmpRepo.dbPath).lbugPath, 'utf8')).toBe( + 'staged fixture', + ); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('keeps the previous durable source when manual checkpoints are disabled', async () => { + vi.stubEnv('GITNEXUS_WAL_MANUAL_CHECKPOINT', '0'); + const tmpRepo = await createTempDir('gitnexus-3456-opt-out-source-'); + try { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + const { checkpoint: original, stagingFile } = await seedRecovery(storagePath, tmpRepo.dbPath); + mockResilienceHarness({ + count: [{ cnt: 9 }], + pipeline: async (options) => { + await options.onCheckpointWindowStart?.({ + nodesProcessed: 0, + totalNodes: 3, + chunksProcessed: 0, + nodeIds: ['active-node'], + }); + await options.onCheckpoint?.({ nodesProcessed: 3, totalNodes: 3, chunksProcessed: 9 }); + expect((await loadMeta(storagePath))?.embeddingCheckpoint).toEqual(original); + throw new Error('endpoint failure with manual checkpoints disabled'); + }, + }); + await mockStagedFiles(); + expect(await runAnalyze(tmpRepo.dbPath, { force: true, embeddings: true }, [])).toMatchObject( + { message: 'endpoint failure with manual checkpoints disabled' }, + ); + expect((await loadMeta(storagePath))?.embeddingCheckpoint).toEqual(original); + expect(await fs.readFile(`${storagePath}/${stagingFile}`, 'utf8')).toBe( + 'previous durable source', + ); + expect( + (await fs.readdir(storagePath)).filter((name) => name.startsWith('lbug.staging.')), + ).toEqual([stagingFile]); + } finally { + await tmpRepo.cleanup(); + } + }); + + it.skipIf(process.platform === 'win32')( + 'retains the referenced stage through a symlinked storage directory', + async () => { + const tmpRepo = await createTempDir('gitnexus-3456-storage-alias-'); + try { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + const actualStorage = `${tmpRepo.dbPath}/actual-index`; + await fs.mkdir(actualStorage); + await fs.symlink(actualStorage, storagePath, 'dir'); + await seedMeta(storagePath, tmpRepo.dbPath, { stats: { embeddings: 7 } }); + mockResilienceHarness({ + count: [{ cnt: 9 }], + pipeline: async (options) => { + await options.onCheckpointWindowStart?.({ + nodesProcessed: 0, + totalNodes: 3, + chunksProcessed: 0, + nodeIds: ['active-node'], + }); + await options.onCheckpoint?.({ nodesProcessed: 1, totalNodes: 3, chunksProcessed: 9 }); + throw new Error('endpoint failed after durable window'); + }, + }); + await mockStagedFiles(); + expect( + await runAnalyze(tmpRepo.dbPath, { force: true, embeddings: true }, []), + ).toMatchObject({ message: 'endpoint failed after durable window' }); + const recovery = (await loadMeta(storagePath))?.embeddingCheckpoint?.recovery; + if (!recovery) throw new Error('expected retained recovery generation'); + expect(await fs.readFile(`${actualStorage}/${recovery.stagingFile}`, 'utf8')).toBe( + 'staged fixture', + ); + expect((await loadMeta(storagePath))?.stats?.embeddings).toBe(7); + } finally { + await tmpRepo.cleanup(); + } + }, + ); + + it.each(['checkpoint', 'metadata'] as const)( + 'keeps the previous source when %s fails before recovery handoff', + async (failure) => { + const tmpRepo = await createTempDir('gitnexus-3456-handoff-failure-'); + try { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + const { checkpoint: original, stagingFile } = await seedRecovery( + storagePath, + tmpRepo.dbPath, + ); + const rename = fs.rename.bind(fs); + mockResilienceHarness({ + count: [{ cnt: 9 }], + pipeline: async (options) => { + if (failure === 'metadata') { + vi.spyOn(fs, 'rename').mockImplementation(async (source, destination) => { + if (basename(String(destination)) === 'gitnexus.json') + throw new Error('metadata write failed'); + return rename(source, destination); + }); + } else { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + vi.mocked(adapter.tryFlushWAL).mockResolvedValue(false); + } + await options.onCheckpointWindowStart?.({ + nodesProcessed: 0, + totalNodes: 3, + chunksProcessed: 0, + nodeIds: ['active-node'], + }); + return cleanResult(); + }, + }); + await mockStagedFiles(); + const error = await runAnalyze(tmpRepo.dbPath, { force: true, embeddings: true }, []); + expect(error).toMatchObject({ + message: + failure === 'metadata' + ? 'metadata write failed' + : 'Could not checkpoint restored embeddings before recovery handoff.', + }); + expect((await loadMeta(storagePath))?.embeddingCheckpoint).toEqual(original); + expect((await loadMeta(storagePath))?.stats?.embeddings).toBe(7); + expect(await fs.readFile(`${storagePath}/${stagingFile}`, 'utf8')).toBe( + 'previous durable source', + ); + expect( + (await fs.readdir(storagePath)).filter((name) => name.startsWith('lbug.staging.')), + ).toEqual([stagingFile]); + } finally { + vi.restoreAllMocks(); + await tmpRepo.cleanup(); + } + }, + ); + + it('keeps an active window unsafe when its completion checkpoint fails', async () => { + const tmpRepo = await createTempDir('gitnexus-3456-completion-failure-'); + try { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + await seedMeta(storagePath, tmpRepo.dbPath, { stats: { embeddings: 7 } }); + mockResilienceHarness({ + count: [{ cnt: 9 }], + pipeline: async (options) => { + await options.onCheckpointWindowStart?.({ + nodesProcessed: 0, + totalNodes: 3, + chunksProcessed: 0, + nodeIds: ['active-node'], + }); + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + vi.mocked(adapter.tryFlushWAL).mockResolvedValue(false); + await options.onCheckpoint?.({ nodesProcessed: 1, totalNodes: 3, chunksProcessed: 9 }); + return cleanResult(); + }, + }); + await mockStagedFiles(); + expect(await runAnalyze(tmpRepo.dbPath, { force: true, embeddings: true }, [])).toMatchObject( + { message: 'Could not checkpoint the completed embedding window for recovery.' }, + ); + const meta = await loadMeta(storagePath); + expect(meta?.stats?.embeddings).toBe(7); + expect(meta?.embeddingCheckpoint?.pendingNodeIds).toEqual(['active-node']); + expect(meta?.embeddingCheckpoint?.recovery?.unsafeNodeIds).toEqual(['active-node']); + if (!meta?.embeddingCheckpoint?.recovery) throw new Error('expected retained active window'); + expect( + await fs.readFile( + `${storagePath}/${meta.embeddingCheckpoint.recovery.stagingFile}`, + 'utf8', + ), + ).toBe('staged fixture'); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('retains a failed stage and keeps future incomplete restore groups unsafe', async () => { + const tmpRepo = await createTempDir('gitnexus-3456-unsafe-restore-'); + try { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + await seedMeta(storagePath, tmpRepo.dbPath, { stats: { nodes: 2, embeddings: 1 } }); + let completedWindow: RepoMeta | null = null; + const { loadCachedEmbeddings, batchInsertEmbeddings } = mockResilienceHarness({ + count: [{ cnt: 5 }], + pipeline: async (options) => { + await options.onCheckpointWindowStart?.({ + nodesProcessed: 0, + totalNodes: 2, + chunksProcessed: 0, + nodeIds: ['earlier-window-node'], + }); + await options.onCheckpoint?.({ nodesProcessed: 1, totalNodes: 2, chunksProcessed: 1 }); + completedWindow = await loadMeta(storagePath); + throw new Error('Maximum database size exceeded'); + }, + }); + loadCachedEmbeddings.mockResolvedValue({ + embeddingNodeIds: new Set([RESILIENCE_NODE_ID]), + embeddings: [ + { + nodeId: RESILIENCE_NODE_ID, + chunkIndex: 0, + startLine: 1, + endLine: 2, + contentHash: 'current-hash', + embedding: new Array(EMBEDDING_DIMS).fill(0), + }, + ], + }); + batchInsertEmbeddings.mockRejectedValue(new Error('restore batch partially inserted')); + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + vi.mocked(adapter.initLbug).mockImplementation(async (dbPath) => { + if (dbPath.includes('.staging.')) await fs.writeFile(dbPath, 'staged fixture'); + }); + vi.mocked(adapter.wipeLbugDbFiles).mockImplementation(async (dbPath) => { + await fs.rm(dbPath, { force: true }); + }); + const logs: string[] = []; + expect( + await runAnalyze( + tmpRepo.dbPath, + { embeddings: true, force: true, skipAgentsMd: true, skipSkills: true }, + logs, + ), + ).toMatchObject({ message: 'Maximum database size exceeded' }); + expect(completedWindow?.embeddingCheckpoint?.recovery?.unsafeNodeIds).toEqual([ + RESILIENCE_NODE_ID, + ]); + expect(completedWindow?.stats?.embeddings).toBe(1); + const recovery = (await loadMeta(storagePath))?.embeddingCheckpoint?.recovery; + if (!recovery) throw new Error('expected retained recovery generation'); + expect(await fs.readFile(`${storagePath}/${recovery.stagingFile}`, 'utf8')).toBe( + 'staged fixture', + ); + expect(logs).toContainEqual(expect.stringContaining('GITNEXUS_LBUG_MAX_DB_SIZE')); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('reclaims a failed current stage before any recovery checkpoint exists', async () => { + const tmpRepo = await createTempDir('gitnexus-3456-no-checkpoint-'); + try { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + await seedMeta(storagePath, tmpRepo.dbPath, {}); + mockResilienceHarness({ + count: [{ cnt: 0 }], + pipeline: async () => { + throw new Error('failed before first window'); + }, + }); + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + vi.mocked(adapter.initLbug).mockImplementation(async (dbPath) => { + if (dbPath.includes('.staging.')) await fs.writeFile(dbPath, 'staged fixture'); + }); + vi.mocked(adapter.wipeLbugDbFiles).mockImplementation(async (dbPath) => { + await fs.rm(dbPath, { force: true }); + }); + expect( + await runAnalyze( + tmpRepo.dbPath, + { embeddings: true, force: true, skipAgentsMd: true, skipSkills: true }, + [], + ), + ).toMatchObject({ message: 'failed before first window' }); + expect( + (await fs.readdir(storagePath)).filter((name) => name.startsWith('lbug.staging.')), + ).toEqual([]); + expect((await loadMeta(storagePath))?.embeddingCheckpoint).toBeUndefined(); + } finally { + await tmpRepo.cleanup(); + } + }); }); diff --git a/gitnexus/test/unit/scope-resolution/swift/callable-visibility.test.ts b/gitnexus/test/unit/scope-resolution/swift/callable-visibility.test.ts new file mode 100644 index 000000000..7cc8daed5 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/swift/callable-visibility.test.ts @@ -0,0 +1,267 @@ +import { + buildDefIndex, + buildMethodDispatchIndex, + buildQualifiedNameIndex, + buildScopeTree, + type Scope, + type SymbolDefinition, +} from 'gitnexus-shared'; +import { describe, expect, it } from 'vitest'; +import { swiftIsCallableVisibleFromCaller } from '../../../../src/core/ingestion/languages/swift/callable-visibility.js'; +import type { ScopeResolutionIndexes } from '../../../../src/core/ingestion/model/scope-resolution-indexes.js'; + +const filePath = 'Service.swift'; +const moduleRange = { startLine: 1, startCol: 0, endLine: 12, endCol: 0 }; +const classRange = { startLine: 2, startCol: 0, endLine: 10, endCol: 0 }; +const functionRange = { startLine: 5, startCol: 0, endLine: 8, endCol: 0 }; + +function scope( + id: string, + parent: string | null, + kind: Scope['kind'], + ownedDefs: SymbolDefinition[], + range: Scope['range'], +): Scope { + return { + id, + parent, + kind, + range, + filePath, + ownedDefs, + bindings: new Map(), + imports: [], + typeBindings: new Map(), + }; +} + +const classDef: SymbolDefinition = { + nodeId: 'Service', + filePath, + type: 'Class', + qualifiedName: 'Service', +}; +const property: SymbolDefinition = { + nodeId: 'Service.clock', + filePath, + type: 'Property', + qualifiedName: 'Service.clock', + ownerId: 'Service', +}; +const method: SymbolDefinition = { + nodeId: 'Service.refresh', + filePath, + type: 'Method', + qualifiedName: 'Service.refresh', + ownerId: 'Service', +}; +const scopes = { + scopeTree: buildScopeTree([ + scope('module', null, 'Module', [], moduleRange), + scope('class', 'module', 'Class', [classDef, property], classRange), + scope('function', 'class', 'Function', [method], functionRange), + ]), + defs: buildDefIndex([classDef, property, method]), + qualifiedNames: buildQualifiedNameIndex([classDef, property, method]), + methodDispatch: buildMethodDispatchIndex({ + owners: [classDef.nodeId], + computeMro: () => [], + implementsOf: () => [], + }), +} as ScopeResolutionIndexes; + +describe('Swift caller-side callable visibility', () => { + it('rejects an unrelated same-name method shadowed by a stored property', () => { + expect( + swiftIsCallableVisibleFromCaller({ + candidate: { + nodeId: 'Other.clock', + filePath: 'Other.swift', + type: 'Method', + qualifiedName: 'Other.clock', + }, + callerScope: 'function', + callArity: 0, + scopes, + }), + ).toBe(false); + }); + + it('keeps differently named methods and unknown caller scopes eligible', () => { + const candidate: SymbolDefinition = { + nodeId: 'Other.run', + filePath: 'Other.swift', + type: 'Method', + qualifiedName: 'Other.run', + }; + expect( + swiftIsCallableVisibleFromCaller({ + candidate, + callerScope: 'function', + callArity: 0, + scopes, + }), + ).toBe(true); + expect( + swiftIsCallableVisibleFromCaller({ + candidate: { ...candidate, qualifiedName: 'Other.clock' }, + }), + ).toBe(true); + }); + + it('preserves a selected local function before checking the enclosing property', () => { + const local: SymbolDefinition = { + nodeId: 'local.clock', + filePath, + type: 'Function', + qualifiedName: 'Service.refresh.clock', + }; + const localScopes = { + ...scopes, + scopeTree: buildScopeTree([ + scope('module', null, 'Module', [], moduleRange), + scope('class', 'module', 'Class', [classDef, property], classRange), + scope('function', 'class', 'Function', [method], functionRange), + scope('local', 'function', 'Function', [local], { + startLine: 6, + startCol: 0, + endLine: 7, + endCol: 0, + }), + ]), + } as ScopeResolutionIndexes; + expect( + swiftIsCallableVisibleFromCaller({ + candidate: local, + callerParsed: { + filePath, + moduleScope: 'module', + scopes: [...localScopes.scopeTree.byId.values()], + localDefs: [classDef, property, method, local], + parsedImports: [], + referenceSites: [], + }, + callerScope: 'function', + callArity: 0, + scopes: localScopes, + }), + ).toBe(true); + }); + + it('preserves a selected method owned by the current type', () => { + expect( + swiftIsCallableVisibleFromCaller({ + candidate: { ...method, nodeId: 'Service.clock', qualifiedName: 'Service.clock' }, + callerScope: 'function', + callArity: 0, + scopes, + }), + ).toBe(true); + }); + + it('does not veto a call with arguments or unknown arity', () => { + const candidate: SymbolDefinition = { + nodeId: 'Other.clock', + filePath: 'Other.swift', + type: 'Method', + qualifiedName: 'Other.clock', + }; + expect( + swiftIsCallableVisibleFromCaller({ + candidate, + callerScope: 'function', + callArity: 1, + scopes, + }), + ).toBe(true); + expect(swiftIsCallableVisibleFromCaller({ candidate, callerScope: 'function', scopes })).toBe( + true, + ); + }); + + it('rejects a decoy when a superclass owns the closure property', () => { + const base: SymbolDefinition = { + nodeId: 'BaseService', + filePath, + type: 'Class', + qualifiedName: 'BaseService', + }; + const derived: SymbolDefinition = { + nodeId: 'DerivedService', + filePath, + type: 'Class', + qualifiedName: 'DerivedService', + }; + const inheritedProperty: SymbolDefinition = { + nodeId: 'BaseService.clock', + filePath, + type: 'Property', + qualifiedName: 'BaseService.clock', + ownerId: base.nodeId, + }; + const inheritedScopes = { + scopeTree: buildScopeTree([ + scope('derivedModule', null, 'Module', [], moduleRange), + scope('derivedClass', 'derivedModule', 'Class', [derived], classRange), + scope('derivedFunction', 'derivedClass', 'Function', [method], functionRange), + ]), + defs: buildDefIndex([base, derived, inheritedProperty]), + qualifiedNames: buildQualifiedNameIndex([base, derived, inheritedProperty]), + methodDispatch: buildMethodDispatchIndex({ + owners: [derived.nodeId], + computeMro: () => [base.nodeId], + implementsOf: () => [], + }), + } as ScopeResolutionIndexes; + expect( + swiftIsCallableVisibleFromCaller({ + candidate: { + nodeId: 'Other.clock', + filePath: 'Other.swift', + type: 'Method', + qualifiedName: 'Other.clock', + }, + callerScope: 'derivedFunction', + callArity: 0, + scopes: inheritedScopes, + }), + ).toBe(false); + + const extensionScopes = { + ...inheritedScopes, + bindings: new Map([ + [ + 'extensionModule', + new Map([['DerivedService', [{ def: derived, origin: 'local' as const }]]]), + ], + ]), + bindingAugmentations: new Map(), + scopeTree: buildScopeTree([ + scope('extensionModule', null, 'Module', [], moduleRange), + scope('extensionClass', 'extensionModule', 'Class', [], classRange), + { + ...scope('extensionFunction', 'extensionClass', 'Function', [method], functionRange), + typeBindings: new Map([ + [ + 'self', + { rawName: 'DerivedService', declaredAtScope: 'extensionFunction', source: 'self' }, + ], + ]), + }, + ]), + } as ScopeResolutionIndexes; + expect( + swiftIsCallableVisibleFromCaller({ + candidate: { + nodeId: 'Other.clock', + filePath: 'Other.swift', + type: 'Method', + qualifiedName: 'Other.clock', + }, + callerScope: 'extensionFunction', + callArity: 0, + scopes: extensionScopes, + }), + ).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/skill-gen.test.ts b/gitnexus/test/unit/skill-gen.test.ts index 0ffb60ec0..57167ce20 100644 --- a/gitnexus/test/unit/skill-gen.test.ts +++ b/gitnexus/test/unit/skill-gen.test.ts @@ -22,6 +22,9 @@ import type { ProcessDetectionResult, } from '../../src/core/ingestion/process-processor.js'; import type { PipelineResult } from '../../src/types/pipeline.js'; +import { communitiesPhase } from '../../src/core/ingestion/pipeline-phases/communities.js'; +import { processesPhase } from '../../src/core/ingestion/pipeline-phases/processes.js'; +import type { PhaseResult } from '../../src/core/ingestion/pipeline-phases/types.js'; // ============================================================================ // FIXTURE HELPERS @@ -135,6 +138,25 @@ function buildPipelineResult(opts: { }; } +/** Run the real community phase, including graph node and membership edge emission. */ +async function detectCommunities(graph: KnowledgeGraph, repoPath: string): Promise { + const { communityResult } = await communitiesPhase.execute( + { graph, repoPath, onProgress: () => {}, pipelineStart: Date.now() }, + new Map([['structure', { phaseName: 'structure', output: { totalFiles: 0 }, durationMs: 0 }]]), + ); + return { + graph, + repoPath, + totalFileCount: 0, + communityResult, + resolutionOutcomes: [], + usedWorkerPool: false, + reparsedFileCount: 0, + scopeExtractionFailures: [], + unavailableScopeLanguageFiles: 0, + }; +} + // ============================================================================ // TESTS — RETURN VALUES // ============================================================================ @@ -407,6 +429,179 @@ describe('generateSkillFiles — return values', () => { expect(result.skills[0].label).toBe('Auth'); }); + it('generates a folder skill from real singleton assignments without dangling membership edges', async () => { + const graph = createKnowledgeGraph(); + graph.addNode(makeNode('file:target', 'target', 'File', `${tmpDir}/target.ts`, 1, false)); + for (const name of ['gamma', 'alpha', 'beta']) { + graph.addNode( + makeNode(`fn:${name}`, name, 'Function', `${tmpDir}/src/auth/${name}.ts`, 1, true), + ); + // The File target admits the symbol to the projection, then is excluded + // itself, leaving a singleton in the real Leiden result. + graph.addRelationship(makeRel(`rel:${name}`, `fn:${name}`, 'file:target', 'CALLS')); + } + + const pipeline = await detectCommunities(graph, tmpDir); + const result = await generateSkillFiles(tmpDir, 'TestProject', pipeline); + + expect(result.skills).toHaveLength(1); + expect(result.skills[0]).toMatchObject({ + name: 'gitnexus-area-auth', + label: 'Auth', + symbolCount: 3, + fileCount: 3, + }); + expect(pipeline.communityResult?.communities).toEqual([]); + expect(pipeline.communityResult?.memberships).toEqual([]); + expect(pipeline.communityResult?.rawMemberships).toEqual([ + { nodeId: 'fn:alpha', communityId: 'comm_0' }, + { nodeId: 'fn:beta', communityId: 'comm_1' }, + { nodeId: 'fn:gamma', communityId: 'comm_2' }, + ]); + expect([...graph.iterRelationships()].filter((rel) => rel.type === 'MEMBER_OF')).toEqual([]); + const content = await fs.readFile( + path.join(result.outputPath, result.skills[0].name, 'SKILL.md'), + 'utf-8', + ); + for (const name of ['alpha', 'beta', 'gamma']) { + expect(content).toContain(name); + expect(content).toContain(`src/auth/${name}.ts`); + } + }); + + it('preserves execution flows for real singleton fallback skills', async () => { + const graph = createKnowledgeGraph(); + // Large-graph projection prunes the degree-one endpoints of each chain, + // leaving only its middle function as a singleton community. + for (let i = 0; i < 10_001; i++) { + graph.addNode(makeNode(`fn:unused${i}`, `unused${i}`, 'Function', 'src/unused.ts', 1, false)); + } + for (let i = 0; i < 4; i++) { + const folder = i < 3 ? 'auth' : 'billing'; + for (const role of ['handle', 'middle', 'end']) { + const name = `${role}${i}`; + graph.addNode( + makeNode( + `fn:${name}`, + name, + 'Function', + `${tmpDir}/src/${folder}/${name}.ts`, + 1, + role === 'handle', + ), + ); + } + graph.addRelationship(makeRel(`rel:handle${i}`, `fn:handle${i}`, `fn:middle${i}`, 'CALLS')); + graph.addRelationship(makeRel(`rel:middle${i}`, `fn:middle${i}`, `fn:end${i}`, 'CALLS')); + } + + const pipeline = await detectCommunities(graph, tmpDir); + const { processResult } = await processesPhase.execute( + { graph, repoPath: tmpDir, onProgress: () => {}, pipelineStart: Date.now() }, + new Map>([ + ['structure', { phaseName: 'structure', output: { totalFiles: 0 }, durationMs: 0 }], + [ + 'communities', + { + phaseName: 'communities', + output: { communityResult: pipeline.communityResult }, + durationMs: 0, + }, + ], + ['routes', { phaseName: 'routes', output: { routeRegistry: new Map() }, durationMs: 0 }], + ['tools', { phaseName: 'tools', output: { toolDefs: [] }, durationMs: 0 }], + ]), + ); + pipeline.processResult = processResult; + + expect(pipeline.communityResult?.communities).toEqual([]); + expect(pipeline.communityResult?.memberships).toEqual([]); + expect(pipeline.communityResult?.rawMemberships).toHaveLength(4); + expect(processResult.processes).toHaveLength(4); + expect(processResult.processes.every((process) => process.communities.length === 0)).toBe(true); + expect([...graph.iterRelationships()].filter((rel) => rel.type === 'MEMBER_OF')).toEqual([]); + + const result = await generateSkillFiles(tmpDir, 'TestProject', pipeline); + expect(result.skills).toHaveLength(1); + expect(result.skills[0]).toMatchObject({ label: 'Auth', symbolCount: 3 }); + const content = await fs.readFile( + path.join(result.outputPath, result.skills[0].name, 'SKILL.md'), + 'utf-8', + ); + expect(content).toContain('## Execution Flows'); + for (const process of processResult.processes) { + if (process.entryPointId === 'fn:handle3') { + expect(content).not.toContain(process.heuristicLabel); + } else { + expect(content).toContain(process.heuristicLabel); + } + } + }); + + it.each([0, 2])('skips real singleton fallback below threshold (%i symbols)', async (count) => { + const graph = createKnowledgeGraph(); + if (count > 0) { + graph.addNode(makeNode('file:target', 'target', 'File', `${tmpDir}/target.ts`, 1, false)); + } + for (let i = 0; i < count; i++) { + graph.addNode( + makeNode(`fn:n${i}`, `n${i}`, 'Function', `${tmpDir}/src/auth/f${i}.ts`, 1, true), + ); + graph.addRelationship(makeRel(`rel:${i}`, `fn:n${i}`, 'file:target', 'CALLS')); + } + + const pipeline = await detectCommunities(graph, tmpDir); + const result = await generateSkillFiles(tmpDir, 'TestProject', pipeline); + + expect(result.skills).toEqual([]); + expect(pipeline.communityResult?.rawMemberships).toHaveLength(count); + expect(pipeline.communityResult?.memberships).toEqual([]); + expect([...graph.iterRelationships()].filter((rel) => rel.type === 'MEMBER_OF')).toEqual([]); + }); + + it('keeps real retained skills and membership edges separate from filtered singletons', async () => { + const graph = createKnowledgeGraph(); + for (const name of ['alpha', 'beta', 'gamma', 'singleton']) { + graph.addNode( + makeNode(`fn:${name}`, name, 'Function', `${tmpDir}/src/auth/${name}.ts`, 1, true), + ); + } + graph.addNode(makeNode('file:target', 'target', 'File', `${tmpDir}/target.ts`, 1, false)); + graph.addRelationship(makeRel('rel:ab', 'fn:alpha', 'fn:beta', 'CALLS')); + graph.addRelationship(makeRel('rel:bc', 'fn:beta', 'fn:gamma', 'CALLS')); + graph.addRelationship(makeRel('rel:ca', 'fn:gamma', 'fn:alpha', 'CALLS')); + graph.addRelationship(makeRel('rel:singleton', 'fn:singleton', 'file:target', 'CALLS')); + + const pipeline = await detectCommunities(graph, tmpDir); + const result = await generateSkillFiles(tmpDir, 'TestProject', pipeline); + + expect(result.skills).toHaveLength(1); + expect(result.skills[0]).toMatchObject({ label: 'Auth', symbolCount: 3, fileCount: 3 }); + expect(pipeline.communityResult?.rawMemberships).toHaveLength(4); + expect(pipeline.communityResult?.memberships.map((membership) => membership.nodeId)).toEqual([ + 'fn:alpha', + 'fn:beta', + 'fn:gamma', + ]); + const membershipEdges = [...graph.iterRelationships()].filter( + (rel) => rel.type === 'MEMBER_OF', + ); + expect(membershipEdges).toHaveLength(3); + for (const edge of membershipEdges) { + expect(graph.getNode(edge.sourceId)).toBeDefined(); + expect(graph.getNode(edge.targetId)?.label).toBe('Community'); + expect(edge.sourceId).not.toBe('fn:singleton'); + } + const content = await fs.readFile( + path.join(result.outputPath, result.skills[0].name, 'SKILL.md'), + 'utf-8', + ); + expect(content).not.toContain('singleton'); + for (const name of ['alpha', 'beta', 'gamma']) { + expect(content).toContain(`src/auth/${name}.ts`); + } + }); + /** * When processResult is undefined, the generator should still work * without crashing — it simply has no execution flows. diff --git a/gitnexus/test/unit/staged-embedding-recovery-child.test.ts b/gitnexus/test/unit/staged-embedding-recovery-child.test.ts new file mode 100644 index 000000000..58711493b --- /dev/null +++ b/gitnexus/test/unit/staged-embedding-recovery-child.test.ts @@ -0,0 +1,300 @@ +import fs from 'node:fs'; +import { EventEmitter } from 'node:events'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const h = vi.hoisted(() => ({ + dbCtor: vi.fn(), + connCtor: vi.fn(), + dbClose: vi.fn<() => Promise>(), + connClose: vi.fn<() => Promise>(), + query: vi.fn(), + abortBuilder: vi.fn(), + spawn: vi.fn(), +})); + +vi.mock('node:child_process', () => ({ spawn: h.spawn })); + +vi.mock('@ladybugdb/core', () => { + class Database { + constructor(...args: unknown[]) { + h.dbCtor(...args); + } + close = h.dbClose; + } + class Connection { + constructor(db: unknown) { + h.connCtor(db); + } + query = h.query; + close = h.connClose; + } + return { default: { Database, Connection } }; +}); + +vi.mock('../../src/core/embeddings/embedding-restore-spill.js', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + abortCachedEmbeddingsBuilder: ( + ...args: Parameters + ) => { + h.abortBuilder(...args); + return actual.abortCachedEmbeddingsBuilder(...args); + }, + }; +}); + +describe('staged embedding recovery child native lifecycle', () => { + const suffixes = [ + '', + '.wal', + '.shadow', + '.wal.checkpoint', + '.lock', + '.checkpoint.intent.lock', + '.checkpoint.apply.lock', + ]; + let tmp: string; + let dbPath: string; + let exportDir: string; + let originalArgv: string[]; + let originalExitCode: typeof process.exitCode; + + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + h.dbCtor.mockReset(); + h.connCtor.mockReset(); + h.dbClose.mockReset().mockResolvedValue(undefined); + h.connClose.mockReset().mockResolvedValue(undefined); + h.query.mockReset().mockResolvedValue({ + hasNext: vi.fn().mockResolvedValue(false), + getNext: vi.fn(), + close: vi.fn().mockResolvedValue(undefined), + }); + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-recovery-child-')); + dbPath = path.join(tmp, 'lbug.stage-test'); + exportDir = path.join(tmp, 'export'); + fs.writeFileSync(dbPath, 'mock native database'); + fs.writeFileSync(`${dbPath}.wal`, 'retained WAL'); + fs.mkdirSync(exportDir); + originalArgv = process.argv; + originalExitCode = process.exitCode; + process.argv = [process.execPath, 'staged-embedding-recovery-child', dbPath, exportDir, '2']; + process.exitCode = undefined; + vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + }); + + afterEach(() => { + vi.useRealTimers(); + process.argv = originalArgv; + process.exitCode = originalExitCode; + vi.restoreAllMocks(); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + function sourceFamily() { + return Object.fromEntries( + suffixes.map((suffix) => [suffix, fs.readFileSync(dbPath + suffix, 'utf8')]), + ); + } + + function seedCompleteFamily() { + for (const suffix of suffixes) fs.writeFileSync(dbPath + suffix, `retained ${suffix}`); + return sourceFamily(); + } + + async function runRejectedChild(message: string): Promise { + await import('../../src/core/embeddings/staged-embedding-recovery-child.js'); + await vi.waitFor(() => { + expect(process.exitCode).toBe(1); + expect(process.stderr.write).toHaveBeenCalledWith(`${message}\n`); + }); + expect(fs.existsSync(path.join(exportDir, 'manifest.json'))).toBe(false); + expect(fs.readFileSync(`${dbPath}.wal`, 'utf8')).toBe('retained WAL'); + } + + it('closes the opened database and aborts the builder when Connection construction fails', async () => { + h.connCtor.mockImplementation(() => { + throw new Error('connection constructor failed'); + }); + + await runRejectedChild('connection constructor failed'); + + expect(h.dbClose).toHaveBeenCalledOnce(); + expect(h.connClose).not.toHaveBeenCalled(); + expect(h.abortBuilder).toHaveBeenCalledOnce(); + expect(h.query).not.toHaveBeenCalled(); + expect(h.dbCtor.mock.calls[0][7]).toBe(true); + }); + + it('aborts the builder when Database construction fails', async () => { + h.dbCtor.mockImplementation(() => { + throw new Error('database constructor failed'); + }); + + await runRejectedChild('database constructor failed'); + + expect(h.abortBuilder).toHaveBeenCalledOnce(); + expect(h.dbClose).not.toHaveBeenCalled(); + expect(h.connCtor).not.toHaveBeenCalled(); + }); + + it('rejects output and closes the database when Connection close fails', async () => { + h.connClose.mockRejectedValue(new Error('connection close failed')); + + await runRejectedChild('connection close failed'); + + expect(h.dbClose).toHaveBeenCalled(); + expect(h.abortBuilder).toHaveBeenCalledOnce(); + }); + + it('rejects output when Database close fails', async () => { + h.dbClose.mockRejectedValue(new Error('database close failed')); + + await runRejectedChild('database close failed'); + + expect(h.connClose).toHaveBeenCalled(); + expect(h.abortBuilder).toHaveBeenCalledOnce(); + }); + + it('confines writable replay and failed checkpoint close to a separate copied family', async () => { + const sourceBefore = seedCompleteFamily(); + h.dbCtor.mockImplementation((openedPath: string) => { + for (const suffix of suffixes) { + expect(fs.readFileSync(openedPath + suffix, 'utf8')).toBe(sourceBefore[suffix]); + fs.writeFileSync(openedPath + suffix, `replayed ${suffix}`); + } + }); + h.dbClose.mockImplementation(async () => { + const openedPath = h.dbCtor.mock.calls[0][0] as string; + fs.writeFileSync(openedPath, 'partial checkpoint'); + throw new Error('checkpoint close failed'); + }); + + await import('../../src/core/embeddings/staged-embedding-recovery-child.js'); + await vi.waitFor(() => expect(process.exitCode).toBe(1)); + + expect(h.dbCtor.mock.calls[0][0]).not.toBe(dbPath); + expect(path.relative(exportDir, h.dbCtor.mock.calls[0][0] as string)).not.toMatch(/^\.\./); + expect(sourceFamily()).toEqual(sourceBefore); + expect(fs.existsSync(path.join(exportDir, 'manifest.json'))).toBe(false); + expect(process.stderr.write).toHaveBeenCalledWith('checkpoint close failed\n'); + }); + + it('reclaims a timed-out writer copy without changing the retained source', async () => { + const sourceBefore = seedCompleteFamily(); + h.query.mockReturnValue(new Promise(() => {})); + h.dbCtor.mockImplementation((openedPath: string) => { + for (const suffix of suffixes) fs.writeFileSync(openedPath + suffix, 'writer opened'); + }); + let childImport: Promise | undefined; + const child = Object.assign(new EventEmitter(), { + stderr: new EventEmitter(), + kill: vi.fn(() => { + queueMicrotask(() => child.emit('close', null, 'SIGKILL')); + return true; + }), + }); + h.spawn.mockImplementation((_command: string, args: string[]) => { + process.argv = [process.execPath, 'staged-embedding-recovery-child', ...args.slice(-3)]; + childImport = import('../../src/core/embeddings/staged-embedding-recovery-child.js'); + return child; + }); + const { recoverStagedEmbeddings } = + await import('../../src/core/embeddings/staged-embedding-recovery.js'); + vi.useFakeTimers(); + const recovering = expect( + recoverStagedEmbeddings(dbPath, { dimensions: 2, timeoutMs: 500 }), + ).rejects.toThrow(/timeout/); + await childImport; + expect(h.dbCtor).toHaveBeenCalledOnce(); + const openedPath = h.dbCtor.mock.calls[0][0] as string; + + await vi.advanceTimersByTimeAsync(500); + await recovering; + + expect(child.kill).toHaveBeenCalledWith('SIGKILL'); + expect(openedPath).not.toBe(dbPath); + expect(fs.existsSync(path.dirname(openedPath))).toBe(false); + expect(sourceFamily()).toEqual(sourceBefore); + expect(h.dbClose).not.toHaveBeenCalled(); + }); + + it.each(['.wal', '.checkpoint.intent.lock', '.checkpoint.apply.lock'])( + 'refuses a dangling family symlink before native open: %s', + async (suffix) => { + fs.rmSync(dbPath + suffix, { force: true }); + fs.symlinkSync(path.join(tmp, 'missing-sidecar'), dbPath + suffix); + + await import('../../src/core/embeddings/staged-embedding-recovery-child.js'); + await vi.waitFor(() => expect(process.exitCode).toBe(1)); + + expect(h.dbCtor).not.toHaveBeenCalled(); + expect(process.stderr.write).toHaveBeenCalledWith( + 'staged embedding family is not a regular file\n', + ); + expect(fs.existsSync(path.join(exportDir, 'manifest.json'))).toBe(false); + }, + ); + + it('refuses a family entry replaced with a symlink between lstat and open', async () => { + const foreignPath = path.join(tmp, 'foreign-file'); + fs.writeFileSync(foreignPath, 'foreign'); + const open = fs.openSync; + const read = vi.spyOn(fs, 'readSync'); + vi.spyOn(fs, 'openSync').mockImplementation((...args) => { + if (args[0] === dbPath) { + fs.rmSync(dbPath); + fs.symlinkSync(foreignPath, dbPath); + } + return open(...args); + }); + + await import('../../src/core/embeddings/staged-embedding-recovery-child.js'); + await vi.waitFor(() => expect(process.exitCode).toBe(1)); + + expect(read).not.toHaveBeenCalled(); + expect(h.dbCtor).not.toHaveBeenCalled(); + expect(fs.readFileSync(foreignPath, 'utf8')).toBe('foreign'); + expect(fs.existsSync(path.join(exportDir, 'manifest.json'))).toBe(false); + }); + + it('fails closed when copying the complete family runs out of space', async () => { + const sourceBefore = seedCompleteFamily(); + vi.spyOn(fs, 'writeFileSync').mockImplementation(() => { + throw Object.assign(new Error('copy ran out of space'), { code: 'ENOSPC' }); + }); + + await import('../../src/core/embeddings/staged-embedding-recovery-child.js'); + await vi.waitFor(() => expect(process.exitCode).toBe(1)); + + expect(h.dbCtor).not.toHaveBeenCalled(); + expect(sourceFamily()).toEqual(sourceBefore); + expect(fs.existsSync(path.join(exportDir, 'manifest.json'))).toBe(false); + expect(process.stderr.write).toHaveBeenCalledWith('copy ran out of space\n'); + }); + + it('writes the manifest only after both native closes succeed', async () => { + const closed: string[] = []; + h.connClose.mockImplementation(async () => { + expect(fs.existsSync(path.join(exportDir, 'manifest.json'))).toBe(false); + closed.push('connection'); + }); + h.dbClose.mockImplementation(async () => { + expect(fs.existsSync(path.join(exportDir, 'manifest.json'))).toBe(false); + closed.push('database'); + }); + + await import('../../src/core/embeddings/staged-embedding-recovery-child.js'); + await vi.waitFor(() => expect(fs.existsSync(path.join(exportDir, 'manifest.json'))).toBe(true)); + + expect(closed).toEqual(['connection', 'database']); + expect(h.abortBuilder).not.toHaveBeenCalled(); + expect(process.exitCode).toBeUndefined(); + expect(fs.readFileSync(`${dbPath}.wal`, 'utf8')).toBe('retained WAL'); + }); +}); diff --git a/gitnexus/test/unit/staged-embedding-recovery.test.ts b/gitnexus/test/unit/staged-embedding-recovery.test.ts new file mode 100644 index 000000000..552e48dc5 --- /dev/null +++ b/gitnexus/test/unit/staged-embedding-recovery.test.ts @@ -0,0 +1,95 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + createCachedEmbeddingsBuilder, + disposeEmbeddingSpill, + finalizeCachedEmbeddingsSnapshot, + ingestCachedEmbeddingRow, + materializeCachedEmbeddings, + type CachedEmbeddingsSnapshot, +} from '../../src/core/embeddings/embedding-restore-spill.js'; +import { + mergeRecoveredEmbeddings, + validateRecoveredNodeGroups, +} from '../../src/core/embeddings/staged-embedding-recovery.js'; + +describe('staged embedding recovery', () => { + const snapshots: CachedEmbeddingsSnapshot[] = []; + let tmp: string | undefined; + afterEach(() => { + for (const snapshot of snapshots) disposeEmbeddingSpill(snapshot.spill); + snapshots.length = 0; + if (tmp) fs.rmSync(tmp, { recursive: true, force: true }); + tmp = undefined; + }); + + function snapshot( + rows: { nodeId: string; chunkIndex: number; contentHash?: string; embedding?: number[] }[], + ) { + tmp ??= fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-stage-test-')); + const builder = createCachedEmbeddingsBuilder({ inMemoryRowLimit: 0, spillDir: tmp }); + for (const row of rows) { + ingestCachedEmbeddingRow( + builder, + { startLine: 1, endLine: 2, embedding: [1, 2], ...row }, + true, + ); + } + const result = finalizeCachedEmbeddingsSnapshot(builder); + snapshots.push(result); + return result; + } + + it('accepts only complete groups with one content hash and unique contiguous chunk ordinals', () => { + const cached = snapshot([ + { nodeId: 'complete', chunkIndex: 1, contentHash: 'same' }, + { nodeId: 'complete', chunkIndex: 0, contentHash: 'same' }, + { nodeId: 'gap', chunkIndex: 1, contentHash: 'same' }, + { nodeId: 'duplicate', chunkIndex: 0, contentHash: 'same' }, + { nodeId: 'duplicate', chunkIndex: 0, contentHash: 'same' }, + { nodeId: 'mixed', chunkIndex: 0, contentHash: 'old' }, + { nodeId: 'mixed', chunkIndex: 1, contentHash: 'new' }, + { nodeId: 'no-hash', chunkIndex: 0 }, + { nodeId: 'unsafe', chunkIndex: 0, contentHash: 'same' }, + ]); + expect([...validateRecoveredNodeGroups(cached.rows, new Set(['unsafe']))]).toEqual([ + 'complete', + ]); + }); + + it('replaces an entire published node group and keeps unrelated rows without retaining vector arrays', () => { + const live = snapshot([ + { nodeId: 'changed', chunkIndex: 0, contentHash: 'old' }, + { nodeId: 'changed', chunkIndex: 1, contentHash: 'old' }, + { nodeId: 'other', chunkIndex: 0, contentHash: 'other' }, + ]); + const recovered = snapshot([ + { nodeId: 'changed', chunkIndex: 0, contentHash: 'new', embedding: [7, 8] }, + ]); + const merged = mergeRecoveredEmbeddings(live, recovered); + snapshots.push(merged); + expect(merged.embeddings).toEqual([]); + expect(merged.rows).toHaveLength(2); + expect(materializeCachedEmbeddings(merged, merged.rows)).toEqual([ + expect.objectContaining({ nodeId: 'other', contentHash: 'other' }), + expect.objectContaining({ + nodeId: 'changed', + chunkIndex: 0, + contentHash: 'new', + embedding: [7, 8], + }), + ]); + expect(fs.existsSync(live.spill.path)).toBe(true); + expect(fs.existsSync(recovered.spill.path)).toBe(true); + }); + + it('does not accept missing vector bytes during a merge', () => { + const cached = snapshot([{ nodeId: 'complete', chunkIndex: 0, contentHash: 'same' }]); + fs.truncateSync(cached.spill.path, 12); + expect(() => mergeRecoveredEmbeddings(snapshot([]), cached)).toThrow( + /short embedding spill read/, + ); + }); +}); diff --git a/gitnexus/test/unit/staleness-fallback.test.ts b/gitnexus/test/unit/staleness-fallback.test.ts index 427c52b40..baed8e13b 100644 --- a/gitnexus/test/unit/staleness-fallback.test.ts +++ b/gitnexus/test/unit/staleness-fallback.test.ts @@ -3,7 +3,7 @@ * OTHER than a timeout. `staleness.test.ts` reaches `diverged` and `unknown` * against real repositories, but not the third arm of `fromHead`: HEAD still * resolves to the indexed commit, so the index is at HEAD however `rev-list` - * failed. Real git cannot fail `..HEAD` while HEAD prints that same SHA + * failed. Real git cannot fail `...HEAD` while HEAD prints that same SHA * without a corrupted object store, so this drives it through a mock. * * Its own file for the same reason as `staleness-timeout.test.ts`: the mock @@ -12,14 +12,21 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; const { plan, spawnedArgs } = vi.hoisted(() => ({ - plan: { head: null as string | null }, + plan: { head: null as string | null, enoent: false }, spawnedArgs: [] as string[][], })); // `rev-list` exits 128 the way git does for a missing object; `rev-parse HEAD` -// answers `plan.head`, or fails when it is null. +// answers `plan.head`, or fails when it is null. `plan.enoent` instead fails +// every invocation the way Node reports a missing executable (git not on +// PATH) — no exit code, `code: 'ENOENT'` — to check that shape of failure is +// caught by the same `catch` as a present-but-failing git (#3127). const answer = (args: readonly string[]): { error: Error | null; stdout: string } => { spawnedArgs.push([...args]); + if (plan.enoent) { + const failure = Object.assign(new Error('spawn git ENOENT'), { code: 'ENOENT' }); + return { error: failure, stdout: '' }; + } if (args[0] === 'rev-parse' && plan.head) return { error: null, stdout: `${plan.head}\n` }; const failure = Object.assign(new Error(`Command failed: git ${args.join(' ')}`), { code: 128, @@ -54,7 +61,7 @@ vi.mock('node:child_process', async (importOriginal) => { import { checkStaleness, checkStalenessAsync } from '../../src/core/git-staleness.js'; const INDEXED_COMMIT = 'a'.repeat(40); -const REV_LIST = ['rev-list', '--count', `${INDEXED_COMMIT}..HEAD`]; +const REV_LIST = ['rev-list', '--left-right', '--count', `${INDEXED_COMMIT}...HEAD`]; const REV_PARSE = ['rev-parse', 'HEAD']; const bothHelpers = { @@ -65,6 +72,7 @@ const bothHelpers = { describe('staleness after a failed (not timed-out) rev-list (#3256)', () => { beforeEach(() => { plan.head = null; + plan.enoent = false; spawnedArgs.length = 0; }); @@ -95,6 +103,19 @@ describe('staleness after a failed (not timed-out) rev-list (#3256)', () => { expect(result).toEqual({ isStale: false, commitsBehind: 0, status: 'unknown' }); expect(spawnedArgs).toEqual([REV_LIST, REV_PARSE]); }); + + it('reports unknown — never current — when git itself is not on PATH', async () => { + // nikolai-vysotskyi (#3127): "git not on PATH" specifically, as a + // distinct failure shape (ENOENT, no exit code) from a present git + // that fails against a bad ref. Same requirement either way: it must + // never be silently read as fresh. + plan.enoent = true; + + const result = await check('/repo', INDEXED_COMMIT); + + expect(result).toEqual({ isStale: false, commitsBehind: 0, status: 'unknown' }); + expect(spawnedArgs).toEqual([REV_LIST, REV_PARSE]); + }); }); } }); diff --git a/gitnexus/test/unit/staleness-timeout.test.ts b/gitnexus/test/unit/staleness-timeout.test.ts index 1b80df5b8..6f4670135 100644 --- a/gitnexus/test/unit/staleness-timeout.test.ts +++ b/gitnexus/test/unit/staleness-timeout.test.ts @@ -43,6 +43,8 @@ describe('checkStalenessAsync — timed-out rev-list (#3256)', () => { expect(result).toEqual({ status: 'unknown', isStale: false, commitsBehind: 0 }); // Exactly one spawn. A `rev-parse HEAD` follow-up here is the regression: // it doubles the hung-mount bound and can flip this answer. - expect(spawnedArgs).toEqual([['rev-list', '--count', `${INDEXED_COMMIT}..HEAD`]]); + expect(spawnedArgs).toEqual([ + ['rev-list', '--left-right', '--count', `${INDEXED_COMMIT}...HEAD`], + ]); }); }); diff --git a/gitnexus/test/unit/staleness-zero-count.test.ts b/gitnexus/test/unit/staleness-zero-count.test.ts new file mode 100644 index 000000000..4e2b510b7 --- /dev/null +++ b/gitnexus/test/unit/staleness-zero-count.test.ts @@ -0,0 +1,162 @@ +/** + * Count both sides in one Git process to distinguish freshness from rollback + * without mixing HEAD snapshots (#3127). + * Keep the child-process mock isolated from tests that use real repositories. + */ +import type { ExecFileOptions } from 'node:child_process'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { plan, invocations } = vi.hoisted(() => ({ + plan: { counts: '0\t0\n' as string | null, head: 'failure', advanceHead: false }, + invocations: [] as { args: string[]; options: ExecFileOptions }[], +})); + +const answer = (args: readonly string[], options: ExecFileOptions): string => { + invocations.push({ args: [...args], options }); + if (args[0] === 'rev-list') { + if (plan.counts === null) { + throw Object.assign(new Error('Command failed: git rev-list'), { code: 128, killed: false }); + } + // Simulate a checkout/commit after Git measured the relationship. A second + // process would see this different HEAD and could misclassify the result. + if (plan.advanceHead) plan.head = 'b'.repeat(40); + return plan.counts; + } + if (plan.head === 'empty') return ' \n'; + if (plan.head === 'timeout') { + throw Object.assign(new Error('Command failed: git rev-parse HEAD'), { + killed: true, + signal: 'SIGTERM', + }); + } + if (plan.head === 'failure') { + throw Object.assign(new Error('Command failed: git rev-parse HEAD'), { + code: 128, + killed: false, + }); + } + return `${plan.head}\n`; +}; + +vi.mock('node:child_process', async (importOriginal) => { + const actual = await importOriginal(); + const { promisify } = await import('node:util'); + const execFile = ( + _file: string, + args: readonly string[], + options: ExecFileOptions, + callback: (error: Error | null, stdout: string, stderr: string) => void, + ): void => { + try { + callback(null, answer(args, options), ''); + } catch (error) { + callback(error as Error, '', ''); + } + }; + // Real execFile has a custom promisifier returning both output streams. + Object.defineProperty(execFile, promisify.custom, { + value: async (_file: string, args: readonly string[], options: ExecFileOptions) => ({ + stdout: answer(args, options), + stderr: '', + }), + }); + const execFileSync = (_file: string, args: readonly string[], options: ExecFileOptions): string => + answer(args, options); + return { + ...actual, + execFile: execFile as unknown as typeof actual.execFile, + execFileSync: execFileSync as unknown as typeof actual.execFileSync, + }; +}); + +import { checkStaleness, checkStalenessAsync } from '../../src/core/git-staleness.js'; + +const INDEXED_COMMIT = 'a'.repeat(40); +const REV_LIST = ['rev-list', '--left-right', '--count', `${INDEXED_COMMIT}...HEAD`]; +const REV_PARSE = ['rev-parse', 'HEAD']; + +const bothHelpers = { + checkStaleness: async (repo: string, lastCommit: string) => checkStaleness(repo, lastCommit), + checkStalenessAsync, +}; + +describe('staleness from one relationship query (#3127)', () => { + beforeEach(() => { + plan.counts = '0\t0\n'; + plan.head = 'failure'; + plan.advanceHead = false; + invocations.length = 0; + }); + + for (const [name, check] of Object.entries(bothHelpers)) { + describe(name, () => { + it.each([ + { counts: '0\t0\n', status: 'current', isStale: false, commitsBehind: 0 }, + { counts: '2\t0\n', status: 'diverged', isStale: true, commitsBehind: 0 }, + { counts: '0\t3\n', status: 'behind', isStale: true, commitsBehind: 3 }, + { counts: '2\t3\n', status: 'behind', isStale: true, commitsBehind: 3 }, + ])('reports $status from $counts in one process', async ({ counts, ...expected }) => { + plan.counts = counts; + + const result = await check('/repo', INDEXED_COMMIT); + + expect(result).toMatchObject(expected); + expect(invocations.map(({ args }) => args)).toEqual([REV_LIST]); + }); + + it('keeps the count snapshot when HEAD advances after the query', async () => { + plan.head = INDEXED_COMMIT; + plan.advanceHead = true; + + const result = await check('/repo', INDEXED_COMMIT); + + expect(plan.head).not.toBe(INDEXED_COMMIT); + expect(result).toEqual({ status: 'current', isStale: false, commitsBehind: 0 }); + expect(invocations.map(({ args }) => args)).toEqual([REV_LIST]); + }); + + it.each(['', '0\n', '0\tbogus\n'])( + 'reports unknown for malformed counts %j', + async (counts) => { + plan.counts = counts; + + const result = await check('/repo', INDEXED_COMMIT); + + expect(result).toEqual({ status: 'unknown', isStale: false, commitsBehind: 0 }); + expect(invocations.map(({ args }) => args)).toEqual([REV_LIST]); + }, + ); + + it('reports unknown when the follow-up HEAD command fails', async () => { + plan.counts = null; + const result = await check('/repo', INDEXED_COMMIT); + + expect(result).toEqual({ status: 'unknown', isStale: false, commitsBehind: 0 }); + expect(invocations.map(({ args }) => args)).toEqual([REV_LIST, REV_PARSE]); + }); + + it('reports unknown when the follow-up HEAD command returns no commit', async () => { + plan.counts = null; + plan.head = 'empty'; + + const result = await check('/repo', INDEXED_COMMIT); + + expect(result).toEqual({ status: 'unknown', isStale: false, commitsBehind: 0 }); + expect(invocations.map(({ args }) => args)).toEqual([REV_LIST, REV_PARSE]); + }); + + it('bounds the HEAD command and reports unknown without retrying after its timeout', async () => { + plan.counts = null; + plan.head = 'timeout'; + + const result = await check('/repo', INDEXED_COMMIT); + + expect(result).toEqual({ status: 'unknown', isStale: false, commitsBehind: 0 }); + expect(invocations.map(({ args }) => args)).toEqual([REV_LIST, REV_PARSE]); + const timeout = invocations[1].options.timeout; + expect(Number.isFinite(timeout)).toBe(true); + expect(timeout).toBeGreaterThan(0); + }); + }); + } +}); diff --git a/gitnexus/test/unit/staleness.test.ts b/gitnexus/test/unit/staleness.test.ts index ef4a6a517..0e6084d7e 100644 --- a/gitnexus/test/unit/staleness.test.ts +++ b/gitnexus/test/unit/staleness.test.ts @@ -335,3 +335,52 @@ describe('branch-pinned serve clone once git prunes the indexed commit (#3256)', }); }); }); + +// ── #3127: a 0 forward count is not "the index matches this tree" ─────────── +// +// nikolai-vysotskyi (issue #3127, comment on the `--stale-policy` proposal): +// `git rev-list --count lastCommit..HEAD` answers "commits reachable from +// HEAD but not lastCommit", which is also 0 when HEAD is an *ancestor* of +// lastCommit — i.e. the working tree checked out an older commit than the one +// indexed, or a release branch behind the indexed tip. Before this fix that +// read as `current`; a `--stale-policy error`-style caller would exit 0 +// exactly when the index is provably wrong about the checked-out tree. +describe('staleness when the checkout has regressed behind the indexed commit (#3127)', () => { + let root: string; + let fixture: ReturnType; + + beforeAll(() => { + root = mkdtempSync(join(tmpdir(), 'gitnexus-staleness-regressed-')); + fixture = makeRepo(root, 'repo'); + // Roll the working tree back to c1. `rev-list --count c3..HEAD` alone + // answers 0 here (HEAD/c1 has no commits c3 lacks) — exactly the count a + // pre-fix caller read as "index matches HEAD". + git(fixture.repo, 'checkout', '-q', fixture.c1); + }); + afterAll(() => removeTree(root)); + + for (const [name, check] of Object.entries(bothHelpers)) { + describe(name, () => { + it('reports diverged, not current, when HEAD is behind the indexed commit', async () => { + const result = await check(fixture.repo, fixture.c3); + + expect(result.status).toBe('diverged'); + // Established by a positive indexed-only count and a zero HEAD-only + // count (not a `rev-list` failure), so — unlike the failure-path + // `diverged` above — `isStale` reflects the mismatch instead of the + // historical fail-open `false`. + expect(result.isStale).toBe(true); + expect(result.commitsBehind).toBe(0); + expect(result.hint).toContain('not reachable from the checked-out commit'); + }); + + it('still reports current for the commit actually checked out', async () => { + expect(await check(fixture.repo, fixture.c1)).toMatchObject({ + status: 'current', + isStale: false, + commitsBehind: 0, + }); + }); + }); + } +}); diff --git a/gitnexus/test/unit/symbol-identity-validation.test.ts b/gitnexus/test/unit/symbol-identity-validation.test.ts new file mode 100644 index 000000000..7fe380a20 --- /dev/null +++ b/gitnexus/test/unit/symbol-identity-validation.test.ts @@ -0,0 +1,234 @@ +/** Regression for unusable database lookup identities (#3424). */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const { lbugMocks } = vi.hoisted(() => ({ + lbugMocks: { + initLbug: vi.fn().mockResolvedValue(undefined), + executeQuery: vi.fn().mockResolvedValue([]), + executeParameterized: vi.fn().mockResolvedValue([]), + closeLbug: vi.fn().mockResolvedValue(undefined), + isLbugReady: vi.fn().mockReturnValue(true), + }, +})); + +vi.mock('../../src/core/lbug/pool-adapter.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, ...lbugMocks }; +}); + +vi.mock('../../src/mcp/core/lbug-adapter.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, ...lbugMocks }; +}); + +vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + listRegisteredRepos: vi.fn().mockResolvedValue([ + { + name: 'test-project', + path: '/tmp/test-project', + storagePath: '/tmp/.gitnexus/test-project', + indexedAt: '2024-06-01T12:00:00Z', + lastCommit: 'abc123', + stats: { files: 10, nodes: 50, edges: 100, communities: 3, processes: 5 }, + }, + ]), + cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }), + findSiblingClones: vi.fn().mockResolvedValue([]), + }; +}); + +vi.mock('../../src/core/git-staleness.js', () => ({ + checkStaleness: vi.fn().mockReturnValue({ isStale: false, commitsBehind: 0 }), + checkStalenessAsync: vi.fn().mockResolvedValue({ isStale: false, commitsBehind: 0 }), + checkCwdMatch: vi.fn().mockResolvedValue({ match: 'none' }), +})); + +vi.mock('../../src/storage/git.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, getGitRoot: vi.fn().mockReturnValue(null) }; +}); + +vi.mock('../../src/core/search/bm25-index.js', () => ({ + searchFTSFromLbug: vi.fn().mockResolvedValue({ results: [], ftsAvailable: true }), +})); + +vi.mock('../../src/mcp/core/embedder.js', () => ({ + embedQuery: vi.fn().mockResolvedValue([]), + getEmbeddingDims: vi.fn().mockReturnValue(384), +})); + +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { executeParameterized } from '../../src/mcp/core/lbug-adapter.js'; + +const SYMBOL = { + id: 'Method:tests/test_supervisor.py:Supervisor.test_run', + name: 'test_run', + type: 'Method', + filePath: 'tests/test_supervisor.py', + startLine: 3, + endLine: 5, +}; + +const badIds = [ + ['missing', undefined], + ['null', null], + ['number', 42], + ['empty', ''], + ['blank', ' \t '], + ['NUL-only', '\0\0'], + ['embedded NUL', 'Method:tests/test_supervisor.py:\0test_run'], +] as const; + +function row(id: unknown, shape: string): unknown { + return shape === 'tuple' + ? [id, SYMBOL.name, SYMBOL.type, SYMBOL.filePath, SYMBOL.startLine, SYMBOL.endLine] + : { ...SYMBOL, id }; +} + +const surfaces = [ + { tool: 'context', params: { name: SYMBOL.name, file_path: SYMBOL.filePath } }, + { tool: 'impact', params: { target: SYMBOL.name, direction: 'upstream' } }, + { tool: 'impact', params: { target: SYMBOL.name, direction: 'upstream', mode: 'pdg' } }, +] as const; + +describe('symbol lookup identity validation (#3424)', () => { + let backend: LocalBackend; + + beforeEach(async () => { + vi.clearAllMocks(); + backend = new LocalBackend(); + await backend.init(); + (backend as any).ensureInitialized = vi.fn().mockResolvedValue(undefined); + }); + + function lookupRows(rows: unknown[]) { + vi.mocked(executeParameterized).mockImplementation(async (_db, _query, params) => { + return params?.symName || params?.uid ? rows : []; + }); + } + + function expectNoExpansion() { + const queries = vi.mocked(executeParameterized).mock.calls.map(([, query]) => query); + expect(queries.length).toBeGreaterThan(0); + expect(queries.every((query) => !query.includes('CodeRelation'))).toBe(true); + expect(queries.every((query) => !query.includes('UNION'))).toBe(true); + } + + function expectIdentityError(result: any, tool: string) { + expect(result.error).toMatch(/symbol identity/i); + expect(result.recoverySuggestion).toMatch(/analyze.*--force/); + expect(result.epistemic).not.toBe('exact'); + expect(result).not.toHaveProperty('symbol'); + expect(result).not.toHaveProperty('incoming'); + if (tool === 'impact') { + expect(result.risk).toBe('UNKNOWN'); + expect(result.impactedCount).toBeNull(); + expect(result.target).not.toHaveProperty('id'); + expect(result.suggestion ?? '').not.toContain('context'); + } + expectNoExpansion(); + } + + for (const surface of surfaces) { + describe(`${surface.tool} ${'mode' in surface.params ? surface.params.mode : 'default'}`, () => { + for (const shape of ['object', 'tuple']) { + it.each(badIds)(`rejects %s IDs in ${shape} rows before traversal`, async (_label, id) => { + lookupRows([row(id, shape)]); + const result = await backend.callTool(surface.tool, surface.params); + expectIdentityError(result, surface.tool); + }); + it.each(badIds)( + `rejects %s IDs from exact UID lookups in ${shape} rows`, + async (_label, id) => { + lookupRows([row(id, shape)]); + const uidParam = + surface.tool === 'context' ? { uid: SYMBOL.id } : { target_uid: SYMBOL.id }; + expectIdentityError( + await backend.callTool(surface.tool, { ...surface.params, ...uidParam }), + surface.tool, + ); + }, + ); + } + + it('does not choose a healthy candidate beside a corrupt candidate', async () => { + lookupRows([SYMBOL, { ...SYMBOL, id: '\0', type: '' }]); + expectIdentityError(await backend.callTool(surface.tool, surface.params), surface.tool); + }); + + it('rejects a different identity returned for an exact UID', async () => { + lookupRows([{ ...SYMBOL, id: 'Constructor:Services.swift:Service.init' }]); + const uidParam = + surface.tool === 'context' ? { uid: SYMBOL.id } : { target_uid: SYMBOL.id }; + expectIdentityError( + await backend.callTool(surface.tool, { ...surface.params, ...uidParam }), + surface.tool, + ); + }); + + it('keeps an empty lookup distinct from an invalid identity', async () => { + lookupRows([]); + const result = await backend.callTool(surface.tool, surface.params); + expect(result.error).toMatch(/not found/); + expect(result.recoverySuggestion).toBeUndefined(); + }); + }); + } + + it('validates every row before exact File narrowing', async () => { + lookupRows([ + { ...SYMBOL, id: 'File:tests/test_supervisor.py', name: 'test_supervisor.py' }, + { ...SYMBOL, id: undefined }, + ]); + expectIdentityError(await backend.callTool('context', { name: SYMBOL.filePath }), 'context'); + }); + + for (const shape of ['object', 'tuple']) { + it(`accepts an opaque legacy identity in a healthy ${shape} row`, async () => { + lookupRows([row('func:alpha', shape)]); + const result = await backend.callTool('context', { uid: 'func:alpha' }); + expect(result).not.toHaveProperty('error'); + expect(result.symbol.uid).toBe('func:alpha'); + }); + } + + it('preserves a valid ID byte-for-byte instead of trimming it', async () => { + lookupRows([{ ...SYMBOL, id: 'func:alpha ' }]); + const result = await backend.callTool('context', { name: SYMBOL.name }); + expect(result).not.toHaveProperty('error'); + expect(result.symbol.uid).toBe('func:alpha '); + }); + + describe('group impact UID adapter', () => { + const opts = { maxDepth: 3, relationTypes: ['CALLS'], minConfidence: 0, includeTests: true }; + + it.each(badIds)('rejects a %s persisted ID before BFS', async (_label, id) => { + lookupRows([{ ...SYMBOL, id }]); + const bfs = vi.spyOn(backend as any, '_runImpactBFS').mockResolvedValue({ byDepth: {} }); + const repoId = [...(backend as any).repos.keys()][0]; + expect(await backend.impactByUid(repoId, SYMBOL.id, 'upstream', opts)).toBeNull(); + expect(bfs).not.toHaveBeenCalled(); + }); + + it('rejects an otherwise valid mismatched UID', async () => { + lookupRows([{ ...SYMBOL, id: 'route:other' }]); + const bfs = vi.spyOn(backend as any, '_runImpactBFS').mockResolvedValue({ byDepth: {} }); + const repoId = [...(backend as any).repos.keys()][0]; + expect(await backend.impactByUid(repoId, SYMBOL.id, 'upstream', opts)).toBeNull(); + expect(bfs).not.toHaveBeenCalled(); + }); + + it('accepts a legitimate synthetic identity', async () => { + lookupRows([{ ...SYMBOL, id: 'Route:svc:/health' }]); + const bfs = vi.spyOn(backend as any, '_runImpactBFS').mockResolvedValue({ byDepth: {} }); + const repoId = [...(backend as any).repos.keys()][0]; + expect(await backend.impactByUid(repoId, 'Route:svc:/health', 'upstream', opts)).toEqual({ + byDepth: {}, + }); + expect(bfs).toHaveBeenCalledOnce(); + }); + }); +}); diff --git a/gitnexus/test/unit/tool-process-linking.test.ts b/gitnexus/test/unit/tool-process-linking.test.ts index 6c536bb38..d28dd9329 100644 --- a/gitnexus/test/unit/tool-process-linking.test.ts +++ b/gitnexus/test/unit/tool-process-linking.test.ts @@ -55,41 +55,50 @@ function addCall(graph: KnowledgeGraph, sourceId: string, targetId: string) { } describe('Tool handler and process linking phases', () => { - it('falls back to the file node when a parsed tool handler is missing from the graph', async () => { - const graph = createKnowledgeGraph(); - addNode(graph, 'File:src/tools.py', 'File', 'tools.py', 'src/tools.py'); + it.each([undefined, false] as const)( + 'retains file attribution policy %s when a parsed handler is missing', + async (allowFileFallback) => { + const graph = createKnowledgeGraph(); + addNode(graph, 'File:src/tools.py', 'File', 'tools.py', 'src/tools.py'); - const output = await toolsPhase.execute( - makeCtx(graph), - new Map([ - [ - 'parse', - phaseResult('parse', { - allToolDefs: [ - { - filePath: 'src/tools.py', - toolName: 'stale_tool', - description: 'Stale handler', - lineNumber: 1, - handlerNodeId: 'Function:src/tools.py:missing', - }, - ], - allPaths: [], - }), - ], - ]), - ); + const output = await toolsPhase.execute( + makeCtx(graph), + new Map([ + [ + 'parse', + phaseResult('parse', { + allToolDefs: [ + { + filePath: 'src/tools.py', + toolName: 'stale_tool', + description: 'Stale handler', + lineNumber: 1, + handlerNodeId: 'Function:src/tools.py:missing', + ...(allowFileFallback === false ? { allowFileFallback } : {}), + }, + ], + allPaths: [], + }), + ], + ]), + ); - expect(output.toolDefs).toEqual([ - { name: 'stale_tool', filePath: 'src/tools.py', description: 'Stale handler' }, - ]); + expect(output.toolDefs).toEqual([ + { + name: 'stale_tool', + filePath: 'src/tools.py', + description: 'Stale handler', + ...(allowFileFallback === false ? { allowFileFallback } : {}), + }, + ]); - const edge = graph.relationships.find((rel) => rel.type === 'HANDLES_TOOL'); - expect(edge).toMatchObject({ - sourceId: 'File:src/tools.py', - targetId: 'Tool:stale_tool', - }); - }); + const edge = graph.relationships.find((rel) => rel.type === 'HANDLES_TOOL'); + expect(edge).toMatchObject({ + sourceId: 'File:src/tools.py', + targetId: 'Tool:stale_tool', + }); + }, + ); it('does not attach file-level fallback tools to handler-specific processes', async () => { const graph = createKnowledgeGraph(); @@ -110,6 +119,7 @@ describe('Tool handler and process linking phases', () => { addNode(graph, fileLeaf, 'Function', 'fileLeaf', filePath); addNode(graph, 'Tool:alpha', 'Tool', 'alpha', filePath); addNode(graph, 'Tool:fallback_tool', 'Tool', 'fallback_tool', filePath); + addNode(graph, 'Tool:unresolved_tool', 'Tool', 'unresolved_tool', filePath); addCall(graph, alpha, alphaHelper); addCall(graph, alphaHelper, alphaLeaf); addCall(graph, fileEntry, fileHelper); @@ -127,6 +137,7 @@ describe('Tool handler and process linking phases', () => { toolDefs: [ { name: 'alpha', filePath, description: '', handlerNodeId: alpha }, { name: 'fallback_tool', filePath, description: '' }, + { name: 'unresolved_tool', filePath, description: '', allowFileFallback: false }, ], }), ], @@ -147,5 +158,6 @@ describe('Tool handler and process linking phases', () => { expect(linkedEntriesByTool.get('Tool:alpha')).toEqual([alpha]); expect(linkedEntriesByTool.get('Tool:fallback_tool')).toEqual([fileEntry]); + expect(linkedEntriesByTool.has('Tool:unresolved_tool')).toBe(false); }); }); diff --git a/gitnexus/test/unit/typescript-tool-definitions.test.ts b/gitnexus/test/unit/typescript-tool-definitions.test.ts new file mode 100644 index 000000000..e248a682c --- /dev/null +++ b/gitnexus/test/unit/typescript-tool-definitions.test.ts @@ -0,0 +1,441 @@ +import { describe, expect, it } from 'vitest'; +import Parser from 'tree-sitter'; +import JavaScript from 'tree-sitter-javascript'; +import TypeScript from 'tree-sitter-typescript'; +import { extractToolDefinitions } from '../../src/core/ingestion/languages/typescript/tool-definitions.js'; + +const tsParser = new Parser(); +tsParser.setLanguage(TypeScript.typescript); +const jsParser = new Parser(); +jsParser.setLanguage(JavaScript); + +const sdkImport = `import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';`; +const server = `${sdkImport}\nconst server = new McpServer({ name: 'example', version: '1' });`; +const extract = (source: string, parser = tsParser, filePath = 'src/server.ts', offset = 0) => + extractToolDefinitions(parser.parse(source), filePath, offset); +const metadata = (source: string) => + extract(source).map(({ toolName, description }) => ({ toolName, description })); + +describe('SDK tool registration extraction', () => { + it.each([ + ['TypeScript', tsParser, 'src/server.ts'], + ['JavaScript', jsParser, 'src/server.js'], + ] as const)( + 'extracts modern %s registrations in ordinary server files', + (_language, parser, filePath) => { + expect( + extract( + `${server}\nserver.registerTool('search', { description: 'Search files' }, handler);`, + parser, + filePath, + 10, + ), + ).toEqual([ + { + filePath, + toolName: 'search', + description: 'Search files', + lineNumber: 13, + allowFileFallback: false, + }, + ]); + }, + ); + + it.each([ + ['TypeScript', tsParser, 'src/server.ts'], + ['JavaScript', jsParser, 'src/server.js'], + ] as const)('recognizes namespace imports in %s', (_language, parser, filePath) => { + expect( + extract( + ` + import * as SDK from '@modelcontextprotocol/sdk/server/mcp.js'; + const server = new SDK.McpServer({}); + server.registerTool('modern', { description: 'Namespace tool' }, handler); + server.tool('legacy', handler); + `, + parser, + filePath, + ).map(({ toolName }) => toolName), + ).toEqual(['modern', 'legacy']); + }); + + it.each(['', 'type '])('recognizes %snamespace imports in directly typed helpers', (typeOnly) => { + expect( + metadata(` + import ${typeOnly}* as SDK from '@modelcontextprotocol/sdk/server/mcp'; + function install(server: SDK.McpServer) { server.tool('typed', handler); } + `), + ).toEqual([{ toolName: 'typed', description: '' }]); + }); + + it.each([ + "function install(SDK) { const server = new SDK.McpServer({}); server.tool('fake', handler); }", + "function install(server: SDK.McpServer) { server.tool('fake', handler); }", + "SDK = other; const server = new SDK.McpServer({}); server.tool('fake', handler);", + "SDK.McpServer = other; const server = new SDK.McpServer({}); server.tool('fake', handler);", + "({ value: SDK.McpServer } = other); const server = new SDK.McpServer({}); server.tool('fake', handler);", + "SDK.McpServer.prototype.tool = other; const server = new SDK.McpServer({}); server.tool('fake', handler);", + "SDK[key] = other; const server = new SDK.McpServer({}); server.tool('fake', handler);", + "const server = new SDK.McpServer({}); ({ method: server.tool } = other); server.tool('fake', handler);", + "const alias = SDK; const server = new alias.McpServer({}); server.tool('fake', handler);", + "const server = new SDK.OtherServer({}); server.tool('fake', handler);", + ])('rejects unproven namespace receivers: %s', (source) => { + expect( + metadata(`import * as SDK from '@modelcontextprotocol/sdk/server/mcp.js'; ${source}`), + ).toEqual([]); + }); + + it('rejects type-only namespace construction and unrelated namespace imports', () => { + expect( + metadata(` + import type * as SDK from '@modelcontextprotocol/sdk/server/mcp.js'; + import * as Other from 'unrelated'; + const first = new SDK.McpServer({}); first.tool('type-only', handler); + const second = new Other.McpServer({}); second.tool('unrelated', handler); + `), + ).toEqual([]); + }); + + it('does not require description or inputSchema, and ignores nested descriptions', () => { + expect( + metadata(`${server} + server.registerTool('empty', {}, () => {}); + server.registerTool('nested', { inputSchema: { description: 'Schema decoy' } }, handler); + server.registerTool('dynamic-description', { description: getDescription() }, handler); + `), + ).toEqual([ + { toolName: 'empty', description: '' }, + { toolName: 'nested', description: '' }, + { toolName: 'dynamic-description', description: '' }, + ]); + }); + + it('recognizes legacy callback-last overloads with descriptions, schemas and annotations', () => { + expect( + metadata(`${server} + server.tool('bare', handler); + server.tool('described', 'Human description', handler); + server.tool('schema', { query: z.string().describe('Field decoy') }, handler); + server.tool('annotated', { readOnlyHint: true }, handler); + server.tool('full', 'Full description', { query: z.string() }, { readOnlyHint: true }, handler); + `), + ).toEqual([ + { toolName: 'bare', description: '' }, + { toolName: 'described', description: 'Human description' }, + { toolName: 'schema', description: '' }, + { toolName: 'annotated', description: '' }, + { toolName: 'full', description: 'Full description' }, + ]); + }); + + it('decodes static names and quoted description keys without distance or property-order limits', () => { + expect( + metadata(`${server} + server.registerTool(\`find\\x2ditems!?\`, { + inputSchema: { description: 'Nested decoy', example: '${'x'.repeat(2000)}' }, + 'descr\\u0069ption': 'Line\\nwith \\"quotes\\" and \\u{1F680}', + }, handler); + server.tool('legacy\\u002fname', \`Static description\`, handler); + `), + ).toEqual([ + { toolName: 'find-items!?', description: 'Line\nwith "quotes" and 🚀' }, + { toolName: 'legacy/name', description: 'Static description' }, + ]); + }); + + it('recognizes SDK aliases, locally constructed instances and directly typed helper parameters', () => { + expect( + metadata(` + import { McpServer as Server } from '@modelcontextprotocol/sdk/server/mcp.js'; + function install(server: Server) { server.registerTool('helper', {}, handler); } + const add = (server: Server) => server.tool('arrow-helper', handler); + function start() { + const local = new Server({ name: 'local', version: '1' }); + local.registerTool('local', {}, handler); + } + `), + ).toEqual([ + { toolName: 'helper', description: '' }, + { toolName: 'arrow-helper', description: '' }, + { toolName: 'local', description: '' }, + ]); + }); + + it('ignores dynamic names, comments, string decoys and unrelated receivers', () => { + expect( + metadata(`${server} + // server.registerTool('comment', { description: 'decoy' }, handler); + const decoy = "server.tool('string', handler)"; + server.registerTool(runtimeName, {}, handler); + server.tool(\`dynamic-\${runtimeName}\`, handler); + server.registerTool('prefix' + suffix, {}, handler); + unrelated.registerTool('unrelated', {}, handler); + const alias = server; + alias.tool('alias', handler); + `), + ).toEqual([]); + }); + + it('rejects shadowed SDK names and receivers, including declarations later in the scope', () => { + expect( + metadata(`${server} + function parameter(server) { server.tool('parameter', handler); } + function constructor(McpServer) { + const fake = new McpServer(); + fake.tool('constructor', handler); + } + { + server.registerTool('temporal-shadow', {}, handler); + const server = unrelated; + } + function localType() { + class McpServer {} + function helper(server: McpServer) { server.tool('type-shadow', handler); } + } + `), + ).toEqual([]); + }); + + it('rejects reassigned receivers and SDK constructors', () => { + expect( + metadata(`${sdkImport} + let changed = new McpServer(); + changed = unrelated; + changed.tool('changed', handler); + const instance = new McpServer(); + function replace() { McpServer = OtherServer; } + instance.registerTool('constructor-mutated', {}, handler); + `), + ).toEqual([]); + }); + + it('uses decoded type-only imports for helper parameters, but not construction', () => { + expect( + metadata(` + import type { McpServer as Server } from '@modelcontextprotocol/\\u0073dk/server/mcp.js'; + function install(server: Server) { server.tool('typed', handler); } + const fake = new Server(); + fake.tool('type-only-constructor', handler); + `), + ).toEqual([{ toolName: 'typed', description: '' }]); + }); + + it.each(['before', 'after'])('allows SDK lifecycle configuration %s registration', (when) => { + const configure = `server.server.oninitialized = () => {}; server.server.onerror = () => {};`; + const registration = `server.registerTool('visible', {}, handler);`; + expect( + metadata( + `${server}\n${when === 'before' ? configure + registration : registration + configure}`, + ), + ).toEqual([{ toolName: 'visible', description: '' }]); + }); + + describe.each([ + ['TypeScript', tsParser], + ['JavaScript', jsParser], + ] as const)('%s destructuring writes', (_language, parser) => { + it.each([ + ['object member', `({ registerTool: server.registerTool } = replacement);`], + ['array member', `[server.tool] = replacement;`], + ['nested quoted member', `({ nested: [server['registerTool']] } = replacement);`], + ['defaulted member', `({ registerTool: server.registerTool = fallback } = replacement);`], + ['rest member', `[...server.tool] = replacement;`], + ['computed member', `[server[method]] = replacement;`], + ['loop target', `for ({ registerTool: server.registerTool } of replacements) {}`], + ['constructor member', `[McpServer.prototype.registerTool] = replacement;`], + ])('rejects registrations after a write to an %s target', (_name, write) => { + expect( + extract(`${server}\n${write}\nserver.registerTool('fake', {}, handler);`, parser), + ).toEqual([]); + }); + + it('preserves lifecycle writes and ignores pattern keys and default-value reads', () => { + expect( + extract( + `${server} + ({ oninitialized: server.server.oninitialized } = callbacks); + ({ [server.registerTool]: ignored } = source); + ({ untouched = server.registerTool } = source); + server.registerTool('visible', {}, handler); + `, + parser, + ).map((tool) => tool.toolName), + ).toEqual(['visible']); + }); + }); + + it.each([ + [ + 'different package', + `import { McpServer } from 'unrelated'; const server = new McpServer(); server.tool('fake', h);`, + ], + [ + 'destructured parameter', + `${server} function install({ server }) { server.tool('fake', h); }`, + ], + ['destructured local', `${server} { const { other: server } = obj; server.tool('fake', h); }`], + ['catch parameter', `${server} try {} catch (server) { server.tool('fake', h); }`], + ['loop binding', `${server} for (const server of other) { server.tool('fake', h); }`], + [ + 'hoisted var', + `${server} function install() { server.tool('fake', h); { var server = other; } }`, + ], + [ + 'generic type', + `${sdkImport} function install(server: McpServer) { server.tool('fake', h); }`, + ], + [ + 'named class expression', + `${sdkImport} const Other = class McpServer { install() { const server = new McpServer(); server.tool('fake', h); } };`, + ], + ['method write', `${server} server.tool = unrelated; server.tool('fake', h);`], + ['quoted method write', `${server} server['tool'] = unrelated; server.tool('fake', h);`], + ['computed method write', `${server} server[method] = unrelated; server.tool('fake', h);`], + ['method delete', `${server} delete server.registerTool; server.registerTool('fake', {}, h);`], + ['destructured write', `${server} ({ server } = other); server.tool('fake', h);`], + [ + 'spread arguments', + `${server} server.registerTool('fake', ...args); server.tool('fake', ...args);`, + ], + ])('rejects %s', (_name, source) => { + expect(metadata(source)).toEqual([]); + }); + + it('keeps evidence outside shadowing scopes and in closures declared before the instance', () => { + expect( + metadata(`${sdkImport} + function install() { server.tool('closure', handler); } + const server = new McpServer(); + { const server = unrelated; server.tool('decoy', handler); } + server.registerTool('outer', {}, handler); + `), + ).toEqual([ + { toolName: 'closure', description: '' }, + { toolName: 'outer', description: '' }, + ]); + }); + + it('bounds descriptions to top-level properties and respects property overrides', () => { + expect( + metadata(`${server} + server.registerTool('shorthand', { description: 'Kept', title }, handler); + server.registerTool('spread-after', { description: 'Unproven', ...config }, handler); + server.registerTool('spread-before', { ...config, description: 'Known' }, handler); + server.registerTool('last-wins', { description: 'Old', description: 'New' }, handler); + server.registerTool('comments', /* first */ 'not an object', /* callback */ handler); + `), + ).toEqual([ + { toolName: 'shorthand', description: 'Kept' }, + { toolName: 'spread-after', description: '' }, + { toolName: 'spread-before', description: 'Known' }, + { toolName: 'last-wins', description: 'New' }, + { toolName: 'comments', description: '' }, + ]); + }); + + it('resolves hoisted declarations and immutable callable bindings using supplied graph IDs', () => { + const tree = tsParser.parse(`${server} + server.tool('declared', declaration); + function declaration() {} + const arrow = () => {}; + const expression = function () {}; + server.registerTool('arrow', {}, arrow); + server.tool('expression', expression); + server.tool('inline', () => {}); + `); + const bindings = new Map(); + for (const declaration of tree.rootNode.descendantsOfType([ + 'function_declaration', + 'variable_declarator', + ])) { + const name = declaration.childForFieldName('name')!; + bindings.set(name.id, `existing-graph-id:${name.text}`); + } + expect( + extractToolDefinitions(tree, 'server.ts', 0, bindings).map((tool) => [ + tool.toolName, + tool.handlerNodeId, + ]), + ).toEqual([ + ['declared', 'existing-graph-id:declaration'], + ['arrow', 'existing-graph-id:arrow'], + ['expression', 'existing-graph-id:expression'], + ['inline', undefined], + ]); + expect( + extractToolDefinitions(tree, 'server.ts').every((tool) => tool.handlerNodeId === undefined), + ).toBe(true); + }); + + it('uses the nearest callable binding without conflating same-name declarations', () => { + const tree = tsParser.parse(`${server} + function handler() {} + function install() { + const handler = () => {}; + server.tool('inner', handler); + } + server.tool('outer', handler); + `); + const outer = tree.rootNode + .descendantsOfType('function_declaration')[0] + .childForFieldName('name')!; + const inner = tree.rootNode + .descendantsOfType('variable_declarator') + .find((node) => node.childForFieldName('name')?.text === 'handler')! + .childForFieldName('name')!; + const bindings = new Map([ + [outer.id, 'emitted-outer'], + [inner.id, 'emitted-inner'], + ]); + expect( + extractToolDefinitions(tree, 'server.ts', 0, bindings).map((tool) => [ + tool.toolName, + tool.handlerNodeId, + ]), + ).toEqual([ + ['inner', 'emitted-inner'], + ['outer', 'emitted-outer'], + ]); + }); + + it('keeps ambiguous, shadowed, mutable and noncallable handler bindings unresolved', () => { + const tree = tsParser.parse(`${server} + import { imported } from './handlers'; + function handler() {} + function parameter(handler) { server.tool('parameter', handler); } + { const handler = 42; server.tool('shadowed', handler); } + const alias = handler; + server.tool('alias', alias); + server.tool('imported', imported); + let mutable = () => {}; + server.tool('mutable', mutable); + const reassigned = () => {}; + ({ reassigned } = replacements); + server.tool('reassigned', reassigned); + function duplicate() {} + function duplicate() {} + server.tool('duplicate', duplicate); + server.tool('before-initialization', later); + const later = () => {}; + `); + const bindings = new Map(); + // Even graph nodes sharing these names cannot establish a safe callback binding. + for (const node of tree.rootNode.descendantsOfType('identifier')) + bindings.set(node.id, `emitted:${node.text}`); + const tools = extractToolDefinitions(tree, 'server.ts', 0, bindings); + expect(tools.map((tool) => tool.toolName)).toEqual([ + 'parameter', + 'shadowed', + 'alias', + 'imported', + 'mutable', + 'reassigned', + 'duplicate', + 'before-initialization', + ]); + expect( + tools.every((tool) => tool.handlerNodeId === undefined && tool.allowFileFallback === false), + ).toBe(true); + }); +}); diff --git a/gitnexus/test/unit/write-reconciliation.test.ts b/gitnexus/test/unit/write-reconciliation.test.ts new file mode 100644 index 000000000..d4d4a86ea --- /dev/null +++ b/gitnexus/test/unit/write-reconciliation.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it, vi } from 'vitest'; +import { reconcileGraphNodeIdentities } from '../../src/core/incremental/write-reconciliation.js'; +import { buildTestGraph } from '../helpers/test-graph.js'; + +const graph = () => + buildTestGraph( + [ + { + id: 'Function:src/lock.ts:acquire', + label: 'Function', + name: 'acquire', + filePath: 'src/lock.ts', + startLine: 504, + endLine: 719, + }, + { + id: 'Function:bench/measure.mjs:retainedHeapBytes', + label: 'Function', + name: 'retainedHeapBytes', + filePath: 'bench/measure.mjs', + startLine: 437, + endLine: 453, + }, + ], + [], + ); +const rows = () => [ + { + id: 'Function:src/lock.ts:acquire', + name: 'acquire', + filePath: 'src/lock.ts', + startLine: 504, + endLine: 719, + }, + { + id: 'Function:bench/measure.mjs:retainedHeapBytes', + name: 'retainedHeapBytes', + filePath: 'bench/measure.mjs', + startLine: 437, + endLine: 453, + }, +]; +const queryFor = (functions: ReturnType) => + vi.fn(async (query: string) => (query.includes('(n:`Function`)') ? functions : [])); + +describe('incremental identity reconciliation', () => { + it('certifies every identity using one unfiltered query per label', async () => { + const query = queryFor(rows()); + const receipt = await reconcileGraphNodeIdentities(graph(), query, 'post-COPY'); + expect(receipt.nodes).toBe(2); + expect(query).toHaveBeenCalledTimes(receipt.tables); + for (const [cypher] of query.mock.calls) { + expect(cypher).toMatch(/^MATCH \(n:`\w+`\) RETURN/); + expect(cypher).not.toMatch(/WHERE|content/); + } + }); + + it('rejects a missing unchanged symbol', async () => { + await expect( + reconcileGraphNodeIdentities(graph(), queryFor(rows().slice(1)), 'post-COPY'), + ).rejects.toThrow(/1 missing ID.*src\/lock.ts:acquire/); + }); + + it.each(['name', 'filePath', 'startLine', 'endLine'] as const)( + 'rejects column/row inconsistency in %s despite identical ID counts', + async (field) => { + const persisted = rows(); + const swapped = persisted.map((row, index) => ({ + ...row, + [field]: persisted[1 - index][field], + })); + await expect( + reconcileGraphNodeIdentities(graph(), queryFor(swapped), 'pre-publish'), + ).rejects.toThrow(field); + }, + ); + + it('rejects a duplicated row that would hide a missing ID in a count check', async () => { + await expect( + reconcileGraphNodeIdentities(graph(), queryFor([rows()[0], rows()[0]]), 'post-COPY'), + ).rejects.toThrow(/duplicate ID/); + }); + + it('surfaces a failed scan rather than certifying an unreadable graph', async () => { + const query = vi.fn().mockRejectedValue(new Error('Invalid UTF-8')); + await expect(reconcileGraphNodeIdentities(graph(), query, 'pre-publish')).rejects.toThrow( + /could not read identity fields.*Invalid UTF-8/, + ); + }); +}); diff --git a/gitnexus/vendor/lbug-fts/manifest.json b/gitnexus/vendor/lbug-fts/manifest.json index 6d93ddd03..62c4da0c7 100644 --- a/gitnexus/vendor/lbug-fts/manifest.json +++ b/gitnexus/vendor/lbug-fts/manifest.json @@ -1,6 +1,6 @@ { - "coreVersion": "0.18.3", - "extensionVersion": "0.18.1", + "coreVersion": "0.21.1", + "extensionVersion": "0.21.0", "filename": "libfts.lbug_extension", "officialRepo": "https://extension.ladybugdb.com/", "urlTemplate": "{officialRepo}v{extensionVersion}/{upstreamPlatform}/fts/{filename}", diff --git a/gitnexus/vendor/lbug-fts/prebuilds/SHA256SUMS b/gitnexus/vendor/lbug-fts/prebuilds/SHA256SUMS index 3aa968510..9683a3704 100644 --- a/gitnexus/vendor/lbug-fts/prebuilds/SHA256SUMS +++ b/gitnexus/vendor/lbug-fts/prebuilds/SHA256SUMS @@ -1,5 +1,5 @@ -01cf0d4debae1b0c7c182bf78747ee1b148a76667bbaa5bb0cb3cc848998028d ./win32-x64/libfts.lbug_extension -4af2602007a4a02d5b18d0b6ecb20b1cc2f493242a915faf7df1c033136107b1 ./linux-arm64/libfts.lbug_extension -cf687fda0f82bfbe116e775f2f17c39faf45be6300fd1937c2b1afd21142c121 ./linux-x64/libfts.lbug_extension -d3564c05ec28a0293a5488eaff2c06591624ac01a44af183d8cfacd92a29d785 ./darwin-arm64/libfts.lbug_extension -d8589c0a91c6667e959da6a81f712f69b330d0563602933da983edeebee60fc6 ./darwin-x64/libfts.lbug_extension +248332de781a7781f612a55b5397e88f58f1f1ba2a973836ada199eab0f6bf10 ./darwin-arm64/libfts.lbug_extension +742f2756c2f80bcd1886b6ee0446483038cff0ecf6cf47f698bc6fe855cbaef6 ./linux-x64/libfts.lbug_extension +7b6132ae6a554e3eeb3c4b0da7e2eb5c483b258b31e848a0e3d6000ac29885ee ./win32-x64/libfts.lbug_extension +b8675b086664e3d6742330a517c5b619f0ac10dd094d250f12bef9822a8d5750 ./linux-arm64/libfts.lbug_extension +c86f427503555a3ccdc55eb66a8d82793a206aac599a6092f7d1d4eb50c21403 ./darwin-x64/libfts.lbug_extension diff --git a/gitnexus/vendor/lbug-fts/prebuilds/darwin-arm64/libfts.lbug_extension b/gitnexus/vendor/lbug-fts/prebuilds/darwin-arm64/libfts.lbug_extension index 31f85a0da..53bcff048 100644 Binary files a/gitnexus/vendor/lbug-fts/prebuilds/darwin-arm64/libfts.lbug_extension and b/gitnexus/vendor/lbug-fts/prebuilds/darwin-arm64/libfts.lbug_extension differ diff --git a/gitnexus/vendor/lbug-fts/prebuilds/darwin-x64/libfts.lbug_extension b/gitnexus/vendor/lbug-fts/prebuilds/darwin-x64/libfts.lbug_extension index b0fd3b7fd..6fa05c0da 100644 Binary files a/gitnexus/vendor/lbug-fts/prebuilds/darwin-x64/libfts.lbug_extension and b/gitnexus/vendor/lbug-fts/prebuilds/darwin-x64/libfts.lbug_extension differ diff --git a/gitnexus/vendor/lbug-fts/prebuilds/linux-arm64/libfts.lbug_extension b/gitnexus/vendor/lbug-fts/prebuilds/linux-arm64/libfts.lbug_extension index 41f2fa575..bf4c01920 100644 Binary files a/gitnexus/vendor/lbug-fts/prebuilds/linux-arm64/libfts.lbug_extension and b/gitnexus/vendor/lbug-fts/prebuilds/linux-arm64/libfts.lbug_extension differ diff --git a/gitnexus/vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension b/gitnexus/vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension index 0971ff059..f7ab6c269 100644 Binary files a/gitnexus/vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension and b/gitnexus/vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension differ diff --git a/gitnexus/vendor/lbug-fts/prebuilds/win32-x64/libfts.lbug_extension b/gitnexus/vendor/lbug-fts/prebuilds/win32-x64/libfts.lbug_extension index 093f8b7f9..dec711432 100644 Binary files a/gitnexus/vendor/lbug-fts/prebuilds/win32-x64/libfts.lbug_extension and b/gitnexus/vendor/lbug-fts/prebuilds/win32-x64/libfts.lbug_extension differ diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 29d44bf92..37c5889f3 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -68,6 +68,7 @@ export default defineConfig({ include: [ 'test/integration/skip-fts.test.ts', 'test/integration/impact-callable-value-references.test.ts', + 'test/integration/impact-context-integrity.test.ts', 'test/integration/impact-epistemic-lower-bound.test.ts', 'test/integration/impact-scope-omission-persistence.test.ts', 'test/integration/lbug-core-adapter.test.ts', @@ -109,6 +110,7 @@ export default defineConfig({ 'test/integration/analyze-wal-checkpoint-failure.test.ts', 'test/integration/lbug-non-ascii-path.test.ts', 'test/integration/lbug-conn-serialization.test.ts', + 'test/integration/lbug-load-overlap-errors.test.ts', 'test/integration/load-cached-embeddings-spill.test.ts', 'test/integration/group/manifest-resolve-symbol-2325.test.ts', 'test/integration/group/manifest-synthetic-impact-lbug.test.ts', @@ -130,6 +132,8 @@ export default defineConfig({ // fake that answers on `query.includes(...)`. 'test/integration/wiki-graph-queries-engine.test.ts', 'test/unit/incremental-dirty-recovery.test.ts', + // Publication reconciliation uses real native COPY/checkpoints. + 'test/unit/incremental-write-integrity.test.ts', 'test/unit/incremental-orchestration.test.ts', // #2841. Native @ladybugdb/core: it runs real analyses, reopens the // DB under different extension-install policies, and reads @@ -159,6 +163,7 @@ export default defineConfig({ exclude: [ 'test/integration/skip-fts.test.ts', 'test/integration/impact-callable-value-references.test.ts', + 'test/integration/impact-context-integrity.test.ts', 'test/integration/impact-epistemic-lower-bound.test.ts', 'test/integration/impact-scope-omission-persistence.test.ts', 'test/integration/lbug-core-adapter.test.ts', @@ -190,6 +195,7 @@ export default defineConfig({ 'test/integration/analyze-wal-checkpoint-failure.test.ts', 'test/integration/lbug-non-ascii-path.test.ts', 'test/integration/lbug-conn-serialization.test.ts', + 'test/integration/lbug-load-overlap-errors.test.ts', 'test/integration/load-cached-embeddings-spill.test.ts', 'test/integration/group/manifest-resolve-symbol-2325.test.ts', 'test/integration/group/manifest-synthetic-impact-lbug.test.ts', @@ -206,6 +212,7 @@ export default defineConfig({ 'test/integration/detect-changes-path-anchoring.test.ts', 'test/integration/wiki-graph-queries-engine.test.ts', 'test/unit/incremental-dirty-recovery.test.ts', + 'test/unit/incremental-write-integrity.test.ts', 'test/unit/incremental-orchestration.test.ts', // Excluded here because it is included by `lbug-db` above; a file // in two projects would be collected (and run) twice.