diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 92c9887a0..584d6befe 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -648,6 +648,24 @@ jobs: done working-directory: gitnexus + # Native embedding replacement measures real COPY, writes and cold reopens. + # Keep it outside the 25-minute language benchmark job: four trials at each + # size take several minutes even on the development baseline machine. + embedding-identity-benchmark: + name: embedding identity persistence benchmark + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/setup-gitnexus + with: + build: 'true' + - name: Embedding identity persistence / scaling guards (#3569) + run: node --import tsx bench/embedding-identities/measure.cjs --check + working-directory: gitnexus + # ── Dedicated benchmark gate ───────────────────────────────────── # The cross-language `*-pipeline-benchmark.test.ts` suites are gated behind # GITNEXUS_BENCH (they generate synthetic codebases at scale), so the main @@ -1033,6 +1051,11 @@ jobs: run: node --expose-gc --import tsx bench/cfg/measure.mjs --check working-directory: gitnexus + - name: Staged publication durability / scaling guards (#3591) + if: ${{ !cancelled() }} + run: node --import tsx bench/staged-publication/measure.cjs --check + working-directory: gitnexus + - name: Emit-persistence throughput / byte-identity guards (#2203) if: ${{ !cancelled() }} # Build-free: asserts streamAllCSVsToDisk output is byte-identical diff --git a/eval/tests/test_oracle_assets.py b/eval/tests/test_oracle_assets.py index 29ebda2ee..6314d99b8 100644 --- a/eval/tests/test_oracle_assets.py +++ b/eval/tests/test_oracle_assets.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import hashlib import os +import re import shutil import subprocess import sys @@ -589,6 +590,9 @@ def scenario_control_base(tmp_path_factory: pytest.TempPathFactory) -> tuple[Pat ref, "gitnexus/src", "gitnexus/vendor", + "gitnexus/test", + "gitnexus/tsconfig.json", + "gitnexus/vitest.config.ts", "gitnexus/package.json", "gitnexus-shared/package.json", ], @@ -607,7 +611,7 @@ def scenario_control_base(tmp_path_factory: pytest.TempPathFactory) -> tuple[Pat pytest.skip("oracle controls need tarfile extraction filters (Python 3.11.4+)") with tarfile.open(archive) as source: source.extractall(base, filter="data") - for relative in ("node_modules", "gitnexus/node_modules", "gitnexus-shared/dist"): + for relative in ("node_modules", "gitnexus/node_modules", "gitnexus-shared/node_modules", "gitnexus-shared/dist"): (base / relative).symlink_to(dependencies / relative, target_is_directory=True) subprocess.run(["git", "init", "--quiet", str(base)], check=True, capture_output=True) return base, {task["id"]: task for task in tasks} @@ -633,12 +637,82 @@ def _verify_shipped_oracle(candidate: Path, task: dict[str, object]) -> subproce ) +def _assert_behavioral_rejection(completed: subprocess.CompletedProcess[str], assertion: str = "") -> None: + output = completed.stdout + completed.stderr + assert completed.returncode == 1, output + assert "AssertionError" in output, output + assert assertion in output, output + # An import/setup crash is not evidence that a behavioral control worked. + assert "Failed Suites" not in output, output + assert "Unhandled Errors" not in output, output + + +def _check_c_provenance_mutants(candidate: Path, base: Path, task: dict[str, object]) -> None: + """Reject broken behavior, and distinguish module-load errors from evidence.""" + + interpret = Path("gitnexus/src/core/ingestion/languages/c/interpret.ts") + resolver = Path("gitnexus/src/core/ingestion/languages/c/scope-resolver.ts") + valid_interpret = (candidate / interpret).read_text() + valid_resolver = (candidate / resolver).read_text() + system_guards = re.findall( + r"\bconst\s+parsedImport\s*=\s*context\s*\?\.\s*parsedImport\s*;\s*" + r"if\s*\(\s*parsedImport\s*\?\.\s*kind\s*===\s*'wildcard'\s*&&\s*" + r"'isSystem'\s+in\s+parsedImport\s*&&\s*parsedImport\s*\.\s*isSystem\s*===\s*true\s*\)\s*" + r"\{\s*return\s+null\s*;\s*\}", + valid_resolver, + ) + fallbacks = re.findall( + r"\breturn\s+resolveCImportTarget\s*\(\s*targetRaw\s*,\s*fromFile\s*,\s*allFilePaths\s*,?\s*\)\s*;", + valid_resolver, + ) + assert len(system_guards) == 1, "expected exactly one C system-include guard" + assert len(fallbacks) == 1, "expected exactly one C resolver fallback" + system_guard, fallback = system_guards[0], fallbacks[0] + late_guard_resolver = valid_resolver.replace(system_guard, "").replace(fallback, system_guard + "\n " + fallback) + assert valid_interpret.count("return parsedImport;") == 1 + assert valid_interpret.count("targetRaw: source,") == 1 + mutants = [ + (interpret, (base / interpret).read_text(), "literal include must be retained"), + ( + interpret, + valid_interpret.replace("return parsedImport;", "return null;"), + 'literal include "stdio.h" must be retained', + ), + (interpret, valid_interpret.replace("targetRaw: source,", "targetRaw: '',"), "retained header identity"), + (resolver, (base / resolver).read_text(), "include resolution"), + (resolver, late_guard_resolver, "include resolution"), + ] + for relative, broken, assertion in mutants: + path = candidate / relative + original = path.read_text() + assert broken != original + path.write_text(broken) + try: + _assert_behavioral_rejection(_verify_shipped_oracle(candidate, task), assertion) + finally: + path.write_text(original) + + # A missing native-parser module must fail setup, never satisfy a negative + # behavioral control just because Vitest exits nonzero. + parser = candidate / "gitnexus/src/core/ingestion/languages/c/query.ts" + original_parser = parser.read_bytes() + parser.unlink() + try: + broken_setup = _verify_shipped_oracle(candidate, task) + assert "Failed Suites" in broken_setup.stdout + broken_setup.stderr + with pytest.raises(AssertionError): + _assert_behavioral_rejection(broken_setup, "literal include must be retained") + finally: + parser.write_bytes(original_parser) + + @pytest.mark.skipif(os.name == "nt", reason="oracle controls use POSIX dependency symlinks and shell commands") @pytest.mark.parametrize( ("task_id", "reference"), [ ("trivial-status-json-alias", "trivial-status-json-alias"), - ("inv-bug-c-system-include", "inv-bug-c-system-include"), + ("inv-feature-c-include-provenance", "inv-feature-c-include-provenance-flag"), + ("inv-feature-c-include-provenance", "inv-feature-c-include-provenance-delimiter"), ("inv-feature-list-repos-filter", "inv-feature-list-repos-filter"), ("cross-module-parse-retry", "cross-module-parse-retry"), ("cross-module-parse-retry", "cross-module-parse-retry-pool"), @@ -653,13 +727,37 @@ def test_shipped_oracles_reject_task_base_and_accept_reference_fixes( base, tasks = scenario_control_base candidate = _oracle_control_candidate(base, tmp_path / "candidate") negative = _verify_shipped_oracle(candidate, tasks[task_id]) - assert negative.returncode == 1, negative.stdout + negative.stderr - assert "AssertionError" in negative.stdout + negative.stderr + _assert_behavioral_rejection( + negative, + "literal include must be retained" if task_id == "inv-feature-c-include-provenance" else "", + ) patches = Path(__file__).resolve().parents[1] / "workflow_bench/oracles/reference" subprocess.run(["git", "apply", str(patches / f"{reference}.patch")], cwd=candidate, check=True) positive = _verify_shipped_oracle(candidate, tasks[task_id]) assert positive.returncode == 0, positive.stdout + positive.stderr + if task_id == "inv-feature-c-include-provenance": + # A reference passing transpile-only Vitest is not enough: use the same + # typed, authored verification that an evaluated repair must satisfy. + authored = subprocess.run( + ["/bin/sh", "-c", str(tasks[task_id]["verify"])], + cwd=candidate, + capture_output=True, + text=True, + check=False, + timeout=180, + ) + assert authored.returncode == 0, authored.stdout + authored.stderr + if reference == "inv-feature-c-include-provenance-flag": + # This entry point is already selected by native CI; keep every C + # validity control reachable without changing workflow selection. + # Indent every existing code line differently without matching the + # reference's formatting; the behavioral controls must still work. + resolver = candidate / "gitnexus/src/core/ingestion/languages/c/scope-resolver.ts" + resolver.write_text(re.sub(r"(?m)^([ \t]+)", r"\t\1", resolver.read_text())) + positive = _verify_shipped_oracle(candidate, tasks[task_id]) + assert positive.returncode == 0, positive.stdout + positive.stderr + _check_c_provenance_mutants(candidate, base, tasks[task_id]) @pytest.mark.skipif(os.name == "nt", reason="oracle controls use POSIX dependency symlinks and shell commands") diff --git a/eval/tests/test_workflow_bench.py b/eval/tests/test_workflow_bench.py index 51387fcde..3f4ed5d30 100644 --- a/eval/tests/test_workflow_bench.py +++ b/eval/tests/test_workflow_bench.py @@ -347,11 +347,18 @@ def test_shipped_scenarios_opt_out_the_cross_module_cell_and_rebuild_graph_asset selected, skipped = select_tasks(tasks, include_expensive=False) assert [task["id"] for task in selected] == [ "trivial-status-json-alias", - "inv-bug-c-system-include", + "inv-feature-c-include-provenance", "inv-feature-list-repos-filter", ] assert skipped == ["cross-module-parse-retry"] - # v1.6.12 predates all four fixes, including the C fix already on main. + assert len(tasks) == 4 + c_task = next(task for task in tasks if task["id"] == "inv-feature-c-include-provenance") + assert c_task["class"] == "investigation-feature" + assert "npx tsc --noEmit" in c_task["verify"] + assert "test/unit/scope-resolution/c/c-include-provenance.test.ts" in c_task["verify"] + assert "test/unit/scope-resolution/external-import-conformance.test.ts" in c_task["verify"] + assert not c_task.get("setup") + # v1.6.12 lacks all four requested behaviors, including retained C include provenance. assert {task["ref"] for task in tasks} == {"c4ecf398de637cf779ab2238e69958cfd5d601ce"} assert all(not task.get("sandbox_copy") for task in tasks) assert all(task["sandbox_dependencies"] for task in tasks) diff --git a/eval/workflow_bench/README.md b/eval/workflow_bench/README.md index e8002e219..a972721d0 100644 --- a/eval/workflow_bench/README.md +++ b/eval/workflow_bench/README.md @@ -786,6 +786,30 @@ the shared `vitest.config.mts` as an oracle file and pass it explicitly with `--config`. Prefer `verify` commands that use the repo's own npm scripts (they carry build pre-hooks). +The C scenario is `inv-feature-c-include-provenance`. At its pinned task +commit, ingestion drops literal system includes before resolution. The task +requires retaining the header identity and enough provenance to prevent +unconfigured system includes from matching unrelated repository headers, +while keeping quoted local and relative includes resolvable. Computed macro +includes remain outside this task's scope. Its hidden oracle parses real C +source and passes the resulting import through the decomposer, interpreter, +and resolver; it does not require a particular metadata field or encoding. + +Offline controls must show that the untouched task base fails the intended +behavioral assertion and that two independent, type-correct repairs pass both +the hidden oracle and the complete authored verification command. Negative +controls also reject dropped imports, lost header identity, and false system +header matches. These controls run with the pinned task dependencies through +`test_shipped_oracles_reject_task_base_and_accept_reference_fixes`. + +This replaces `inv-bug-c-system-include`, whose oracle constructed an import +state the pinned parser did not produce and constrained the repair's internal +representation. Historical scores for that task are not valid evidence of C +ingestion quality and are not comparable with the replacement task. The new +task ID, prompt, verification command, and oracle bytes produce a new task +digest. Old agent patches are not available for rescoring; keep historical +measurements unchanged and compare only runs with matching task contracts. + ## Relation to the SWE-bench harness The rest of `eval/` benchmarks GitNexus _tools_ inside a litellm agent loop diff --git a/eval/workflow_bench/oracles/inv-bug-c-system-include.oracle.test.ts b/eval/workflow_bench/oracles/inv-bug-c-system-include.oracle.test.ts deleted file mode 100644 index 082dc4990..000000000 --- a/eval/workflow_bench/oracles/inv-bug-c-system-include.oracle.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { cScopeResolver } from '../gitnexus/src/core/ingestion/languages/c/scope-resolver.js'; - -describe('hidden oracle: C system headers do not bind to repository decoys', () => { - it.each([ - ['stdio.h', true, null], - ['project.h', true, null], - ['stdio.h', false, 'src/stdio.h'], - ['util.h', false, 'include/util.h'], - ] as const)( - 'resolves %s with isSystem=%s from its include form', - (targetRaw, isSystem, expected) => { - const files = new Set(['src/stdio.h', 'include/project.h', 'include/util.h', 'src/main.c']); - const parsedImport = { kind: 'wildcard' as const, targetRaw, isSystem }; - - // Both spellings name stdio.h. Preserve the parsed include form; - // refusing every header by name would break the quoted local include. - const resolved = cScopeResolver.resolveImportTarget( - parsedImport.targetRaw, - 'src/main.c', - files, - undefined, - { parsedFiles: [], parsedImport }, - ); - expect(resolved).toBe(expected); - }, - ); -}); diff --git a/eval/workflow_bench/oracles/inv-feature-c-include-provenance.oracle.test.ts b/eval/workflow_bench/oracles/inv-feature-c-include-provenance.oracle.test.ts new file mode 100644 index 000000000..bdab94bb9 --- /dev/null +++ b/eval/workflow_bench/oracles/inv-feature-c-include-provenance.oracle.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; +import { getCParser } from '../gitnexus/src/core/ingestion/languages/c/query.js'; +import { splitCInclude } from '../gitnexus/src/core/ingestion/languages/c/import-decomposer.js'; +import { interpretCImport } from '../gitnexus/src/core/ingestion/languages/c/interpret.js'; +import { cScopeResolver } from '../gitnexus/src/core/ingestion/languages/c/scope-resolver.js'; + +function parseInclude(spelling: string) { + const tree = getCParser().parse(`#include ${spelling}\n`); + const node = tree.rootNode.namedChildren.find((child) => child.type === 'preproc_include'); + if (node === undefined) throw new Error('native parser must produce an include node'); + const captures = splitCInclude(node); + return captures === null ? null : interpretCImport(captures); +} + +// Header identity is public behavior. Bare targets or targets carrying their +// literal delimiters are both valid; no private marker field is prescribed. +function headerIdentity(target: string): string { + if ( + (target.startsWith('<') && target.endsWith('>')) || + (target.startsWith('"') && target.endsWith('"')) + ) + return target.slice(1, -1); + return target; +} + +const workspaceFiles = new Set([ + 'src/main.c', + 'src/stdio.h', + 'include/project.h', + 'src/local.h', + 'include/util.h', +]); + +describe.each([ + { name: 'workspace headers', files: workspaceFiles, resolutionConfig: undefined }, + { + name: 'discovered headers', + files: new Set(['src/main.c']), + resolutionConfig: new Set([...workspaceFiles].filter((file) => file.endsWith('.h'))), + }, +])('hidden oracle: retained C include provenance ($name)', ({ files, resolutionConfig }) => { + it.each([ + ['', 'stdio.h', null], + ['', 'project.h', null], + ['"stdio.h"', 'stdio.h', 'src/stdio.h'], + ['"local.h"', 'local.h', 'src/local.h'], + ['"../include/util.h"', '../include/util.h', 'include/util.h'], + ] as const)('retains and resolves literal %s', (spelling, header, expected) => { + const parsedImport = parseInclude(spelling); + expect(parsedImport, `literal include ${spelling} must be retained`).not.toBeNull(); + expect(parsedImport!.kind).toBe('wildcard'); + expect(headerIdentity(parsedImport!.targetRaw), 'retained header identity').toBe(header); + const resolved = cScopeResolver.resolveImportTarget( + parsedImport!.targetRaw, + 'src/main.c', + files, + resolutionConfig, + { parsedFiles: [], parsedImport: parsedImport! }, + ); + const targets = resolved === null ? [] : typeof resolved === 'string' ? [resolved] : resolved; + expect(targets, `include ${spelling} resolution`).toEqual(expected === null ? [] : [expected]); + }); + + it('does not invent a literal import for a computed macro include', () => { + expect(parseInclude('HEADER_MACRO')).toBeNull(); + }); +}); diff --git a/eval/workflow_bench/oracles/reference/inv-bug-c-system-include.patch b/eval/workflow_bench/oracles/reference/inv-bug-c-system-include.patch deleted file mode 100644 index 49ac341fc..000000000 --- a/eval/workflow_bench/oracles/reference/inv-bug-c-system-include.patch +++ /dev/null @@ -1,12 +0,0 @@ ---- a/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts -+++ b/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts -@@ -81,7 +81,8 @@ - // this process. Runs BEFORE `populateOwners`. - applyCaptureSideChannel: applyCStaticLinkageSideChannel, - -- resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => { -+ resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig, context) => { -+ if (context?.parsedImport?.kind === 'wildcard' && context.parsedImport.isSystem) return null; - // Augment allFilePaths with .h files discovered via loadResolutionConfig - // since the phase only passes .c files to the C resolver but #include - // targets .h files classified as C++ in language detection. diff --git a/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-delimiter.patch b/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-delimiter.patch new file mode 100644 index 000000000..be802e723 --- /dev/null +++ b/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-delimiter.patch @@ -0,0 +1,146 @@ +--- a/gitnexus/src/core/ingestion/languages/c/interpret.ts ++++ b/gitnexus/src/core/ingestion/languages/c/interpret.ts +@@ -8,10 +8,9 @@ + const source = captures['@import.source']?.text; + if (source === undefined) return null; + +- // System headers (e.g. ) are not resolved to local files +- if (captures['@import.system'] !== undefined) return null; +- +- return { kind: 'wildcard', targetRaw: source }; ++ // Preserve the literal spelling in the existing target field. ++ const targetRaw = captures['@import.system'] === undefined ? source : `<${source}>`; ++ return { kind: 'wildcard', targetRaw }; + } + + /** +--- a/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts ++++ b/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts +@@ -82,6 +82,7 @@ + applyCaptureSideChannel: applyCStaticLinkageSideChannel, + + resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => { ++ if (targetRaw.startsWith('<') && targetRaw.endsWith('>')) return null; + // Augment allFilePaths with .h files discovered via loadResolutionConfig + // since the phase only passes .c files to the C resolver but #include + // targets .h files classified as C++ in language detection. +--- a/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts ++++ b/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts +@@ -38,6 +38,10 @@ + import { describe, expect, it } from 'vitest'; + import type { ParsedFile, ParsedImport, ScopeId, SymbolDefinition } from 'gitnexus-shared'; + import { SupportedLanguages } from 'gitnexus-shared'; ++import { getCParser } from '../../../src/core/ingestion/languages/c/query.js'; ++import { splitCInclude } from '../../../src/core/ingestion/languages/c/import-decomposer.js'; ++import { interpretCImport } from '../../../src/core/ingestion/languages/c/interpret.js'; ++import type { SyntaxNode } from '../../../src/core/ingestion/utils/ast-helpers.js'; + import { SCOPE_RESOLVERS } from '../../../src/core/ingestion/scope-resolution/pipeline/registry.js'; + import type { ComposerConfig } from '../../../src/core/ingestion/language-config.js'; + import { +@@ -143,6 +147,16 @@ + }; + } + ++function parseCInclude(spelling: string): ParsedImport { ++ const tree = getCParser().parse(`#include ${spelling}\n`); ++ const node = tree.rootNode.namedChildren.find((child) => child.type === 'preproc_include'); ++ if (!node) throw new Error('C conformance fixture did not parse'); ++ const captures = splitCInclude(node as SyntaxNode); ++ const parsedImport = captures === null ? null : interpretCImport(captures); ++ if (parsedImport === null) throw new Error('C literal include was not retained'); ++ return parsedImport; ++} ++ + const PHP_FUNCTION_IMPORT = (targetRaw: string): ParsedImport => ({ + kind: 'named', + localName: 'imported', +@@ -345,9 +359,10 @@ + files: ['src/stdio.h', 'include/util.h', 'src/main.c'], + fromFile: 'src/main.c', + resolutionConfig: undefined, +- external: 'stdio.h', ++ external: '', + decoy: 'src/stdio.h', +- reachesDecoy: 'util.h', ++ reachesDecoy: '"stdio.h"', ++ parsedImport: parseCInclude, + }, + ], + [ +@@ -418,7 +433,6 @@ + const KNOWN_GAPS: ReadonlyMap = new Map([ + [SupportedLanguages.Dart, '`package:http/http.dart` -> `lib/http.dart`'], + [SupportedLanguages.Swift, '`Foundation` -> `Sources/Foundation/Thing.swift`'], +- [SupportedLanguages.C, '`stdio.h` -> `src/stdio.h`'], + [SupportedLanguages.CPlusPlus, '`cstdio.h` -> `src/cstdio.h`'], + ]); + +@@ -463,14 +477,15 @@ + : testCase.files.map( + testCase.parsedFile ?? ((filePath) => ({ filePath }) as unknown as ParsedFile), + ); ++ const parsedImport = testCase.parsedImport?.(targetRaw); + return resolver.resolveImportTarget( +- targetRaw, ++ parsedImport?.targetRaw ?? targetRaw, + testCase.fromFile, + files, + testCase.resolutionConfig, + { + parsedFiles, +- parsedImport: testCase.parsedImport?.(targetRaw), ++ parsedImport, + }, + ); + } +--- /dev/null ++++ b/gitnexus/test/unit/scope-resolution/c/c-include-provenance.test.ts +@@ -0,0 +1,48 @@ ++import { describe, expect, it } from 'vitest'; ++import { getCParser } from '../../../../src/core/ingestion/languages/c/query.js'; ++import { splitCInclude } from '../../../../src/core/ingestion/languages/c/import-decomposer.js'; ++import { interpretCImport } from '../../../../src/core/ingestion/languages/c/interpret.js'; ++import { cScopeResolver } from '../../../../src/core/ingestion/languages/c/scope-resolver.js'; ++import type { SyntaxNode } from '../../../../src/core/ingestion/utils/ast-helpers.js'; ++ ++function parseIncludes(source: string) { ++ return getCParser().parse(source).rootNode.namedChildren ++ .filter((node) => node.type === 'preproc_include') ++ .map((node) => { ++ const captures = splitCInclude(node as SyntaxNode); ++ return captures === null ? null : interpretCImport(captures); ++ }); ++} ++ ++function headerIdentity(target: string): string { ++ if ((target.startsWith('<') && target.endsWith('>')) || ++ (target.startsWith('"') && target.endsWith('"'))) return target.slice(1, -1); ++ return target; ++} ++ ++describe('C literal include provenance through parsing and resolution', () => { ++ it('retains system and quoted headers, resolving only local includes', () => { ++ const imports = parseIncludes([ ++ '#include ', ++ '#include "api.h"', ++ '#include "../include/common.h"', ++ ].join('\n') + '\n'); ++ expect(imports).toHaveLength(3); ++ expect(imports.every((item) => item !== null)).toBe(true); ++ expect(imports.map((item) => headerIdentity(item!.targetRaw))) ++ .toEqual(['sys/net.h', 'api.h', '../include/common.h']); ++ const files = new Set(['src/main.c', 'include/sys/net.h', 'src/api.h', 'include/common.h']); ++ const targets = imports.map((parsedImport) => { ++ const resolved = cScopeResolver.resolveImportTarget( ++ parsedImport!.targetRaw, 'src/main.c', files, undefined, ++ { parsedFiles: [], parsedImport: parsedImport! }, ++ ); ++ return resolved === null ? [] : typeof resolved === 'string' ? [resolved] : resolved; ++ }); ++ expect(targets).toEqual([[], ['src/api.h'], ['include/common.h']]); ++ }); ++ ++ it('does not treat a computed macro as a literal header path', () => { ++ expect(parseIncludes('#include HEADER_MACRO\n')).toEqual([null]); ++ }); ++}); diff --git a/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-flag.patch b/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-flag.patch new file mode 100644 index 000000000..726aebe18 --- /dev/null +++ b/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-flag.patch @@ -0,0 +1,155 @@ +--- a/gitnexus/src/core/ingestion/languages/c/interpret.ts ++++ b/gitnexus/src/core/ingestion/languages/c/interpret.ts +@@ -8,10 +8,13 @@ + const source = captures['@import.source']?.text; + if (source === undefined) return null; + +- // System headers (e.g. ) are not resolved to local files +- if (captures['@import.system'] !== undefined) return null; +- +- return { kind: 'wildcard', targetRaw: source }; ++ // Keep provenance locally without changing immutable shared declarations. ++ const parsedImport: ParsedImport & { isSystem: boolean } = { ++ kind: 'wildcard', ++ targetRaw: source, ++ isSystem: captures['@import.system'] !== undefined, ++ }; ++ return parsedImport; + } + + /** +--- a/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts ++++ b/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts +@@ -81,7 +81,11 @@ + // this process. Runs BEFORE `populateOwners`. + applyCaptureSideChannel: applyCStaticLinkageSideChannel, + +- resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => { ++ resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig, context) => { ++ const parsedImport = context?.parsedImport; ++ if (parsedImport?.kind === 'wildcard' && 'isSystem' in parsedImport && parsedImport.isSystem === true) { ++ return null; ++ } + // Augment allFilePaths with .h files discovered via loadResolutionConfig + // since the phase only passes .c files to the C resolver but #include + // targets .h files classified as C++ in language detection. +--- a/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts ++++ b/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts +@@ -38,6 +38,10 @@ + import { describe, expect, it } from 'vitest'; + import type { ParsedFile, ParsedImport, ScopeId, SymbolDefinition } from 'gitnexus-shared'; + import { SupportedLanguages } from 'gitnexus-shared'; ++import { getCParser } from '../../../src/core/ingestion/languages/c/query.js'; ++import { splitCInclude } from '../../../src/core/ingestion/languages/c/import-decomposer.js'; ++import { interpretCImport } from '../../../src/core/ingestion/languages/c/interpret.js'; ++import type { SyntaxNode } from '../../../src/core/ingestion/utils/ast-helpers.js'; + import { SCOPE_RESOLVERS } from '../../../src/core/ingestion/scope-resolution/pipeline/registry.js'; + import type { ComposerConfig } from '../../../src/core/ingestion/language-config.js'; + import { +@@ -143,6 +147,16 @@ + }; + } + ++function parseCInclude(spelling: string): ParsedImport { ++ const tree = getCParser().parse(`#include ${spelling}\n`); ++ const node = tree.rootNode.namedChildren.find((child) => child.type === 'preproc_include'); ++ if (!node) throw new Error('C conformance fixture did not parse'); ++ const captures = splitCInclude(node as SyntaxNode); ++ const parsedImport = captures === null ? null : interpretCImport(captures); ++ if (parsedImport === null) throw new Error('C literal include was not retained'); ++ return parsedImport; ++} ++ + const PHP_FUNCTION_IMPORT = (targetRaw: string): ParsedImport => ({ + kind: 'named', + localName: 'imported', +@@ -345,9 +359,10 @@ + files: ['src/stdio.h', 'include/util.h', 'src/main.c'], + fromFile: 'src/main.c', + resolutionConfig: undefined, +- external: 'stdio.h', ++ external: '', + decoy: 'src/stdio.h', +- reachesDecoy: 'util.h', ++ reachesDecoy: '"stdio.h"', ++ parsedImport: parseCInclude, + }, + ], + [ +@@ -418,7 +433,6 @@ + const KNOWN_GAPS: ReadonlyMap = new Map([ + [SupportedLanguages.Dart, '`package:http/http.dart` -> `lib/http.dart`'], + [SupportedLanguages.Swift, '`Foundation` -> `Sources/Foundation/Thing.swift`'], +- [SupportedLanguages.C, '`stdio.h` -> `src/stdio.h`'], + [SupportedLanguages.CPlusPlus, '`cstdio.h` -> `src/cstdio.h`'], + ]); + +@@ -463,14 +477,15 @@ + : testCase.files.map( + testCase.parsedFile ?? ((filePath) => ({ filePath }) as unknown as ParsedFile), + ); ++ const parsedImport = testCase.parsedImport?.(targetRaw); + return resolver.resolveImportTarget( +- targetRaw, ++ parsedImport?.targetRaw ?? targetRaw, + testCase.fromFile, + files, + testCase.resolutionConfig, + { + parsedFiles, +- parsedImport: testCase.parsedImport?.(targetRaw), ++ parsedImport, + }, + ); + } +--- /dev/null ++++ b/gitnexus/test/unit/scope-resolution/c/c-include-provenance.test.ts +@@ -0,0 +1,48 @@ ++import { describe, expect, it } from 'vitest'; ++import { getCParser } from '../../../../src/core/ingestion/languages/c/query.js'; ++import { splitCInclude } from '../../../../src/core/ingestion/languages/c/import-decomposer.js'; ++import { interpretCImport } from '../../../../src/core/ingestion/languages/c/interpret.js'; ++import { cScopeResolver } from '../../../../src/core/ingestion/languages/c/scope-resolver.js'; ++import type { SyntaxNode } from '../../../../src/core/ingestion/utils/ast-helpers.js'; ++ ++function parseIncludes(source: string) { ++ return getCParser().parse(source).rootNode.namedChildren ++ .filter((node) => node.type === 'preproc_include') ++ .map((node) => { ++ const captures = splitCInclude(node as SyntaxNode); ++ return captures === null ? null : interpretCImport(captures); ++ }); ++} ++ ++function headerIdentity(target: string): string { ++ if ((target.startsWith('<') && target.endsWith('>')) || ++ (target.startsWith('"') && target.endsWith('"'))) return target.slice(1, -1); ++ return target; ++} ++ ++describe('C literal include provenance through parsing and resolution', () => { ++ it('retains system and quoted headers, resolving only local includes', () => { ++ const imports = parseIncludes([ ++ '#include ', ++ '#include "api.h"', ++ '#include "../include/common.h"', ++ ].join('\n') + '\n'); ++ expect(imports).toHaveLength(3); ++ expect(imports.every((item) => item !== null)).toBe(true); ++ expect(imports.map((item) => headerIdentity(item!.targetRaw))) ++ .toEqual(['sys/net.h', 'api.h', '../include/common.h']); ++ const files = new Set(['src/main.c', 'include/sys/net.h', 'src/api.h', 'include/common.h']); ++ const targets = imports.map((parsedImport) => { ++ const resolved = cScopeResolver.resolveImportTarget( ++ parsedImport!.targetRaw, 'src/main.c', files, undefined, ++ { parsedFiles: [], parsedImport: parsedImport! }, ++ ); ++ return resolved === null ? [] : typeof resolved === 'string' ? [resolved] : resolved; ++ }); ++ expect(targets).toEqual([[], ['src/api.h'], ['include/common.h']]); ++ }); ++ ++ it('does not treat a computed macro as a literal header path', () => { ++ expect(parseIncludes('#include HEADER_MACRO\n')).toEqual([null]); ++ }); ++}); diff --git a/eval/workflow_bench/tasks.scenarios.yaml b/eval/workflow_bench/tasks.scenarios.yaml index 08ec59699..98f532aed 100644 --- a/eval/workflow_bench/tasks.scenarios.yaml +++ b/eval/workflow_bench/tasks.scenarios.yaml @@ -3,7 +3,7 @@ # Task fields: # id: short slug used in reports # class: task class for cost/quality routing analysis. The suite spans: -# trivial → investigation-bug → investigation-feature → cross-module +# trivial → investigation-feature → cross-module # repo: path to a git repo that is GitNexus-indexed # ref: git ref benchmarked (fresh detached worktree per arm per run) # setup: optional shell command run in the fresh worktree first @@ -29,7 +29,7 @@ # prebuilt .gitnexus directory is never copied. Three declared dependency trees # are mounted read-only; missing assets fail before model execution. # Every task starts at the exact v1.6.12 commit, where all four tasks remain -# unsolved. In particular, moving main already contains the C include fix. +# unsolved. The C provenance feature replaces the invalid historical system-include task. tasks: # Overhead floor — measured 2026-07-11 (see README calibration): the @@ -67,30 +67,39 @@ tasks: - source: trivial-status-json-alias.oracle.test.ts target: trivial-status-json-alias.oracle.test.ts - # Investigation-heavy bug (#2965): requires following the C import marker - # through the shared scope-resolution pipeline without breaking local - # quoted-header resolution. - - id: inv-bug-c-system-include - class: investigation-bug + # Investigation-heavy feature: retain literal include provenance through + # parsing and resolution without binding system headers to repository decoys. + - id: inv-feature-c-include-provenance + class: investigation-feature repo: ~/GitNexus ref: c4ecf398de637cf779ab2238e69958cfd5d601ce sandbox_dependencies: *gitnexus_sandbox_dependencies prompt: > - Fix the C scope resolver so a system include such as never - resolves to an unrelated same-named header inside the repository, while - quoted/local includes still resolve. Remove C from the conformance - suite's KNOWN_GAPS and cover the negative and positive cases there. - verify: cd gitnexus && npx tsc --noEmit && npx vitest run test/unit/scope-resolution/external-import-conformance.test.ts + Retain literal C #include records after parsing and interpretation, including + system includes such as . Preserve each header's identity and enough + provenance to distinguish system from quoted includes. With no configured + system include roots, system includes must never resolve to unrelated + same-named repository headers; quoted sibling and relative includes must + still resolve. Computed macro includes remain unsupported and produce no + parsed import. Cover the real parser, include decomposition, interpretation + and resolver in test/unit/scope-resolution/c/c-include-provenance.test.ts. + Correct the C conformance fixture to distinguish system and quoted forms of + the same header, and remove C from KNOWN_GAPS. Choose the representation; + shared dependency declarations are immutable. + verify: >- + cd gitnexus && npx tsc --noEmit && npx vitest run + test/unit/scope-resolution/c/c-include-provenance.test.ts + test/unit/scope-resolution/external-import-conformance.test.ts oracle: command: >- cd gitnexus && ./node_modules/.bin/vitest run --config "$GITNEXUS_BENCH_ORACLE_ROOT/vitest.config.mts" - "$GITNEXUS_BENCH_ORACLE_ROOT/inv-bug-c-system-include.oracle.test.ts" + "$GITNEXUS_BENCH_ORACLE_ROOT/inv-feature-c-include-provenance.oracle.test.ts" files: - source: vitest.config.mts target: vitest.config.mts - - source: inv-bug-c-system-include.oracle.test.ts - target: inv-bug-c-system-include.oracle.test.ts + - source: inv-feature-c-include-provenance.oracle.test.ts + target: inv-feature-c-include-provenance.oracle.test.ts # Investigation-heavy feature: touches tool schema, backend filtering, and # pagination totals — three seams that must stay consistent. diff --git a/gitnexus/bench/embedding-identities/baseline.json b/gitnexus/bench/embedding-identities/baseline.json new file mode 100644 index 000000000..38eb961b1 --- /dev/null +++ b/gitnexus/bench/embedding-identities/baseline.json @@ -0,0 +1,25 @@ +{ + "version": 1, + "linear_scaling_budget": 2, + "sizes": [1024, 4096], + "dimensions": 8, + "batch_size": 128, + "results": [ + { + "nodes": 1024, + "copy_ms": 85.39003699999012, + "replace_ms": 23018.108886999995 + }, + { + "nodes": 4096, + "copy_ms": 94.20082500006538, + "replace_ms": 95806.77968499996 + } + ], + "runtime": { + "node": "v24.19.0", + "platform": "linux", + "cpu": "12th Gen Intel(R) Core(TM) i9-12900K" + }, + "regression_budget": 1.3 +} diff --git a/gitnexus/bench/embedding-identities/measure.cjs b/gitnexus/bench/embedding-identities/measure.cjs new file mode 100644 index 000000000..c8ec59317 --- /dev/null +++ b/gitnexus/bench/embedding-identities/measure.cjs @@ -0,0 +1,137 @@ +#!/usr/bin/env node +// Real CSV/COPY and composite-key replacement; no mocked persistence. +// node --import tsx bench/embedding-identities/measure.cjs [--check] +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const { pathToFileURL } = require('node:url'); +const root = path.resolve(__dirname, '../..'); +const source = (name) => import(pathToFileURL(path.join(root, 'src', name)).href); +(async () => { + const baseline = JSON.parse(await fs.readFile(path.join(__dirname, 'baseline.json'), 'utf8')); + assert.equal(baseline.version, 1); + process.env.GITNEXUS_EMBEDDING_DIMS = String(baseline.dimensions); + const adapter = await source('core/lbug/lbug-adapter.ts'); + const { batchInsertEmbeddings } = await source('core/embeddings/embedding-pipeline.ts'); + const { createKnowledgeGraph } = await source('core/graph/graph.ts'); + const results = []; + for (const nodes of baseline.sizes) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-embedding-identity-bench-')); + try { + const graph = createKnowledgeGraph(); + const rows = []; + for (let i = 0; i < nodes; i++) { + const id = `Function:source${Math.floor(i / 32)}.ts:fn${i}`; + graph.addNode({ + id, + label: 'Function', + properties: { + name: `fn${i}`, + filePath: `source${Math.floor(i / 32)}.ts`, + startLine: i, + endLine: i + 1, + }, + }); + rows.push({ + nodeId: id, + chunkIndex: 0, + startLine: i, + endLine: i + 1, + contentHash: `hash-${i}`, + embedding: Array(baseline.dimensions).fill(0.125), + }); + } + const copy = []; + const insert = []; + for (let trial = 0; trial < 4; trial++) { + const dbPath = path.join(dir, `lbug-${trial}`); + await adapter.initLbug(dbPath, { skipFts: true }); + let start = performance.now(); + await adapter.loadGraphToLbug(graph, dir, dir, undefined, undefined, undefined, 'none'); + const copyMs = performance.now() - start; + // Seed then replace the exact same keys, exercising idempotent retries. + for (let i = 0; i < rows.length; i += baseline.batch_size) { + await batchInsertEmbeddings( + adapter.executeWithReusedStatement, + rows.slice(i, i + baseline.batch_size), + ); + } + start = performance.now(); + for (let i = 0; i < rows.length; i += baseline.batch_size) { + await batchInsertEmbeddings( + adapter.executeWithReusedStatement, + rows.slice(i, i + baseline.batch_size), + ); + } + const insertMs = performance.now() - start; + await adapter.closeLbug(); + await adapter.initLbug(dbPath, { skipFts: true }); + const persisted = await adapter.executeQuery( + 'MATCH (e:CodeEmbedding) RETURN e.id AS id, e.nodeId AS nodeId, e.contentHash AS hash', + ); + assert.equal(persisted.length, nodes); + const expected = new Map(rows.map((row) => [`${row.nodeId}:0`, row])); + for (const row of persisted) { + const wanted = expected.get(row.id); + assert.ok(wanted); + assert.equal(row.nodeId, wanted.nodeId); + assert.equal(row.hash, wanted.contentHash); + expected.delete(row.id); + } + assert.equal(expected.size, 0); + await adapter.closeLbug(); + if (trial > 0) { + copy.push(copyMs); + insert.push(insertMs); + } + } + const median = (samples) => samples.sort((a, b) => a - b)[Math.floor(samples.length / 2)]; + results.push({ nodes, copy_ms: median(copy), replace_ms: median(insert) }); + } finally { + await adapter.closeLbug(); + await fs.rm(dir, { recursive: true, force: true }); + } + } + const scaling = {}; + for (const key of ['copy_ms', 'replace_ms']) { + scaling[key] = results[1][key] / results[0][key] / (results[1].nodes / results[0].nodes); + } + console.log( + JSON.stringify( + { + node: process.version, + platform: process.platform, + cpu: os.cpus()[0]?.model, + results, + normalized_scaling: scaling, + }, + null, + 2, + ), + ); + if (process.argv.includes('--check')) { + const sameRuntime = + baseline.runtime.node === process.version && + baseline.runtime.platform === process.platform && + baseline.runtime.cpu === os.cpus()[0]?.model; + if (!sameRuntime) + console.warn( + 'Absolute timing gate skipped: baseline runtime differs; correctness and scaling gates still apply.', + ); + for (const [key, value] of Object.entries(scaling)) + assert.ok(value <= baseline.linear_scaling_budget, `${key} scaling ${value}`); + for (const row of results) { + const original = baseline.results.find((item) => item.nodes === row.nodes); + for (const key of ['copy_ms', 'replace_ms']) + if (sameRuntime) + assert.ok( + row[key] / original[key] <= baseline.regression_budget, + `${row.nodes} ${key} regression ${row[key] / original[key]}`, + ); + } + } +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/gitnexus/bench/staged-publication/baseline.json b/gitnexus/bench/staged-publication/baseline.json new file mode 100644 index 000000000..8261adae9 --- /dev/null +++ b/gitnexus/bench/staged-publication/baseline.json @@ -0,0 +1,40 @@ +{ + "version": 1, + "commit": "22a7608b7c5ff77302357c0f90c8d31294801b25", + "runtime": "Node 24.19.0, Linux x64, overlay, Intel Core i9-12900K", + "linear_scaling_budget": 2, + "functions_per_file": 32, + "timed_samples": 6, + "sizes": [ + { + "files": 16, + "functions": 512, + "fingerprint": "19518326db7f2d095a77e3a19642edf38249e653d626387ebcc57df8aaba9c7d", + "median_ms": 2960.9297275, + "samples_ms": [ + 2875.6184279999998, 2884.447086000002, 2945.486928999999, 2976.372526000001, + 3072.810632999999, 3118.3443149999985 + ] + }, + { + "files": 64, + "functions": 2048, + "fingerprint": "89f9dd905722438427100c61cac5113046abc9368c9e6f91e2dde95b0daf0c1f", + "median_ms": 3259.2709509999986, + "samples_ms": [ + 2963.401339, 3237.9479589999974, 3252.9258029999983, 3265.616098999999, 3284.1276569999973, + 3310.6556599999967 + ] + } + ], + "regression_budget": 1.25, + "environment": { + "node": "v24.19.0", + "platform": "linux", + "arch": "x64", + "cpu": "12th Gen Intel(R) Core(TM) i9-12900K", + "filesystem": "0x794c7630" + }, + "measuredAt": "2026-10-17", + "measurement_order": "base/candidate/candidate/base; six samples per size and arm" +} diff --git a/gitnexus/bench/staged-publication/measure.cjs b/gitnexus/bench/staged-publication/measure.cjs new file mode 100644 index 000000000..444d279da --- /dev/null +++ b/gitnexus/bench/staged-publication/measure.cjs @@ -0,0 +1,132 @@ +#!/usr/bin/env node +/** Real forced rebuild + checkpoint/publication timing, with cold-reader oracles. + * node --import tsx bench/staged-publication/measure.cjs [--check] + * Run unchanged against base and candidate, each with its own built worker. + */ +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const { tmpdir } = os; +const path = require('node:path'); +const { pathToFileURL } = require('node:url'); +const { execFileSync } = require('node:child_process'); +const { createHash } = require('node:crypto'); + +(async () => { + const root = path.resolve(__dirname, '../..'); + const baseline = JSON.parse(await fs.readFile(path.join(__dirname, 'baseline.json'), 'utf8')); + const environment = { + node: process.version, + platform: process.platform, + arch: process.arch, + cpu: os.cpus()[0].model, + filesystem: `0x${(await fs.statfs(tmpdir())).type.toString(16)}`, + }; + const sameEnvironment = Object.entries(environment).every( + ([key, value]) => baseline.environment?.[key] === value, + ); + const scratch = await fs.mkdtemp(path.join(tmpdir(), 'gn-publication-bench-')); + process.env.GITNEXUS_HOME = path.join(scratch, 'home'); + process.env.GITNEXUS_SHARED_STORE = 'off'; + process.env.GITNEXUS_WORKER_READY_TIMEOUT_MS = '60000'; + const options = { + force: true, + skipGit: true, + skipFts: true, + skipAgentsMd: true, + skipSkills: true, + }; + const results = []; + try { + const { runFullAnalysis } = await import( + pathToFileURL(path.join(root, 'src/core/run-analyze.ts')) + ); + for (const size of baseline.sizes) { + const { files } = size; + const repo = path.join(scratch, `repo-${files}`); + await fs.mkdir(repo); + const expected = []; + for (let file = 0; file < files; file++) { + const lines = []; + for (let fn = 0; fn < baseline.functions_per_file; fn++) { + const name = `fn_${file}_${fn}`; + expected.push(name); + lines.push(`export function ${name}(input: string) { return input.trim(); }`); + } + await fs.writeFile(path.join(repo, `file-${file}.ts`), lines.join('\n')); + } + expected.sort(); + const fingerprint = createHash('sha256').update(JSON.stringify(expected)).digest('hex'); + assert.equal(expected.length, size.functions); + assert.equal(fingerprint, size.fingerprint); + const samples = []; + for (let trial = 0; trial <= baseline.timed_samples; trial++) { + const started = performance.now(); + await runFullAnalysis(repo, options, { onProgress() {}, onLog() {} }); + const elapsed = performance.now() - started; + if (trial > 0) samples.push(elapsed); + // A different process has no writer or pool state to hide bad publication. + const names = JSON.parse( + execFileSync( + process.execPath, + [ + '-e', + ` + const { Database, Connection } = require('@ladybugdb/core'); + (async () => { + const db = new Database(process.argv[1], 64 * 1024 * 1024, false, true, 1024 ** 3); + const conn = new Connection(db); + const rows = await (await conn.query('MATCH (f:Function) RETURN f.name AS name')).getAll(); + console.log(JSON.stringify(rows.map(row => row.name).sort())); + await conn.close(); await db.close(); + })().catch(error => { console.error(error); process.exitCode = 1; }); + `, + path.join(repo, '.gitnexus/lbug'), + ], + { encoding: 'utf8', cwd: root }, + ), + ); + assert.deepEqual(names, expected); + assert.equal( + (await fs.readdir(path.join(repo, '.gitnexus'))).some((entry) => + /^lbug\.(?:wal|shadow|checkpoint|staging)(?:\.|$)/.test(entry), + ), + false, + ); + } + samples.sort((a, b) => a - b); + const median = + (samples[Math.floor((samples.length - 1) / 2)] + samples[Math.floor(samples.length / 2)]) / + 2; + results.push({ + files, + functions: expected.length, + median_ms: median, + samples_ms: samples, + fingerprint, + baseline_ratio: median / size.median_ms, + }); + } + const scaling = + results[1].median_ms / results[0].median_ms / (results[1].files / results[0].files); + console.log(JSON.stringify({ environment, results, normalized_scaling: scaling }, null, 2)); + if (process.argv.includes('--check')) { + assert.ok(scaling <= baseline.linear_scaling_budget); + if (!sameEnvironment) + console.warn( + 'Absolute timing gate skipped: baseline environment differs; correctness and scaling gates still apply.', + ); + for (const result of results) + if (sameEnvironment) + assert.ok( + result.baseline_ratio <= baseline.regression_budget, + `publication regression: ${result.baseline_ratio}`, + ); + } + } finally { + await fs.rm(scratch, { recursive: true, force: true }); + } +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/gitnexus/scripts/cross-platform-shard.ts b/gitnexus/scripts/cross-platform-shard.ts index 07483169e..a49d8396e 100644 --- a/gitnexus/scripts/cross-platform-shard.ts +++ b/gitnexus/scripts/cross-platform-shard.ts @@ -89,7 +89,14 @@ export const WINDOWS_WEIGHTS_SEC: Readonly> = { 'test/integration/skills-e2e.test.ts': 550, 'test/integration/fts-extension-e2e.test.ts': 380, 'test/integration/skip-fts.test.ts': 110, + // Provisional: twice the local native-suite duration, pending Windows CI. + // These scheduling hints do not change test timeouts or pass/fail gates. + 'test/integration/embedding-identity-integrity.test.ts': 180, + 'test/integration/shared-store-embedding-reuse.test.ts': 140, 'test/integration/analyze-wal-checkpoint-failure.test.ts': 86, + // Windows run 39046807237 measured 308s before two cases reached recovery. + // Allow the measured ~19s recovery cost for each corrected in-place case. + 'test/unit/incremental-write-integrity.test.ts': 360, 'test/integration/cli-limit-e2e.test.ts': 75, 'test/unit/hooks.test.ts': 26, 'test/integration/analyze-heap-oom-e2e.test.ts': 23, diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 65384de7e..a53877ba4 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -180,6 +180,9 @@ const LBUG_NATIVE = [ 'test/integration/pdg-explicit-owner.test.ts', 'test/integration/skip-fts.test.ts', 'test/integration/lbug-core-adapter.test.ts', + // Native COPY, cached vectors and cold shared-store readers vary by platform. + 'test/integration/embedding-identity-integrity.test.ts', + 'test/integration/shared-store-embedding-reuse.test.ts', 'test/integration/lbug-vector-extension.test.ts', 'test/integration/lbug-pool.test.ts', 'test/integration/lbug-pool-stability.test.ts', @@ -211,6 +214,10 @@ const LBUG_NATIVE = [ 'test/integration/fts-description-search.test.ts', 'test/integration/staleness-and-stability.test.ts', 'test/integration/analyze-wal-checkpoint-failure.test.ts', + // Publication, cold readers and sidecar handling depend on native filesystem behavior. + 'test/integration/analyze-atomic-swap.test.ts', + 'test/unit/incremental-write-integrity.test.ts', + 'test/unit/lbug-publication.test.ts', 'test/integration/fts-stemmer-sweep.test.ts', 'test/integration/lbug-multiwriter-deadlock.test.ts', // #2409 batched incremental writeback: chunked IN-list DETACH DELETEs + diff --git a/gitnexus/src/core/embeddings/embedding-keys.ts b/gitnexus/src/core/embeddings/embedding-keys.ts new file mode 100644 index 000000000..96c644993 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-keys.ts @@ -0,0 +1,35 @@ +import { assertValidGraphNodeId } from '../graph/node-identity.js'; + +/** Integrity failures must stop preservation before graph writeback. */ +export class EmbeddingIntegrityError extends Error {} + +export function assertValidEmbeddingKey(nodeId: string, chunkIndex: number, context: string): void { + try { + assertValidGraphNodeId(nodeId, context); + } catch (cause) { + throw new EmbeddingIntegrityError((cause as Error).message, { cause }); + } + // CodeEmbedding.chunkIndex is INT32, not a JavaScript safe integer. + if (!Number.isInteger(chunkIndex) || chunkIndex < 0 || chunkIndex > 2147483647) { + throw new EmbeddingIntegrityError( + `${context}: invalid chunk index; expected a non-negative INT32.`, + ); + } +} + +export function assertUniqueEmbeddingKeys( + rows: Iterable<{ nodeId: string; chunkIndex: number }>, + context: string, +): void { + const keys = new Set(); + for (const row of rows) { + assertValidEmbeddingKey(row.nodeId, row.chunkIndex, context); + const key = `${row.nodeId}:${row.chunkIndex}`; + if (keys.has(key)) { + throw new EmbeddingIntegrityError( + `${context}: duplicate pending key ${JSON.stringify(key.slice(0, 160))}.`, + ); + } + keys.add(key); + } +} diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index 78aec5b02..f0634fad4 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -47,6 +47,8 @@ import { escapeCypherString } from '../lbug/cypher-escape.js'; import type { ExtensionInstallPolicy } from '../lbug/extension-loader.js'; import { getExactScanLimit } from '../platform/capabilities.js'; import { logger } from '../logger.js'; +import { assertValidGraphNodeId } from '../graph/node-identity.js'; +import { assertUniqueEmbeddingKeys } from './embedding-keys.js'; const isDev = process.env.NODE_ENV === 'development'; @@ -157,8 +159,10 @@ const queryEmbeddableNodes = async ( const rows = await executeQuery(query); for (const row of rows) { const hasExportedColumn = label === LABEL_METHOD || LABELS_WITH_EXPORTED.has(label); + const nodeId = row.id ?? row[0]; + assertValidGraphNodeId(nodeId, `Embedding ${label}`); allNodes.push({ - id: row.id ?? row[0], + id: nodeId, name: row.name ?? row[1], label: row.label ?? row[2], filePath: row.filePath ?? row[3], @@ -176,11 +180,11 @@ const queryEmbeddableNodes = async ( }); } } catch (error) { - if (isDev) { - // `err` is pino's standard serializer key — an arbitrary key serializes an - // Error to `{}`, losing the message and stack (#2114). - logger.warn({ err: error }, `Query for ${label} nodes failed:`); - } + // A failed label read is unresolved embedding work, never an empty table. + throw new Error( + `Cannot read ${label} nodes for embedding: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); } } @@ -205,8 +209,10 @@ const queryFallbackFileNodes = async ( return rows .map((row) => { const content = row.content ?? row[4] ?? ''; + const nodeId = row.id ?? row[0]; + assertValidGraphNodeId(nodeId, 'Embedding File'); return { - id: row.id ?? row[0], + id: nodeId, name: row.name ?? row[1], label: row.label ?? row[2] ?? 'File', filePath: row.filePath ?? row[3], @@ -217,17 +223,15 @@ const queryFallbackFileNodes = async ( }) .filter( (node) => - node.id && node.filePath && node.content.trim() && node.content !== '[Binary file - content not stored]', ); } catch (error) { - if (isDev) { - // `err`, not `error` — see the serializer note above (#2114). - logger.warn({ err: error }, 'Fallback File-node embedding query failed:'); - } - return []; + throw new Error( + `Cannot read File nodes for embedding: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); } }; @@ -248,15 +252,21 @@ export const batchInsertEmbeddings = async ( contentHash?: string; }>, ): Promise => { - const paramsList = updates.map((u) => ({ - id: `${u.nodeId}:${u.chunkIndex}`, - nodeId: u.nodeId, - chunkIndex: u.chunkIndex, - startLine: u.startLine, - endLine: u.endLine, - embedding: u.embedding, - contentHash: u.contentHash ?? STALE_HASH_SENTINEL, - })); + // Validate the whole pending batch before DELETE can replace any valid row. + // Existing-row retries and duplicate keys within one batch are distinct. + assertUniqueEmbeddingKeys(updates, 'Embedding write'); + const paramsList = updates.map((u) => { + const id = `${u.nodeId}:${u.chunkIndex}`; + return { + id, + nodeId: u.nodeId, + chunkIndex: u.chunkIndex, + startLine: u.startLine, + endLine: u.endLine, + embedding: u.embedding, + contentHash: u.contentHash ?? STALE_HASH_SENTINEL, + }; + }); if (paramsList.length === 0) return; await executeWithReusedStatement( @@ -547,6 +557,20 @@ export const runEmbeddingPipeline = async ( const throwIfCancelled = (): void => pipelineOptions.signal?.throwIfAborted(); try { + throwIfCancelled(); + // Check the graph before model initialization, endpoint work, or stale-row + // deletion. Reuse this set for pending-node cleanup instead of allocating + // a second identity index later. + let nodes = await queryEmbeddableNodes(executeQuery); + const embeddableNodeIds = new Set(); + for (const node of nodes) { + if (embeddableNodeIds.has(node.id)) { + throw new Error( + `Embedding read: duplicate graph node identity ${JSON.stringify(node.id.slice(0, 160))}. Rebuild with analyze --force.`, + ); + } + embeddableNodeIds.add(node.id); + } throwIfCancelled(); const vectorAvailable = await ensureVectorExtensionAvailable(); throwIfCancelled(); @@ -579,15 +603,6 @@ export const runEmbeddingPipeline = async ( modelDownloadPercent: 100, }); - if (isDev) { - logger.info('🔍 Querying embeddable nodes...'); - } - - // Phase 2: Query embeddable nodes - let nodes = await queryEmbeddableNodes(executeQuery); - throwIfCancelled(); - const embeddableNodeIds = new Set(nodes.map((node) => node.id)); - // Incremental mode: compare content hashes, delete stale rows, skip fresh ones. // Computed hashes for stale nodes are cached so batchInsertEmbeddings can reuse them // (avoids double computation). diff --git a/gitnexus/src/core/embeddings/embedding-restore-spill.ts b/gitnexus/src/core/embeddings/embedding-restore-spill.ts index 10ca51b45..4780ff6dc 100644 --- a/gitnexus/src/core/embeddings/embedding-restore-spill.ts +++ b/gitnexus/src/core/embeddings/embedding-restore-spill.ts @@ -16,6 +16,7 @@ import { randomBytes } from 'node:crypto'; import os from 'node:os'; import path from 'node:path'; import type { CachedEmbedding } from './types.js'; +import { assertValidEmbeddingKey } from './embedding-keys.js'; /** In-RAM vector copies stay below this row count; larger tables use the spill. */ export const DEFAULT_EMBEDDING_CACHE_IN_MEMORY_ROW_LIMIT = 2048; @@ -385,7 +386,8 @@ export function ingestCachedEmbeddingRow( ): void { const rec = row as Record & unknown[]; const nodeId = String(rec.nodeId ?? rec[0] ?? ''); - if (!nodeId) return; + const chunkIndex = Number(rec.chunkIndex ?? rec[1] ?? 0); + assertValidEmbeddingKey(nodeId, chunkIndex, 'Cached embedding'); const embedding = rec.embedding ?? rec[4]; const f32 = coerceEmbeddingToFloat32(embedding); if (!f32) return; @@ -393,7 +395,7 @@ export function ingestCachedEmbeddingRow( builder.embeddingNodeIds.add(nodeId); const meta: CachedEmbeddingMeta = { nodeId, - chunkIndex: Number(rec.chunkIndex ?? rec[1] ?? 0), + chunkIndex, startLine: Number(rec.startLine ?? rec[2] ?? 0), endLine: Number(rec.endLine ?? rec[3] ?? 0), contentHash: hasContentHash diff --git a/gitnexus/src/core/graph/node-identity.ts b/gitnexus/src/core/graph/node-identity.ts new file mode 100644 index 000000000..1e9016453 --- /dev/null +++ b/gitnexus/src/core/graph/node-identity.ts @@ -0,0 +1,15 @@ +/** Identity columns must survive CSV encoding and address exactly one node. */ +export function assertValidGraphNodeId(id: unknown, context: string): asserts id is string { + if ( + typeof id === 'string' && + id.trim().length > 0 && + // CSV normalizes CR and strips these characters. The Unicode flag matches + // lone surrogates while preserving valid supplementary code points. + !/[\x00-\x08\x0B-\x1F\x7F\uD800-\uDFFF\uFFFE\uFFFF]/u.test(id) + ) + return; + const sample = typeof id === 'string' ? JSON.stringify(id.slice(0, 120)) : typeof id; + throw new Error( + `${context}: invalid graph node ID (${sample}). Index integrity cannot be verified; rebuild with analyze --force.`, + ); +} diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index 31b8fac8a..3fb31a256 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -17,6 +17,7 @@ import { createWriteStream, WriteStream } from 'fs'; import path from 'path'; import type { GraphNode, GraphRelationship } from 'gitnexus-shared'; import { KnowledgeGraph } from '../graph/types.js'; +import { assertValidGraphNodeId } from '../graph/node-identity.js'; import { NodeTableName, RELATION_SCHEMA } from './schema.js'; import type { ContentRetention } from '../../storage/repo-meta.js'; import { VALID_NODE_TABLES, parseRelationSchemaPairs, RelPairRouter } from './rel-pair-routing.js'; @@ -105,7 +106,7 @@ export const sanitizeUTF8 = (str: string): string => { .replace(/\r\n/g, '\n') .replace(/\r/g, '\n') .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') - .replace(/[\uD800-\uDFFF]/g, '') + .replace(/[\uD800-\uDFFF]/gu, '') .replace(/[\uFFFE\uFFFF]/g, ''); }; @@ -479,7 +480,7 @@ export interface StreamedCSVResult { /** * Stream all CSV data directly to disk files. - * Iterates graph nodes exactly ONCE — routes each node to the right writer. + * Validates node identities in one pass, then streams nodes to their writers. * File contents are lazy-read from disk with a generous LRU cache. * * `onNodePhaseComplete` (optional, #2203 parallelism leg): fired exactly once, @@ -503,6 +504,10 @@ export const streamAllCSVsToDisk = async ( onNodePhaseComplete?: (nodeFiles: Map) => void, contentRetention: ContentRetention = 'full', ): Promise => { + // Validate before opening writers: rejection must not strand their handles + // or replace existing CSV artifacts. The iterator adds no graph-sized copy. + for (const node of graph.iterNodes()) assertValidGraphNodeId(node.id, 'Graph COPY'); + // Deterministic (id-sorted) node/relationship row order when enabled; // default off = today's graph-insertion order (byte-identical). const sortOutput = parseTruthyEnv(process.env.GITNEXUS_SORT_GRAPH_OUTPUT); diff --git a/gitnexus/src/core/lbug/publication.ts b/gitnexus/src/core/lbug/publication.ts new file mode 100644 index 000000000..e09c8aa24 --- /dev/null +++ b/gitnexus/src/core/lbug/publication.ts @@ -0,0 +1,92 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { closeLbugConnection, openLbugConnection } from './lbug-config.js'; +import { closeQueryResults } from './query-result-utils.js'; +import { runCheckpointWithRetry } from './wal-checkpoint-driver.js'; +import { acquireInitLock } from './lbug-adapter.js'; + +async function residualSidecars(dbPath: string): Promise { + let entries: string[]; + try { + entries = await fs.readdir(path.dirname(dbPath)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + const prefix = `${path.basename(dbPath)}.`; + return entries.filter((name) => { + if (!name.startsWith(prefix)) return false; + const suffix = name.slice(prefix.length); + // Dirty recovery deliberately retains these completed forensic archives. + // They cannot be replayed by the engine. Incomplete `.next` parks and all + // other recovery residues still require intervention before publication. + if (suffix === 'wal.dirty-recovery' || suffix === 'shadow.dirty-recovery') return false; + return /^(wal|shadow|checkpoint)(?:\.|$)/.test(suffix); + }); +} + +/** A publication receipt must describe one consolidated file, including on 9p + * where a renamed, held-open sidecar can be listed while stat returns ENOENT. */ +export async function assertLbugPublicationReady(dbPath: string): Promise { + let residual = await residualSidecars(dbPath); + for (const delay of [25, 50, 100]) { + if (residual.length === 0) return; + await new Promise((resolve) => setTimeout(resolve, delay)); + residual = await residualSidecars(dbPath); + } + if (residual.length > 0) { + throw new Error( + `Cannot publish index at ${dbPath}: residual recovery files require consolidation: ${residual.join(', ')}.`, + ); + } +} + +/** Recover the live family before building its replacement. Do not park or + * unlink its delta: a later failed publication must still serve all old rows. + * This raw handle performs recovery only, without schema or extension writes. */ +export async function consolidateLbugForPublication(dbPath: string): Promise { + if ((await residualSidecars(dbPath)).length === 0) return; + const releaseInitLock = await acquireInitLock(dbPath); + try { + // A missing-shadow quarantine cannot be replayed. Refuse before even a + // read-triggered recovery/checkpoint can change the retained main file. + const quarantinePrefix = `${path.basename(dbPath)}.wal.missing-shadow.`; + if ((await residualSidecars(dbPath)).some((name) => name.startsWith(quarantinePrefix))) { + throw new Error('Quarantined WAL requires recovery with its missing shadow file.'); + } + const handle = await openLbugConnection(lbug, dbPath, { throwOnWalReplayFailure: true }); + const query = async (cypher: string): Promise => { + const result = await handle.conn.query(cypher); + try { + for (const cursor of Array.isArray(result) ? result : [result]) await cursor.getAll(); + } finally { + await closeQueryResults(result); + } + }; + try { + // A lazy writable open can defer WAL replay until its first graph read. + // determinism: probe — rows are drained and discarded; only successful WAL replay matters. + await query('MATCH (n) RETURN n LIMIT 1'); + const receipt = await runCheckpointWithRetry({ + checkpointFn: async () => { + await query('CHECKPOINT'); + return true; + }, + }); + if (!receipt.flushed) + throw new Error(`Live index checkpoint could not be verified: ${dbPath}`); + } finally { + await closeLbugConnection(handle); + } + await assertLbugPublicationReady(dbPath); + } catch (cause) { + throw new Error( + `Cannot consolidate the previous index at ${dbPath}; publication was stopped to preserve its recovery files. ` + + 'Inspect or back up that database family before using gitnexus clean --lbug-sidecars to discard unrecoverable deltas.', + { cause }, + ); + } finally { + await releaseInitLock(); + } +} diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 492b582cf..f1e0b9f01 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -37,6 +37,7 @@ import { constants as fsConstants, existsSync } from 'node:fs'; import { randomUUID } from 'node:crypto'; import { isDeepStrictEqual } from 'node:util'; import { retryRename } from '../storage/fs-atomic.js'; +import { assertLbugPublicationReady, consolidateLbugForPublication } from './lbug/publication.js'; import { acquireIndexLock, requireExclusiveIndexLock, @@ -61,6 +62,7 @@ import { reconcileGraphNodeIdentities } from './incremental/write-reconciliation import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js'; import { initLbug, + acquireInitLock, loadGraphToLbug, getLbugStats, executeQuery, @@ -140,6 +142,7 @@ import { resolveFtsVersionPair } from './lbug/vendored-extension-path.js'; import { startWalCheckpointDriver, checkpointOnce, + runCheckpointWithRetry, isManualCheckpointEnabled, readCheckpointActivity, type CheckpointActivity, @@ -287,6 +290,7 @@ import { snapshotEmbeddingDims, type CachedEmbeddingsSnapshot, } from './embeddings/embedding-restore-spill.js'; +import { assertUniqueEmbeddingKeys, EmbeddingIntegrityError } from './embeddings/embedding-keys.js'; import { generateAIContextFiles } from '../cli/ai-context.js'; import { formatRejectedBranchForLog, sanitizeDetectedBranch } from './git-ref.js'; import { @@ -2796,6 +2800,23 @@ async function runFullAnalysisInner( // before the embedding plan, the parse cache, and the emit mode are decided. applyCollectedForce(); + // Where the swap is allowed: + // - POSIX renames an open file, so the usual skip-native-close (#2264) is + // fine and the swap always applies. + // - Windows can swap only when a real close is safe to release the build + // handle before the rename — i.e. NOT a --pdg run (the #2264 destructor + // crash). Unverified on Windows CI; falls back to in-place otherwise. + const posixSwap = process.platform !== 'win32'; + // #2614 Windows: the forced real-close before the rename re-bets that #2264 is + // --pdg-only, which is unproven (the CLI/worker skip the native close + // UNCONDITIONALLY) and unverifiable without a Windows runner. Keep it opt-in + // (GITNEXUS_ATOMIC_WINDOWS_SWAP=1) so the default Windows analyze stays on the + // proven in-place path; enable it only to test the Windows swap. + const windowsSwapOk = + process.platform === 'win32' && + options.pdg !== true && + process.env.GITNEXUS_ATOMIC_WINDOWS_SWAP === '1'; + // ── Cache embeddings from existing index before rebuild ──────────── // Four modes: // --embeddings -> load cache, restore, then generate any new ones @@ -2814,6 +2835,7 @@ async function runFullAnalysisInner( let cachedSnapshot: CachedEmbeddingsSnapshot = emptyCachedEmbeddingsSnapshot(); const adoptCachedEmbeddings = (raw: CachedEmbeddingsSnapshot): void => { cachedSnapshot = normalizeCachedEmbeddings(raw); + assertUniqueEmbeddingKeys(cachedSnapshot.rows, 'Cached embedding'); }; const discardCachedEmbeddings = (): void => { disposeEmbeddingSpill(cachedSnapshot.spill); @@ -2868,12 +2890,24 @@ async function runFullAnalysisInner( // possible the dirty block above already threw a LbugWipeError, so this // open is replay-free by construction (FIX 1 of this shipping review). if (shouldLoadCache && existingMeta) { + // Cache initialization is writable and may discard quarantined WALs. + // Preserve the live family before this first open, including runs whose + // incremental write later escalates to a staged full rebuild. Keep this + // outside the cache-load catch: a preservation failure must stop analysis. + if (posixSwap || windowsSwapOk) await consolidateLbugForPublication(lbugPath); try { progress('embeddings', 0, 'Caching embeddings...'); await initAnalysisLbug(lbugPath); adoptCachedEmbeddings(await loadCachedEmbeddings()); await closeLbug(); } catch (err: any) { + discardCachedEmbeddings(); + try { + await closeLbug(); + } catch { + /* preserve the original cache failure */ + } + if (err instanceof EmbeddingIntegrityError) throw err; // Surface cache-load failures explicitly: silently swallowing here would // re-introduce the original silent-data-loss symptom (embeddings end up // at 0 in meta.json with no diagnostic) through a different door. @@ -2882,12 +2916,6 @@ async function runFullAnalysisInner( `(${err?.message ?? String(err)}). ` + `Embeddings will not be preserved on this run.`, ); - discardCachedEmbeddings(); - try { - await closeLbug(); - } catch { - /* swallow */ - } } } @@ -2947,6 +2975,7 @@ async function runFullAnalysisInner( `for ${recovered.embeddingNodeIds.size} node(s); unchanged content can reuse them.`, ); } catch (err) { + if (err instanceof EmbeddingIntegrityError) throw err; log( `Warning: could not recover staged embeddings (${(err as Error).message}); ` + 'the retry will regenerate missing chunks.', @@ -3207,22 +3236,6 @@ async function runFullAnalysisInner( // rename on Windows. Windows keeps the current in-place behavior // (buildPath === lbugPath, no swap) until that is resolved (see §12/follow-up). const isFullRebuild = !(isIncremental && hashDiff); - // Where the swap is allowed: - // - POSIX renames an open file, so the usual skip-native-close (#2264) is - // fine and the swap always applies. - // - Windows can swap only when a real close is safe to release the build - // handle before the rename — i.e. NOT a --pdg run (the #2264 destructor - // crash). Unverified on Windows CI; falls back to in-place otherwise. - const posixSwap = process.platform !== 'win32'; - // #2614 Windows: the forced real-close before the rename re-bets that #2264 is - // --pdg-only, which is unproven (the CLI/worker skip the native close - // UNCONDITIONALLY) and unverifiable without a Windows runner. Keep it opt-in - // (GITNEXUS_ATOMIC_WINDOWS_SWAP=1) so the default Windows analyze stays on the - // proven in-place path; enable it only to test the Windows swap. - const windowsSwapOk = - process.platform === 'win32' && - options.pdg !== true && - process.env.GITNEXUS_ATOMIC_WINDOWS_SWAP === '1'; // Incremental atomicity stages the whole index before mutation. It remains // opt-in for ordinary analyze runs; watch mode requests it for failure // preservation. The copy requests a filesystem clone and records its actual @@ -3268,6 +3281,9 @@ async function runFullAnalysisInner( // valve. Nothing between here and there reads either binding except // `initLbug(buildPath)`, which the upgrade re-runs against the staging path. let useAtomicSwap = (isFullRebuild || atomicIncremental) && (posixSwap || windowsSwapOk); + // Recover a valid live delta before staging its replacement. In-place runs + // retain the adapter's existing recovery path. + if (useAtomicSwap) await consolidateLbugForPublication(lbugPath); // Set only at the first operation that can mutate the live graph store. // Pre-write failures (config, lock, parsing, metadata, importer expansion) // remain retryable even when this platform cannot use an atomic swap. @@ -3807,12 +3823,13 @@ async function runFullAnalysisInner( ); } } catch (err: any) { + discardCachedEmbeddings(); + if (err instanceof EmbeddingIntegrityError) throw err; log( `Warning: could not load cached embeddings ` + `(${err?.message ?? String(err)}). ` + `Embeddings will not be preserved on this run.`, ); - discardCachedEmbeddings(); } } // Hoisted out of the `||` below (§5.D): the size verdict has to be KNOWN @@ -4514,6 +4531,7 @@ async function runFullAnalysisInner( } materialized = materializeCachedEmbeddings(cachedSnapshot, batch, spillReader); } catch (err) { + if (!shouldGenerateEmbeddings) throw err; for (const row of batch) restoreFailedNodeIds.add(row.nodeId); log( `Warning: could not materialize ${batch.length} cached embedding(s) for restore ` + @@ -4525,6 +4543,7 @@ async function runFullAnalysisInner( await batchInsert(executeWithReusedStatement, materialized); restoredEmbeddingCount += batch.length; } catch (err) { + if (err instanceof EmbeddingIntegrityError || !shouldGenerateEmbeddings) throw err; for (const row of batch) restoreFailedNodeIds.add(row.nodeId); log( `Warning: could not restore ${batch.length} cached embedding(s) ` + @@ -5435,6 +5454,11 @@ async function runFullAnalysisInner( // Stop the manual checkpoint driver before closeLbug so its // in-flight CHECKPOINT cannot race the `safeClose` CHECKPOINT. await walCheckpointDriver.stop(); + if (useAtomicSwap && !wroteChangedSubgraphOnly && !(await runCheckpointWithRetry()).flushed) { + // Publication requires durability even when periodic/manual checkpoints + // are disabled. A best-effort close cannot certify the staged file. + throw new Error('Cannot publish staged index: final checkpoint could not be verified.'); + } // CLI callers (about to process.exit) skip the native close to dodge a // LadybugDB destructor double-free after --pdg writes — closeLbugBeforeExit // CHECKPOINTs for durability then leaves the handles for process exit to @@ -5469,20 +5493,14 @@ async function runFullAnalysisInner( ) : false; if (useAtomicSwap && builtDbExists) { - await retryRename(buildPath, lbugPath); - liveIndexMutationStarted = true; - // Clear any sidecars orphaned beside the replaced file. A cleanly-closed - // prior index has none; a crashed one could, and it would be replay - // poison next to the freshly published index. Best-effort. - for (const suffix of ['.wal', '.shadow', '.wal.checkpoint'] as const) { - await fs.rm(`${lbugPath}${suffix}`, { force: true }).catch(() => {}); - } - // #2614 F4: if the final checkpoint silently failed, the build may still - // carry a residual .wal/.shadow under the temp name. MOVE it beside the - // published index (not orphan/delete it) so the next open replays the - // delta, rather than leaving it under a name LadybugDB never reconciles. - for (const suffix of ['.wal', '.shadow'] as const) { - await fs.rename(`${buildPath}${suffix}`, `${lbugPath}${suffix}`).catch(() => {}); + await assertLbugPublicationReady(buildPath); + const releaseInitLock = await acquireInitLock(lbugPath); + try { + await assertLbugPublicationReady(lbugPath); + await retryRename(buildPath, lbugPath); + liveIndexMutationStarted = true; + } finally { + await releaseInitLock(); } } diff --git a/gitnexus/test/helpers/interrupted-publication-runner.mjs b/gitnexus/test/helpers/interrupted-publication-runner.mjs new file mode 100644 index 000000000..b76baf950 --- /dev/null +++ b/gitnexus/test/helpers/interrupted-publication-runner.mjs @@ -0,0 +1,22 @@ +// A real analyzer process exits at a JS-visible publication boundary. No parser, +// graph, storage, or checkpoint implementation is replaced. +import fs from 'node:fs/promises'; +import path from 'node:path'; + +const rename = fs.rename; +fs.rename = async (source, destination) => { + if ( + path.basename(String(source)).startsWith('lbug.staging.') && + path.basename(String(destination)) === 'lbug' + ) { + process.exit(73); + } + return rename(source, destination); +}; +const { runFullAnalysis } = await import('../../dist/core/run-analyze.js'); +await runFullAnalysis( + process.argv[2], + { force: true, skipNativeCloseOnExit: true }, + { onProgress() {} }, +); +throw new Error('Expected interruption immediately before publication'); diff --git a/gitnexus/test/helpers/publication-live-wal.mjs b/gitnexus/test/helpers/publication-live-wal.mjs new file mode 100644 index 000000000..d28675295 --- /dev/null +++ b/gitnexus/test/helpers/publication-live-wal.mjs @@ -0,0 +1,34 @@ +import fs from 'node:fs/promises'; +import lbug from '@ladybugdb/core'; + +const [dbPath, mode] = process.argv.slice(2); +const db = new lbug.Database(dbPath, 256 * 1024 * 1024, false, false, 1024 ** 3); +const conn = new lbug.Connection(db); +async function query(cypher) { + const result = await conn.query(cypher); + for (const cursor of Array.isArray(result) ? result : [result]) { + try { + await cursor.getAll(); + } finally { + await cursor.close(); + } + } +} +await query('CHECKPOINT'); +await query( + "CREATE (:Function {id: 'Function:wal.ts:walOnly', name: 'walOnly', filePath: 'wal.ts', startLine: 1, endLine: 1})", +); +const main = await fs.readFile(dbPath); +const wal = await fs.readFile(`${dbPath}.wal`); +await conn.close(); +await db.close(); +// Restore real committed WAL bytes to simulate interruption before consolidation. +await fs.writeFile(dbPath, main); +if (mode === 'checkpoint') { + await fs.writeFile(`${dbPath}.wal.checkpoint`, wal); + for (const suffix of ['.wal', '.shadow', '.checkpoint.intent.lock', '.checkpoint.apply.lock']) { + await fs.writeFile(`${dbPath}${suffix}`, ''); + } +} else { + await fs.writeFile(`${dbPath}.wal`, wal); +} diff --git a/gitnexus/test/integration/analyze-atomic-swap.test.ts b/gitnexus/test/integration/analyze-atomic-swap.test.ts index 8a9edad05..b8fef5c85 100644 --- a/gitnexus/test/integration/analyze-atomic-swap.test.ts +++ b/gitnexus/test/integration/analyze-atomic-swap.test.ts @@ -14,14 +14,16 @@ * these swap invariants do not apply — see run-analyze's platform guard. */ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; -import { execSync } from 'child_process'; +import { execFileSync, execSync, spawnSync } from 'child_process'; import { promises as fs } from 'node:fs'; import path from 'node:path'; +import { fileURLToPath } from 'node:url'; type LbugAdapter = typeof import('../../src/core/lbug/lbug-adapter.js'); type RepoManager = typeof import('../../src/storage/repo-manager.js'); type FsAtomic = typeof import('../../src/storage/fs-atomic.js'); type EscalationGate = typeof import('../../src/core/incremental/escalation-gate.js'); +type CheckpointDriver = typeof import('../../src/core/lbug/wal-checkpoint-driver.js'); const ctx = vi.hoisted(() => ({ loadMock: vi.fn(), realLoad: null as LbugAdapter['loadGraphToLbug'] | null, @@ -29,13 +31,23 @@ const ctx = vi.hoisted(() => ({ realDelete: null as LbugAdapter['deleteNodesForFiles'] | null, closeMock: vi.fn(), realClose: null as LbugAdapter['closeLbug'] | null, + closeBeforeExitMock: vi.fn(), + realCloseBeforeExit: null as LbugAdapter['closeLbugBeforeExit'] | null, saveMetaMock: vi.fn(), realSaveMeta: null as RepoManager['saveMeta'] | null, renameMock: vi.fn(), realRename: null as FsAtomic['retryRename'] | null, escalateMock: vi.fn(), realEscalate: null as EscalationGate['shouldEscalateIncrementalWrite'] | null, + finalCheckpointMock: vi.fn(), + realFinalCheckpoint: null as CheckpointDriver['runCheckpointWithRetry'] | null, })); +vi.mock('../../src/core/lbug/wal-checkpoint-driver.js', async (importOriginal) => { + const actual = await importOriginal(); + ctx.realFinalCheckpoint = actual.runCheckpointWithRetry; + ctx.finalCheckpointMock.mockImplementation(actual.runCheckpointWithRetry); + return { ...actual, runCheckpointWithRetry: ctx.finalCheckpointMock }; +}); // Delegating mocks keep real graph, metadata, and registry writes while // allowing failures at the write and publication boundaries. vi.mock('../../src/core/lbug/lbug-adapter.js', async (importOriginal) => { @@ -43,14 +55,17 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', async (importOriginal) => { ctx.realLoad = actual.loadGraphToLbug; ctx.realDelete = actual.deleteNodesForFiles; ctx.realClose = actual.closeLbug; + ctx.realCloseBeforeExit = actual.closeLbugBeforeExit; ctx.loadMock.mockImplementation(actual.loadGraphToLbug); ctx.deleteMock.mockImplementation(actual.deleteNodesForFiles); ctx.closeMock.mockImplementation(actual.closeLbug); + ctx.closeBeforeExitMock.mockImplementation(actual.closeLbugBeforeExit); return { ...actual, loadGraphToLbug: ctx.loadMock, deleteNodesForFiles: ctx.deleteMock, closeLbug: ctx.closeMock, + closeLbugBeforeExit: ctx.closeBeforeExitMock, }; }); vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => { @@ -88,6 +103,7 @@ import { import { runWorkerAnalysis } from '../../src/server/analyze-worker-core.js'; import type { WorkerMessage } from '../../src/server/analyze-worker.js'; import { createTempDir } from '../helpers/test-db.js'; +import { seedEmbeddingsForFiles, stampEmbeddingCount } from '../helpers/embedding-seed.js'; const isWin = process.platform === 'win32'; @@ -122,6 +138,8 @@ describe.skipIf(isWin)('atomic full-rebuild swap (#2)', () => { ); ctx.closeMock.mockReset(); ctx.closeMock.mockImplementation(() => ctx.realClose!()); + ctx.closeBeforeExitMock.mockReset(); + ctx.closeBeforeExitMock.mockImplementation(() => ctx.realCloseBeforeExit!()); ctx.saveMetaMock.mockReset(); ctx.saveMetaMock.mockImplementation((...a: Parameters) => ctx.realSaveMeta!(...a), @@ -135,6 +153,11 @@ describe.skipIf(isWin)('atomic full-rebuild swap (#2)', () => { (...a: Parameters) => ctx.realEscalate!(...a), ); + ctx.finalCheckpointMock.mockReset(); + ctx.finalCheckpointMock.mockImplementation( + (...args: Parameters) => + ctx.realFinalCheckpoint!(...args), + ); }); afterEach(async () => { @@ -196,6 +219,352 @@ describe.skipIf(isWin)('atomic full-rebuild swap (#2)', () => { } }, 180_000); + it('preserves a pre-existing quarantine before opening the embedding cache (#3591)', async () => { + const { repo, cleanup } = await makeRepo(); + try { + await runFullAnalysis(repo, { skipFts: true }, { onProgress: () => {} }); + const { lbugPath, storagePath } = getStoragePaths(repo); + const seeded = await seedEmbeddingsForFiles(repo, ['a.ts'], 2); + const nodeIds = seeded.get('a.ts')!; + expect(nodeIds).toHaveLength(2); + await stampEmbeddingCount(storagePath, nodeIds.length); + const oldGraph = await fs.readFile(lbugPath); + const oldMeta = await loadMeta(storagePath); + const quarantine = `${lbugPath}.wal.missing-shadow.1234-abcd`; + const evidence = Buffer.from('controlled quarantine: retain for recovery'); + await fs.writeFile(quarantine, evidence); + // Stop the unfixed implementation before it reaches embedding generation. + // The preservation guard must reject earlier, before any graph write. + ctx.loadMock.mockRejectedValueOnce(new Error('injected later rebuild failure')); + await expect( + runFullAnalysis(repo, { force: true, skipFts: true }, { onProgress: () => {} }), + ).rejects.toThrow(/consolidat|recovery files/i); + expect(await fs.readFile(quarantine)).toEqual(evidence); + expect(await fs.readFile(lbugPath)).toEqual(oldGraph); + expect(await loadMeta(storagePath)).toEqual(oldMeta); + const coldIds = execFileSync( + process.execPath, + [ + '-e', + ` + const { Database, Connection } = require('@ladybugdb/core'); + (async () => { + const db = new Database(process.argv[1], 256 * 1024 * 1024, false, true, 1024 ** 3); + const conn = new Connection(db); + const result = await conn.query('MATCH (e:CodeEmbedding) RETURN e.nodeId AS id'); + console.log(JSON.stringify((await result.getAll()).map(row => row.id).sort())); + await conn.close(); await db.close(); + })().catch(error => { console.error(error); process.exitCode = 1; }); + `, + lbugPath, + ], + { encoding: 'utf8', timeout: 30_000 }, + ); + expect(JSON.parse(coldIds)).toEqual(nodeIds.sort()); + } finally { + await cleanup(); + } + }, 180_000); + + it.each([ + { mode: 'wal', failure: 'publication' }, + { mode: 'checkpoint', failure: 'publication' }, + { mode: 'wal', failure: 'checkpoint' }, + { mode: 'checkpoint', failure: 'checkpoint' }, + ])( + 'preserves live $mode recovery data across a failed $failure and retry (#3591)', + async ({ mode, failure }) => { + const { repo, cleanup } = await makeRepo(); + const readerId = `live-recovery-${mode}`; + try { + await runFullAnalysis(repo, { skipFts: true }, { onProgress: () => {} }); + const { lbugPath, storagePath } = getStoragePaths(repo); + const metadata = await loadMeta(storagePath); + execFileSync( + process.execPath, + [ + fileURLToPath(new URL('../helpers/publication-live-wal.mjs', import.meta.url)), + lbugPath, + mode, + ], + { timeout: 30_000 }, + ); + await fs.writeFile( + path.join(repo, 'a.ts'), + 'export function replacement() { return 2; }\n', + ); + if (failure === 'publication') { + ctx.renameMock.mockRejectedValueOnce( + new Error('injected live recovery boundary failure'), + ); + } else { + let injected = false; + ctx.finalCheckpointMock.mockImplementation( + (options: Parameters[0]) => { + if (!options?.checkpointFn && !injected) { + injected = true; + return Promise.reject(new Error('injected live recovery boundary failure')); + } + return ctx.realFinalCheckpoint!(options); + }, + ); + } + await expect( + runFullAnalysis(repo, { force: true, skipFts: true }, { onProgress: () => {} }), + ).rejects.toThrow('injected live recovery boundary failure'); + expect((await loadMeta(storagePath))?.indexedAt).toBe(metadata?.indexedAt); + // Observe the writer's receipt before a pool reader can repair anything. + const residual = (await fs.readdir(path.dirname(lbugPath))).filter((name) => + /^lbug\.(wal|shadow|checkpoint)/.test(name), + ); + expect(residual).toEqual([]); + expect(await lingeringTemp(lbugPath)).toEqual([]); + const coldNames = execFileSync( + process.execPath, + [ + '-e', + ` + const { Database, Connection } = require('@ladybugdb/core'); + (async () => { + const db = new Database(process.argv[1], 256 * 1024 * 1024, false, true, 1024 ** 3); + const conn = new Connection(db); + const result = await conn.query('MATCH (f:Function) RETURN f.name AS name'); + console.log(JSON.stringify((await result.getAll()).map(row => row.name).sort())); + await conn.close(); await db.close(); + })().catch(error => { console.error(error); process.exitCode = 1; }); + `, + lbugPath, + ], + { encoding: 'utf8', cwd: process.cwd(), timeout: 30_000 }, + ); + expect(JSON.parse(coldNames)).toEqual(['caller', 'greet', 'walOnly']); + await poolInit(readerId, lbugPath); + const rows = await poolQuery( + readerId, + 'MATCH (f:Function) RETURN f.name AS name ORDER BY name', + ); + expect(rows.map((row) => row.name)).toEqual(['caller', 'greet', 'walOnly']); + await poolClose(readerId); + await runFullAnalysis(repo, { force: true, skipFts: true }, { onProgress: () => {} }); + await poolInit(readerId, lbugPath); + expect(await poolQuery(readerId, 'MATCH (f:Function) RETURN f.name AS name')).toEqual([ + { name: 'replacement' }, + ]); + } finally { + await poolClose(readerId); + await cleanup(); + } + }, + 180_000, + ); + + it.each([ + { suffix: '.wal', skipNativeCloseOnExit: false }, + { suffix: '.shadow', skipNativeCloseOnExit: false }, + { suffix: '.wal.checkpoint', skipNativeCloseOnExit: false }, + { suffix: '.wal.checkpoint', skipNativeCloseOnExit: true }, + { suffix: '.wal.missing-shadow.1234-abcd', skipNativeCloseOnExit: false }, + ])( + 'refuses residual $suffix (skipNativeCloseOnExit=$skipNativeCloseOnExit) and preserves cold readers (#3591)', + async ({ suffix, skipNativeCloseOnExit }) => { + const { repo, cleanup } = await makeRepo(); + const readerId = `publication-residual-${suffix}`; + try { + await runFullAnalysis(repo, {}, { onProgress: () => {} }); + const { lbugPath, storagePath } = getStoragePaths(repo); + const oldGraph = await fs.readFile(lbugPath); + const oldMeta = await loadMeta(storagePath); + await fs.writeFile( + path.join(repo, 'a.ts'), + 'export function replacement() { return 42; }\n', + ); + execSync('git -c user.name=t -c user.email=t@t commit -am replacement', { + cwd: repo, + stdio: 'pipe', + }); + + // Controlled publication-boundary fault: real indexing, checkpoint and + // close run first. A residual recovery file must never accompany a + // published main file, irrespective of the earlier checkpoint receipt. + ctx.loadMock.mockClear(); + const closeMock = skipNativeCloseOnExit ? ctx.closeBeforeExitMock : ctx.closeMock; + const realClose = skipNativeCloseOnExit ? ctx.realCloseBeforeExit! : ctx.realClose!; + closeMock.mockImplementation(async () => { + await realClose(); + if (ctx.loadMock.mock.calls.length === 0) return; + const stage = (await fs.readdir(path.dirname(lbugPath))).find( + (entry) => + entry.startsWith('lbug.staging.') && + !entry.includes('.wal') && + !entry.endsWith('.shadow'), + ); + expect(stage).toBeDefined(); + await fs.writeFile( + path.join(path.dirname(lbugPath), `${stage}${suffix}`), + 'injected residual', + ); + }); + await expect( + runFullAnalysis(repo, { force: true, skipNativeCloseOnExit }, { onProgress: () => {} }), + ).rejects.toThrow(/publish|consolidat|sidecar/i); + // This harness stays alive to inspect/retry. Reclaim the stage handle + // that the CLI's process exit would release before opening any reader. + if (skipNativeCloseOnExit) await ctx.realClose!(); + expect((await fs.readFile(lbugPath)).equals(oldGraph)).toBe(true); + expect(await loadMeta(storagePath)).toMatchObject({ + lastCommit: oldMeta!.lastCommit, + indexedAt: oldMeta!.indexedAt, + }); + await poolInit(readerId, lbugPath); + const rows = await poolQuery(readerId, 'MATCH (f:Function) RETURN f.name AS name'); + expect(rows.map((row) => (row as { name: string }).name).sort()).toEqual([ + 'caller', + 'greet', + ]); + await poolClose(readerId); + // A new native process has no pool cache or retained reader handle. + const coldNames = execFileSync( + process.execPath, + [ + '-e', + ` + const { Database, Connection } = require('@ladybugdb/core'); + (async () => { + const db = new Database(process.argv[1], 256 * 1024 * 1024, false, true, 1024 ** 3); + const conn = new Connection(db); + const result = await conn.query('MATCH (f:Function) RETURN f.name AS name'); + console.log(JSON.stringify((await result.getAll()).map(row => row.name).sort())); + await conn.close(); await db.close(); + })().catch(error => { console.error(error); process.exitCode = 1; }); + `, + lbugPath, + ], + { encoding: 'utf8', timeout: 30_000 }, + ); + expect(JSON.parse(coldNames)).toEqual(['caller', 'greet']); + + // A failed stage must not make recovery a permanent failure loop. + closeMock.mockImplementation(realClose); + await runFullAnalysis(repo, { force: true }, { onProgress: () => {} }); + await poolInit(readerId, lbugPath); + const recovered = await poolQuery(readerId, 'MATCH (f:Function) RETURN f.name AS name'); + expect(recovered.map((row) => (row as { name: string }).name).sort()).toEqual([ + 'replacement', + ]); + } finally { + if (skipNativeCloseOnExit) await ctx.realClose!(); + await poolClose(readerId); + await cleanup(); + } + }, + 180_000, + ); + + it.each(['rejected', 'unverified', 'manual-disabled-unverified'])( + 'keeps the published graph when the final full-rebuild checkpoint is %s (#3591)', + async (outcome) => { + const { repo, cleanup } = await makeRepo(); + try { + await runFullAnalysis(repo, {}, { onProgress: () => {} }); + const { lbugPath, storagePath } = getStoragePaths(repo); + const oldGraph = await fs.readFile(lbugPath); + const oldMeta = await loadMeta(storagePath); + if (outcome === 'manual-disabled-unverified') + vi.stubEnv('GITNEXUS_WAL_MANUAL_CHECKPOINT', '0'); + if (outcome === 'rejected') { + ctx.finalCheckpointMock.mockRejectedValueOnce( + new Error('injected final checkpoint failure'), + ); + } else { + ctx.finalCheckpointMock.mockResolvedValueOnce({ attempts: 1, flushed: false }); + } + await expect( + runFullAnalysis(repo, { force: true }, { onProgress: () => {} }), + ).rejects.toThrow(/checkpoint/i); + expect((await fs.readFile(lbugPath)).equals(oldGraph)).toBe(true); + expect(await loadMeta(storagePath)).toMatchObject({ + lastCommit: oldMeta!.lastCommit, + indexedAt: oldMeta!.indexedAt, + }); + } finally { + vi.unstubAllEnvs(); + await cleanup(); + } + }, + 180_000, + ); + + it('requires a real final checkpoint even when periodic manual checkpoints are disabled (#3591)', async () => { + const { repo, cleanup } = await makeRepo(); + try { + vi.stubEnv('GITNEXUS_WAL_MANUAL_CHECKPOINT', '0'); + await runFullAnalysis(repo, { force: true }, { onProgress: () => {} }); + const finalCall = ctx.finalCheckpointMock.mock.calls.findIndex((args) => args.length === 0); + expect(finalCall).toBeGreaterThanOrEqual(0); + await expect(ctx.finalCheckpointMock.mock.results[finalCall].value).resolves.toMatchObject({ + flushed: true, + }); + const { lbugPath } = getStoragePaths(repo); + expect( + (await fs.readdir(path.dirname(lbugPath))).some((name) => + /^lbug\.(wal|shadow|checkpoint)(\.|$)/.test(name), + ), + ).toBe(false); + } finally { + vi.unstubAllEnvs(); + await cleanup(); + } + }, 180_000); + + it('recovers after a CLI process exits immediately before publication (#3591)', async () => { + const { repo, cleanup } = await makeRepo(); + const readerId = 'interrupted-publication'; + try { + await runFullAnalysis(repo, {}, { onProgress: () => {} }); + const { lbugPath, storagePath } = getStoragePaths(repo); + const oldGraph = await fs.readFile(lbugPath); + const oldMeta = await loadMeta(storagePath); + await fs.writeFile(path.join(repo, 'a.ts'), 'export function replacement() { return 42; }\n'); + execSync('git -c user.name=t -c user.email=t@t commit -am replacement', { + cwd: repo, + stdio: 'pipe', + }); + const child = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('../helpers/interrupted-publication-runner.mjs', import.meta.url)), + repo, + ], + { encoding: 'utf8', timeout: 120_000 }, + ); + expect(child.error, child.stderr).toBeUndefined(); + expect(child.status, child.stderr).toBe(73); + expect((await fs.readFile(lbugPath)).equals(oldGraph)).toBe(true); + expect(await loadMeta(storagePath)).toMatchObject({ + indexedAt: oldMeta!.indexedAt, + lastCommit: oldMeta!.lastCommit, + }); + await poolInit(readerId, lbugPath); + expect( + (await poolQuery(readerId, 'MATCH (f:Function) RETURN f.name AS name')) + .map((row) => row.name) + .sort(), + ).toEqual(['caller', 'greet']); + await poolClose(readerId); + await runFullAnalysis(repo, { force: true }, { onProgress: () => {} }); + expect(await lingeringTemp(lbugPath)).toEqual([]); + await poolInit(readerId, lbugPath); + expect( + (await poolQuery(readerId, 'MATCH (f:Function) RETURN f.name AS name')).map( + (row) => row.name, + ), + ).toEqual(['replacement']); + } finally { + await poolClose(readerId); + await cleanup(); + } + }, 180_000); + // Commit a replacement graph, then force a rebuild whose staged load loses // every relationship, and assert the collapse guard rejects it unpublished. const expectCollapsedRebuildRejected = async (repo: string) => { diff --git a/gitnexus/test/integration/csv-pipeline.test.ts b/gitnexus/test/integration/csv-pipeline.test.ts index 78be85580..edf52eb4c 100644 --- a/gitnexus/test/integration/csv-pipeline.test.ts +++ b/gitnexus/test/integration/csv-pipeline.test.ts @@ -75,6 +75,33 @@ afterAll(async () => { }); describe('streamAllCSVsToDisk', () => { + it('rejects malformed identities before opening CSV streams and permits a corrected retry', async () => { + const target = path.join(tmpHandle.dbPath, 'invalid-identity-csv'); + await fs.mkdir(target); + const graph = buildTestGraph([ + { id: 'Function:healthy', label: 'Function', name: 'healthy', filePath: 'src/index.ts' }, + ]); + const invalidId = 'Function:bad\0id'; + graph.addNode({ + id: invalidId, + label: 'Function', + properties: { name: 'bad', filePath: 'src/index.ts' }, + }); + const listeners = process.getMaxListeners(); + + await expect(streamAllCSVsToDisk(graph, repoDir, target)).rejects.toThrow( + /invalid graph node ID/, + ); + expect(await fs.readdir(target)).toEqual([]); + expect(process.getMaxListeners()).toBe(listeners); + + graph.removeNode(invalidId); + const result = await streamAllCSVsToDisk(graph, repoDir, target); + const functions = result.nodeFiles.get('Function')!; + expect(functions.rows).toBe(1); + expect(dataRowsOf(await fs.readFile(functions.csvPath, 'utf8'))).toHaveLength(1); + }); + it('generates CSV files for all node types in the graph', async () => { const graph = buildTestGraph( [ diff --git a/gitnexus/test/integration/embedding-identity-integrity.test.ts b/gitnexus/test/integration/embedding-identity-integrity.test.ts new file mode 100644 index 000000000..51c579dd1 --- /dev/null +++ b/gitnexus/test/integration/embedding-identity-integrity.test.ts @@ -0,0 +1,286 @@ +import path from 'node:path'; +import { promises as fs } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { createServer } from 'node:http'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createTempDir, type TestDBHandle } from '../helpers/test-db.js'; +import { + batchInsertEmbeddings, + runEmbeddingPipeline, +} from '../../src/core/embeddings/embedding-pipeline.js'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { EMBEDDING_DIMS } from '../../src/core/lbug/schema.js'; +import * as adapter from '../../src/core/lbug/lbug-adapter.js'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { getStoragePaths, loadMeta } from '../../src/storage/repo-manager.js'; +import { seedEmbeddingsForFiles, stampEmbeddingCount } from '../helpers/embedding-seed.js'; + +describe('embedding identity integrity (#3569)', () => { + let tmp: TestDBHandle | undefined; + afterEach(async () => { + await adapter.closeLbug(); + await tmp?.cleanup(); + tmp = undefined; + vi.unstubAllEnvs(); + }); + + it.each([ + '', + ' ', + '\0broken', + 'bad\x01id', + 'bad\uD800id', + 'duplicate', + 'negative-chunk', + 'fractional-chunk', + 'overflow-chunk', + ])('rejects %j before replacing any existing embedding', async (invalid) => { + tmp = await createTempDir('gitnexus-test-embedding-identity-'); + const dbPath = path.join(tmp.dbPath, 'lbug'); + await adapter.initLbug(dbPath, { skipFts: true }); + const original = { + nodeId: 'Function:source.ts:healthy', + chunkIndex: 0, + startLine: 1, + endLine: 3, + embedding: Array(EMBEDDING_DIMS).fill(0), + contentHash: 'previous-valid', + }; + await batchInsertEmbeddings(adapter.executeWithReusedStatement, [original]); + const replacement = { ...original, contentHash: 'replacement' }; + const invalidUpdate = + invalid === 'duplicate' + ? { ...replacement } + : { + ...replacement, + nodeId: invalid.endsWith('-chunk') ? 'Function:source.ts:other' : invalid, + chunkIndex: + invalid === 'negative-chunk' + ? -1 + : invalid === 'fractional-chunk' + ? 0.5 + : invalid === 'overflow-chunk' + ? 2147483648 + : 0, + }; + const failure = await batchInsertEmbeddings(adapter.executeWithReusedStatement, [ + invalidUpdate, + replacement, + ]).catch((error: unknown) => error); + await adapter.closeLbug(); + await adapter.initLbug(dbPath, { skipFts: true }); + const rows = await adapter.executeQuery( + 'MATCH (e:CodeEmbedding) RETURN e.nodeId AS nodeId, e.contentHash AS contentHash', + ); + expect(rows).toEqual([{ nodeId: original.nodeId, contentHash: original.contentHash }]); + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toMatch(/embedding.*(identity|node.?id|key|chunk)/i); + }); + + it.each([ + { limit: '2048', invalid: 'negative' }, + { limit: '0', invalid: 'negative' }, + { limit: '0', invalid: 'blank' }, + { limit: '2048', invalid: 'duplicate' }, + ])( + 'preserves healthy cached vectors before writeback ($invalid, limit=$limit)', + async ({ limit, invalid }) => { + tmp = await createTempDir('gitnexus-test-corrupt-cache-'); + const repo = path.join(tmp.dbPath, 'repo'); + await fs.mkdir(repo); + vi.stubEnv('GITNEXUS_HOME', path.join(tmp.dbPath, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); + vi.stubEnv('GITNEXUS_EMBEDDING_CACHE_IN_MEMORY_LIMIT', limit); + execFileSync('git', ['init', '-q'], { cwd: repo }); + const source = path.join(repo, 'source.ts'); + await fs.writeFile( + source, + 'export function good() { return 1; }\nexport function bad() { return 2; }\n', + ); + execFileSync('git', ['add', '.'], { cwd: repo }); + execFileSync( + 'git', + [ + '-c', + 'user.name=t', + '-c', + 'user.email=t@t', + '-c', + 'commit.gpgsign=false', + 'commit', + '-qm', + 'initial', + ], + { cwd: repo }, + ); + await runFullAnalysis(repo, { skipFts: true }, { onProgress: () => {} }); + const { lbugPath, storagePath } = getStoragePaths(repo); + const seeded = await seedEmbeddingsForFiles(repo, ['source.ts'], 2); + expect(seeded.get('source.ts')).toHaveLength(2); + await stampEmbeddingCount(storagePath, 2); + const oldMeta = await loadMeta(storagePath); + await adapter.initLbug(lbugPath, { skipFts: true }); + const badId = seeded.get('source.ts')![1]!; + const goodId = seeded.get('source.ts')![0]!; + // Controlled stored corruption, not a reproduction of the host's producer. + await adapter.executeWithReusedStatement( + invalid === 'negative' + ? 'MATCH (e:CodeEmbedding {nodeId: $id}) SET e.chunkIndex = -1' + : 'MATCH (e:CodeEmbedding {nodeId: $id}) SET e.nodeId = $replacement', + [{ id: badId, replacement: invalid === 'blank' ? '' : goodId }], + ); + const query = + 'MATCH (e:CodeEmbedding) RETURN e.id AS id, e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.embedding AS embedding ORDER BY id'; + const before = await adapter.executeQuery(query); + await adapter.closeLbug(); + await fs.appendFile(source, '// incremental update\n'); + const failure = await runFullAnalysis( + repo, + { skipFts: true }, + { onProgress: () => {} }, + ).catch((error: unknown) => error); + await adapter.closeLbug(); + await adapter.initLbug(lbugPath, { skipFts: true }); + expect(await adapter.executeQuery(query)).toEqual(before); + expect(await loadMeta(storagePath)).toEqual(oldMeta); + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toMatch(/embedding.*(identity|node.?id|key|chunk)/i); + }, + 180_000, + ); + + it.each([ + '', + ' ', + 'Function:bad\0id', + 'Function:bad\x01id', + 'Function:bad\uD800id', + 'Function:bad\rid', + 'Function:bad\uFFFFid', + ])('rejects malformed graph ID %j before CSV COPY', async (id) => { + tmp = await createTempDir('gitnexus-test-graph-identity-'); + await adapter.initLbug(path.join(tmp.dbPath, 'lbug'), { skipFts: true }); + const graph = createKnowledgeGraph(); + graph.addNode({ + id, + label: 'Function', + properties: { name: 'bad', filePath: 'source.ts', startLine: 1, endLine: 1 }, + }); + await expect( + adapter.loadGraphToLbug( + graph, + tmp.dbPath, + tmp.dbPath, + undefined, + undefined, + undefined, + 'none', + ), + ).rejects.toThrow(/invalid.*node.*id/i); + expect(await adapter.executeQuery('MATCH (n:Function) RETURN n.id AS id')).toEqual([]); + }); + + it('preserves distinct supplementary Unicode identities through COPY and cold reopen', async () => { + tmp = await createTempDir('gitnexus-test-unicode-identity-'); + const dbPath = path.join(tmp.dbPath, 'lbug'); + await adapter.initLbug(dbPath, { skipFts: true }); + const graph = createKnowledgeGraph(); + const ids = ['Function:source.ts:ab', 'Function:source.ts:a😀b']; + for (const id of ids) { + graph.addNode({ + id, + label: 'Function', + properties: { + name: id, + filePath: 'source.ts', + startLine: 1, + endLine: 1, + }, + }); + } + await adapter.loadGraphToLbug( + graph, + tmp.dbPath, + tmp.dbPath, + undefined, + undefined, + undefined, + 'none', + ); + expect(await adapter.tryFlushWAL()).toBe(true); + await adapter.closeLbug(); + await adapter.initLbug(dbPath, { skipFts: true }); + const rows = await adapter.executeQuery('MATCH (n:Function) RETURN n.id AS id'); + expect(new Set(rows.map((row) => row.id))).toEqual(new Set(ids)); + }); + + it.each(['blank', 'duplicate'])( + 'rejects a persisted %s identity before endpoint calls or replacing valid vectors', + async (invalid) => { + const inputs: string[] = []; + const server = createServer(async (request, response) => { + const pieces: Buffer[] = []; + for await (const piece of request) pieces.push(Buffer.from(piece)); + const body = JSON.parse(Buffer.concat(pieces).toString()) as { input: string[] }; + inputs.push(...body.input); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ + data: body.input.map((_, index) => ({ + index, + embedding: Array(EMBEDDING_DIMS).fill(0), + })), + }), + ); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Expected loopback server'); + vi.stubEnv('GITNEXUS_EMBEDDING_URL', `http://127.0.0.1:${address.port}`); + vi.stubEnv('GITNEXUS_EMBEDDING_MODEL', 'identity-fixture'); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'never'); + tmp = await createTempDir('gitnexus-test-persisted-identity-'); + await adapter.initLbug(path.join(tmp.dbPath, 'lbug'), { skipFts: true }); + // Controlled malformed stored row; this does not claim to reproduce the + // original Windows Java producer that left thousands of blank scan rows. + await adapter.executeQuery( + invalid === 'blank' + ? "CREATE (:Function {id: '', name: 'broken', filePath: 'bad.ts', content: 'export function broken() {}', startLine: 1, endLine: 1})" + : "CREATE (:Method {id: 'Function:good.ts:good', name: 'broken', filePath: 'bad.ts', content: 'broken() {}', startLine: 1, endLine: 1})", + ); + await adapter.executeQuery( + "CREATE (:Function {id: 'Function:good.ts:good', name: 'good', filePath: 'good.ts', content: 'export function good() { return 1; }', startLine: 1, endLine: 1})", + ); + const good = { + nodeId: 'Function:good.ts:good', + chunkIndex: 0, + startLine: 1, + endLine: 1, + embedding: Array(EMBEDDING_DIMS).fill(0), + contentHash: 'old-valid', + }; + await batchInsertEmbeddings(adapter.executeWithReusedStatement, [good]); + const failure = await runEmbeddingPipeline( + adapter.executeQuery, + adapter.executeWithReusedStatement, + () => {}, + {}, + undefined, + new Map([[good.nodeId, good.contentHash]]), + ).catch((error: unknown) => error); + expect(inputs).toEqual([]); + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toMatch(/invalid.*node.*id|duplicate.*node.*id/i); + expect( + await adapter.executeQuery('MATCH (e:CodeEmbedding) RETURN e.contentHash AS contentHash'), + ).toEqual([{ contentHash: 'old-valid' }]); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }, + ); +}); diff --git a/gitnexus/test/integration/shared-store-embedding-reuse.test.ts b/gitnexus/test/integration/shared-store-embedding-reuse.test.ts new file mode 100644 index 000000000..4ca24087b --- /dev/null +++ b/gitnexus/test/integration/shared-store-embedding-reuse.test.ts @@ -0,0 +1,154 @@ +import { execFileSync } from 'node:child_process'; +import { createServer } from 'node:http'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import lbug from '@ladybugdb/core'; +import { expect, it, vi } from 'vitest'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { EMBEDDING_DIMS } from '../../src/core/lbug/schema.js'; +import { getStoragePaths, listRegisteredRepos, loadMeta } from '../../src/storage/repo-manager.js'; +import { createTempDir } from '../helpers/test-db.js'; + +const git = (cwd: string, ...args: string[]) => + execFileSync('git', args, { cwd, encoding: 'utf8', stdio: 'pipe' }).trim(); + +interface Row { + id: string; + nodeId: string; + chunkIndex: number; + hash: string; +} + +// Each receipt uses a fresh native read-only handle after analyze has closed. +async function readRows(dbPath: string): Promise { + const db = new lbug.Database(dbPath, 256 * 1024 * 1024, true, true); + const conn = new lbug.Connection(db); + try { + const result = await conn.query( + 'MATCH (e:CodeEmbedding) RETURN e.id AS id, e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.contentHash AS hash ORDER BY id', + ); + if (Array.isArray(result)) throw new Error('Expected one result'); + try { + return (await result.getAll()) as unknown as Row[]; + } finally { + await result.close(); + } + } finally { + await conn.close(); + await db.close(); + } +} + +it('reuses sibling vectors across dirty clone copies, shrinking chunks, retries and rebuilds (#3474)', async () => { + const tmp = await createTempDir('gn-shared-embeddings-'); + const root = await fs.realpath(tmp.dbPath); + const first = path.join(root, 'first'); + const second = path.join(root, 'second'); + const submitted: string[] = []; + const server = createServer(async (request, response) => { + const pieces: Buffer[] = []; + for await (const piece of request) pieces.push(Buffer.from(piece)); + const { input } = JSON.parse(Buffer.concat(pieces).toString()) as { input: string[] }; + submitted.push(...input); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ + data: input.map((_, index) => ({ + index, + embedding: Array(EMBEDDING_DIMS).fill(0.125), + })), + }), + ); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Expected loopback endpoint'); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_SHARED_STORE', ''); + vi.stubEnv('GITNEXUS_EMBEDDING_URL', `http://127.0.0.1:${address.port}`); + vi.stubEnv('GITNEXUS_EMBEDDING_MODEL', 'shared-store-fixture'); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'auto'); + await fs.mkdir(first); + git(first, 'init', '-q', '-b', 'main'); + git(first, 'remote', 'add', 'origin', 'https://example.invalid/shared/embeddings.git'); + for (let i = 0; i < 12; i++) { + const body = + i === 0 + ? Array.from({ length: 100 }, (_, n) => ` console.log('original line ${n}');`).join('\n') + : ` return 'healthy ${i}';`; + await fs.writeFile( + path.join(first, `source${i}.ts`), + `export function item${i}() {\n${body}\n}\n`, + ); + } + git(first, 'add', '-A'); + git(first, '-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'fixture'); + git(root, 'clone', '-q', first, second); + git(second, 'remote', 'set-url', 'origin', 'https://example.invalid/shared/embeddings.git'); + const logs: string[] = []; + const analyze = (repo: string, force = false) => + runFullAnalysis( + repo, + { + embeddings: true, + embeddingsNodeLimit: 0, + force, + skipAgentsMd: true, + skipSkills: true, + }, + { onProgress: () => {}, onLog: (message) => logs.push(message) }, + ); + const graph = async (repo: string) => { + const registration = (await listRegisteredRepos()).find((entry) => entry.path === repo); + expect(registration).toBeDefined(); + return getStoragePaths(repo, undefined, registration!.storagePath); + }; + await analyze(first); + const before = await readRows((await graph(first)).lbugPath); + expect(new Set(before.map((row) => row.nodeId)).size).toBe(12); + expect( + before.filter((row) => row.nodeId === 'Function:source0.ts:item0').length, + ).toBeGreaterThan(1); + submitted.length = 0; + await analyze(second); // Establish the shared store from two standalone clones. + submitted.length = 0; + await analyze(first); // Both members now point at the same immutable commit graph. + expect((await graph(first)).lbugPath).toBe((await graph(second)).lbugPath); + expect(submitted, logs.join('\n')).toEqual([]); + + await fs.writeFile( + path.join(second, 'source0.ts'), + "export function item0() { return 'changed'; }\n", + ); + logs.length = 0; + await analyze(second); + expect( + logs.some((message) => message.startsWith('Shared store: copied the shared graph')), + ).toBe(true); + expect(logs.some((message) => message.startsWith('Incremental:'))).toBe(true); + expect(submitted.length).toBeGreaterThan(0); + expect(submitted.every((text) => text.includes('source0.ts'))).toBe(true); + const changedGraph = await graph(second); + const after = await readRows(changedGraph.lbugPath); + expect(after.filter((row) => row.nodeId === 'Function:source0.ts:item0')).toHaveLength(1); + expect(after.filter((row) => row.nodeId !== 'Function:source0.ts:item0')).toEqual( + before.filter((row) => row.nodeId !== 'Function:source0.ts:item0'), + ); + expect(new Set(after.map((row) => row.id)).size).toBe(after.length); + expect(await readRows((await graph(first)).lbugPath)).toEqual(before); + expect((await loadMeta(changedGraph.storagePath))?.embeddingCheckpoint).toBeUndefined(); + + submitted.length = 0; + await analyze(second); + await analyze(second, true); + expect(submitted).toEqual([]); + expect(await readRows((await graph(second)).lbugPath)).toEqual(after); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + vi.unstubAllEnvs(); + await tmp.cleanup(); + } +}, 240_000); diff --git a/gitnexus/test/unit/csv-escaping.test.ts b/gitnexus/test/unit/csv-escaping.test.ts index 2e8d8a36a..4887fea10 100644 --- a/gitnexus/test/unit/csv-escaping.test.ts +++ b/gitnexus/test/unit/csv-escaping.test.ts @@ -5,6 +5,7 @@ * Covers hardening fix #23 (keyword arrays with backslashes and commas) */ import { describe, it, expect } from 'vitest'; +import { assertValidGraphNodeId } from '../../src/core/graph/node-identity.js'; import { escapeCSVField, escapeCSVNumber, @@ -141,11 +142,40 @@ describe('sanitizeUTF8', () => { expect(sanitizeUTF8('hello\uD800world')).toBe('helloworld'); }); + it('preserves valid supplementary Unicode characters', () => { + expect(sanitizeUTF8('hello😀𐐀world')).toBe('hello😀𐐀world'); + }); + it('strips BOM-like characters (FFFE/FFFF)', () => { expect(sanitizeUTF8('hello\uFFFEworld')).toBe('helloworld'); }); }); +describe('graph identity CSV fidelity', () => { + it.each([ + ...Array.from({ length: 32 }, (_, code) => String.fromCharCode(code)).filter( + (char) => char !== '\t' && char !== '\n', + ), + '\x7F', + '\uD800', + '\uDFFF', + '\uFFFE', + '\uFFFF', + ])('rejects a character that CSV would alter: %j', (char) => { + expect(() => assertValidGraphNodeId(`Function:source.ts:a${char}b`, 'Graph COPY')).toThrow( + /invalid graph node ID/, + ); + }); + + it.each(['Function:café.ts:😀', 'Function:𐐀.ts:method', 'Function:a\tb.ts:line\nname'])( + 'preserves accepted identity %j through CSV escaping', + (id) => { + expect(() => assertValidGraphNodeId(id, 'Graph COPY')).not.toThrow(); + expect(escapeCSVField(id)).toBe(`"${id}"`); + }, + ); +}); + // ─── isBinaryContent ───────────────────────────────────────────────── describe('isBinaryContent', () => { diff --git a/gitnexus/test/unit/embedding-pipeline.test.ts b/gitnexus/test/unit/embedding-pipeline.test.ts index 51f6c011e..96e41930b 100644 --- a/gitnexus/test/unit/embedding-pipeline.test.ts +++ b/gitnexus/test/unit/embedding-pipeline.test.ts @@ -368,6 +368,22 @@ describe('runEmbeddingPipeline incremental filter', () => { progressUpdates.push({ ...p }); }; + it('fails a graph read without reporting a complete embedding index', async () => { + mockEmbedderSetup(); + const { runEmbeddingPipeline } = + await import('../../src/core/embeddings/embedding-pipeline.js'); + const executeQuery = mockExecuteQuery([makeNode()]); + executeQuery.mockRejectedValueOnce(new Error('injected graph read failure')); + const executeStatement = mockExecuteWithReusedStatement(); + await expect(runEmbeddingPipeline(executeQuery, executeStatement, onProgress)).rejects.toThrow( + /read.*Function.*injected graph read failure/, + ); + expect(stmtCalls).toEqual([]); + expect(vectorIndexMock).not.toHaveBeenCalled(); + expect(progressUpdates.at(-1)?.phase).toBe('error'); + expect(progressUpdates.some((update) => update.phase === 'ready')).toBe(false); + }); + it('falls back to text-bearing File nodes when a repo has no code symbols', async () => { mockEmbedderSetup(); diff --git a/gitnexus/test/unit/incremental-write-integrity.test.ts b/gitnexus/test/unit/incremental-write-integrity.test.ts index bd201851a..5ed597e8d 100644 --- a/gitnexus/test/unit/incremental-write-integrity.test.ts +++ b/gitnexus/test/unit/incremental-write-integrity.test.ts @@ -28,6 +28,7 @@ async function touchHandler(repoPath: string): Promise { describe('incremental graph identity before publication', () => { it('chooses a full write before selective mutation when manual checkpoints are disabled', async () => { + vi.stubEnv('GITNEXUS_ATOMIC_WINDOWS_SWAP', '0'); const repo = await setupMiniRepo('gitnexus-test-checkpoint-opt-out-'); const { storagePath } = getStoragePaths(repo.dbPath); try { @@ -42,7 +43,14 @@ describe('incremental graph identity before publication', () => { onLog: (line) => logs.push(line), }); expect(deleteNodes).not.toHaveBeenCalled(); - expect(flush).not.toHaveBeenCalled(); + // The opt-out disables periodic/incremental checkpoints, but a staged + // full rebuild still requires one verified checkpoint before publishing. + if (process.platform === 'win32') { + expect(flush).not.toHaveBeenCalled(); // Default Windows writes stay in place. + } else { + expect(flush).toHaveBeenCalledTimes(1); + await expect(flush.mock.results[0].value).resolves.toBe(true); + } expect(result.incrementalStats).toBeUndefined(); expect(logs).toContain( 'Manual WAL checkpoints are disabled; switching to a full DB write before mutation.', @@ -52,7 +60,7 @@ describe('incremental graph identity before publication', () => { await adapter.closeLbug(); await repo.cleanup(); } - }); + }, 120_000); it('retries a transient checkpoint I/O failure at the final publication gate', async () => { const repo = await setupMiniRepo('gitnexus-test-final-checkpoint-retry-'); @@ -85,7 +93,7 @@ describe('incremental graph identity before publication', () => { await adapter.closeLbug(); await repo.cleanup(); } - }); + }, 120_000); it('matches native CSV normalization for nullable, absent-range and Unicode identity fields', async () => { const repo = await setupMiniRepo('gitnexus-test-identity-parity-'); const { storagePath, lbugPath } = getStoragePaths(repo.dbPath); @@ -111,7 +119,9 @@ describe('incremental graph identity before publication', () => { properties: { name: 'topic', filePath: '' }, }); graph.addNode({ - id: 'Function:src/Å.ts:王\u0000\uD800', + // IDs must be valid before CSV encoding; malformed-ID rejection has its + // own real persistence coverage in embedding-identity-integrity.test.ts. + id: 'Function:src/Å.ts:王', label: 'Function', properties: { name: '王\r\n\uD800', filePath: 'src/Å.ts' }, }); @@ -146,6 +156,8 @@ describe('incremental graph identity before publication', () => { ])( 'refuses freshness after $phase failure (atomic=$atomicIncremental)', async ({ phase, atomicIncremental }) => { + // Windows uses in-place writes even when a caller requests staging. + const expectsStaging = atomicIncremental && process.platform !== 'win32'; const repo = await setupMiniRepo('gitnexus-test-write-integrity-'); const { storagePath, lbugPath } = getStoragePaths(repo.dbPath); try { @@ -215,14 +227,14 @@ describe('incremental graph identity before publication', () => { expect(after?.lastCommit).toBe(before?.lastCommit); expect(after?.indexedAt).toBe(before?.indexedAt); expect(logs.filter((line) => line.includes('atomic-incremental'))).toEqual( - atomicIncremental ? [expect.stringContaining('staged')] : [], + expectsStaging ? [expect.stringContaining('staged')] : [], ); - expect(Boolean(after?.incrementalInProgress)).toBe(!atomicIncremental); + expect(Boolean(after?.incrementalInProgress)).toBe(!expectsStaging); const registeredAfter = (await readRegistry()).find((entry) => entry.path === repo.dbPath); expect(registeredAfter?.indexedAt).toBe(registeredBefore?.indexedAt); expect(registeredAfter?.lastCommit).toBe(registeredBefore?.lastCommit); vi.restoreAllMocks(); - if (atomicIncremental) { + if (expectsStaging) { await adapter.initLbug(lbugPath); const retained = await adapter.executePrepared( 'MATCH (n:Function {id: $id}) RETURN n.id AS id', diff --git a/gitnexus/test/unit/lbug-publication.test.ts b/gitnexus/test/unit/lbug-publication.test.ts new file mode 100644 index 000000000..8bae82494 --- /dev/null +++ b/gitnexus/test/unit/lbug-publication.test.ts @@ -0,0 +1,53 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, expect, it } from 'vitest'; +import { assertLbugPublicationReady } from '../../src/core/lbug/publication.js'; + +let root: string; +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publication-')); +}); +afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); +}); + +it('allows a settling sidecar to disappear before the bounded publication check', async () => { + const db = path.join(root, 'lbug'); + await fs.writeFile(`${db}.wal`, 'settling'); + const removal = new Promise((resolve, reject) => + setTimeout(() => { + fs.unlink(`${db}.wal`).then(resolve, reject); + }, 10), + ); + await assertLbugPublicationReady(db); + await removal; +}); + +it.each([ + '.wal', + '.shadow', + '.wal.checkpoint', + '.checkpoint.intent.lock', + '.checkpoint.apply.lock', + '.wal.missing-shadow.1234-abcd', + '.shadow.orphan.1234', + '.wal.dirty-recovery.next', + '.shadow.dirty-recovery.next', +])('refuses a persistent %s and names the affected database', async (suffix) => { + const db = path.join(root, 'lbug'); + await fs.writeFile(`${db}${suffix}`, 'pending recovery'); + await expect(assertLbugPublicationReady(db)).rejects.toThrow(db); + expect(await fs.readFile(`${db}${suffix}`, 'utf8')).toBe('pending recovery'); +}); + +it('retains completed dirty-recovery archives without blocking publication', async () => { + const db = path.join(root, 'lbug'); + for (const suffix of ['.wal.dirty-recovery', '.shadow.dirty-recovery']) { + await fs.writeFile(`${db}${suffix}`, 'retained crash evidence'); + } + await assertLbugPublicationReady(db); + for (const suffix of ['.wal.dirty-recovery', '.shadow.dirty-recovery']) { + expect(await fs.readFile(`${db}${suffix}`, 'utf8')).toBe('retained crash evidence'); + } +}); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 65d56ea2c..c2746766b 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -2638,6 +2638,7 @@ describe('runFullAnalysis embedding-checkpoint resilience (#2790 review)', () => embeddings: [], })); vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ + acquireInitLock: vi.fn(async () => async () => {}), initLbug: vi.fn(async () => undefined), loadGraphToLbug: vi.fn(async () => undefined), getLbugStats: vi.fn(async () => ({ nodes: 2, edges: 0, communities: 0, processes: 0 })),