From 2251adeeb2fb50cc8bbf52060f066dfe88997f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 17 Oct 2026 17:03:42 +0300 Subject: [PATCH] fix(eval): grade real C include provenance (#3611) * fix(eval): verify independent C repairs and reject broken controls * fix(eval): replace invalid C task with provenance behavior * fix(eval): grade real C include provenance * test(eval): add independent typed C reference repairs * docs(eval): record C oracle evidence contract * fix(eval): remove invalid synthetic C oracle * fix(eval): remove type-invalid C reference * docs: remove C oracle planning document * test(eval): tolerate formatting in C oracle mutants * test(eval): clarify C oracle node and file bindings --- eval/tests/test_oracle_assets.py | 106 +++++++++++- eval/tests/test_workflow_bench.py | 11 +- eval/workflow_bench/README.md | 24 +++ .../inv-bug-c-system-include.oracle.test.ts | 28 ---- ...eature-c-include-provenance.oracle.test.ts | 67 ++++++++ .../reference/inv-bug-c-system-include.patch | 12 -- ...ature-c-include-provenance-delimiter.patch | 146 +++++++++++++++++ ...nv-feature-c-include-provenance-flag.patch | 155 ++++++++++++++++++ eval/workflow_bench/tasks.scenarios.yaml | 39 +++-- 9 files changed, 527 insertions(+), 61 deletions(-) delete mode 100644 eval/workflow_bench/oracles/inv-bug-c-system-include.oracle.test.ts create mode 100644 eval/workflow_bench/oracles/inv-feature-c-include-provenance.oracle.test.ts delete mode 100644 eval/workflow_bench/oracles/reference/inv-bug-c-system-include.patch create mode 100644 eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-delimiter.patch create mode 100644 eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-flag.patch diff --git a/eval/tests/test_oracle_assets.py b/eval/tests/test_oracle_assets.py index 29ebda2ee..6314d99b8 100644 --- a/eval/tests/test_oracle_assets.py +++ b/eval/tests/test_oracle_assets.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import hashlib import os +import re import shutil import subprocess import sys @@ -589,6 +590,9 @@ def scenario_control_base(tmp_path_factory: pytest.TempPathFactory) -> tuple[Pat ref, "gitnexus/src", "gitnexus/vendor", + "gitnexus/test", + "gitnexus/tsconfig.json", + "gitnexus/vitest.config.ts", "gitnexus/package.json", "gitnexus-shared/package.json", ], @@ -607,7 +611,7 @@ def scenario_control_base(tmp_path_factory: pytest.TempPathFactory) -> tuple[Pat pytest.skip("oracle controls need tarfile extraction filters (Python 3.11.4+)") with tarfile.open(archive) as source: source.extractall(base, filter="data") - for relative in ("node_modules", "gitnexus/node_modules", "gitnexus-shared/dist"): + for relative in ("node_modules", "gitnexus/node_modules", "gitnexus-shared/node_modules", "gitnexus-shared/dist"): (base / relative).symlink_to(dependencies / relative, target_is_directory=True) subprocess.run(["git", "init", "--quiet", str(base)], check=True, capture_output=True) return base, {task["id"]: task for task in tasks} @@ -633,12 +637,82 @@ def _verify_shipped_oracle(candidate: Path, task: dict[str, object]) -> subproce ) +def _assert_behavioral_rejection(completed: subprocess.CompletedProcess[str], assertion: str = "") -> None: + output = completed.stdout + completed.stderr + assert completed.returncode == 1, output + assert "AssertionError" in output, output + assert assertion in output, output + # An import/setup crash is not evidence that a behavioral control worked. + assert "Failed Suites" not in output, output + assert "Unhandled Errors" not in output, output + + +def _check_c_provenance_mutants(candidate: Path, base: Path, task: dict[str, object]) -> None: + """Reject broken behavior, and distinguish module-load errors from evidence.""" + + interpret = Path("gitnexus/src/core/ingestion/languages/c/interpret.ts") + resolver = Path("gitnexus/src/core/ingestion/languages/c/scope-resolver.ts") + valid_interpret = (candidate / interpret).read_text() + valid_resolver = (candidate / resolver).read_text() + system_guards = re.findall( + r"\bconst\s+parsedImport\s*=\s*context\s*\?\.\s*parsedImport\s*;\s*" + r"if\s*\(\s*parsedImport\s*\?\.\s*kind\s*===\s*'wildcard'\s*&&\s*" + r"'isSystem'\s+in\s+parsedImport\s*&&\s*parsedImport\s*\.\s*isSystem\s*===\s*true\s*\)\s*" + r"\{\s*return\s+null\s*;\s*\}", + valid_resolver, + ) + fallbacks = re.findall( + r"\breturn\s+resolveCImportTarget\s*\(\s*targetRaw\s*,\s*fromFile\s*,\s*allFilePaths\s*,?\s*\)\s*;", + valid_resolver, + ) + assert len(system_guards) == 1, "expected exactly one C system-include guard" + assert len(fallbacks) == 1, "expected exactly one C resolver fallback" + system_guard, fallback = system_guards[0], fallbacks[0] + late_guard_resolver = valid_resolver.replace(system_guard, "").replace(fallback, system_guard + "\n " + fallback) + assert valid_interpret.count("return parsedImport;") == 1 + assert valid_interpret.count("targetRaw: source,") == 1 + mutants = [ + (interpret, (base / interpret).read_text(), "literal include must be retained"), + ( + interpret, + valid_interpret.replace("return parsedImport;", "return null;"), + 'literal include "stdio.h" must be retained', + ), + (interpret, valid_interpret.replace("targetRaw: source,", "targetRaw: '',"), "retained header identity"), + (resolver, (base / resolver).read_text(), "include resolution"), + (resolver, late_guard_resolver, "include resolution"), + ] + for relative, broken, assertion in mutants: + path = candidate / relative + original = path.read_text() + assert broken != original + path.write_text(broken) + try: + _assert_behavioral_rejection(_verify_shipped_oracle(candidate, task), assertion) + finally: + path.write_text(original) + + # A missing native-parser module must fail setup, never satisfy a negative + # behavioral control just because Vitest exits nonzero. + parser = candidate / "gitnexus/src/core/ingestion/languages/c/query.ts" + original_parser = parser.read_bytes() + parser.unlink() + try: + broken_setup = _verify_shipped_oracle(candidate, task) + assert "Failed Suites" in broken_setup.stdout + broken_setup.stderr + with pytest.raises(AssertionError): + _assert_behavioral_rejection(broken_setup, "literal include must be retained") + finally: + parser.write_bytes(original_parser) + + @pytest.mark.skipif(os.name == "nt", reason="oracle controls use POSIX dependency symlinks and shell commands") @pytest.mark.parametrize( ("task_id", "reference"), [ ("trivial-status-json-alias", "trivial-status-json-alias"), - ("inv-bug-c-system-include", "inv-bug-c-system-include"), + ("inv-feature-c-include-provenance", "inv-feature-c-include-provenance-flag"), + ("inv-feature-c-include-provenance", "inv-feature-c-include-provenance-delimiter"), ("inv-feature-list-repos-filter", "inv-feature-list-repos-filter"), ("cross-module-parse-retry", "cross-module-parse-retry"), ("cross-module-parse-retry", "cross-module-parse-retry-pool"), @@ -653,13 +727,37 @@ def test_shipped_oracles_reject_task_base_and_accept_reference_fixes( base, tasks = scenario_control_base candidate = _oracle_control_candidate(base, tmp_path / "candidate") negative = _verify_shipped_oracle(candidate, tasks[task_id]) - assert negative.returncode == 1, negative.stdout + negative.stderr - assert "AssertionError" in negative.stdout + negative.stderr + _assert_behavioral_rejection( + negative, + "literal include must be retained" if task_id == "inv-feature-c-include-provenance" else "", + ) patches = Path(__file__).resolve().parents[1] / "workflow_bench/oracles/reference" subprocess.run(["git", "apply", str(patches / f"{reference}.patch")], cwd=candidate, check=True) positive = _verify_shipped_oracle(candidate, tasks[task_id]) assert positive.returncode == 0, positive.stdout + positive.stderr + if task_id == "inv-feature-c-include-provenance": + # A reference passing transpile-only Vitest is not enough: use the same + # typed, authored verification that an evaluated repair must satisfy. + authored = subprocess.run( + ["/bin/sh", "-c", str(tasks[task_id]["verify"])], + cwd=candidate, + capture_output=True, + text=True, + check=False, + timeout=180, + ) + assert authored.returncode == 0, authored.stdout + authored.stderr + if reference == "inv-feature-c-include-provenance-flag": + # This entry point is already selected by native CI; keep every C + # validity control reachable without changing workflow selection. + # Indent every existing code line differently without matching the + # reference's formatting; the behavioral controls must still work. + resolver = candidate / "gitnexus/src/core/ingestion/languages/c/scope-resolver.ts" + resolver.write_text(re.sub(r"(?m)^([ \t]+)", r"\t\1", resolver.read_text())) + positive = _verify_shipped_oracle(candidate, tasks[task_id]) + assert positive.returncode == 0, positive.stdout + positive.stderr + _check_c_provenance_mutants(candidate, base, tasks[task_id]) @pytest.mark.skipif(os.name == "nt", reason="oracle controls use POSIX dependency symlinks and shell commands") diff --git a/eval/tests/test_workflow_bench.py b/eval/tests/test_workflow_bench.py index 51387fcde..3f4ed5d30 100644 --- a/eval/tests/test_workflow_bench.py +++ b/eval/tests/test_workflow_bench.py @@ -347,11 +347,18 @@ def test_shipped_scenarios_opt_out_the_cross_module_cell_and_rebuild_graph_asset selected, skipped = select_tasks(tasks, include_expensive=False) assert [task["id"] for task in selected] == [ "trivial-status-json-alias", - "inv-bug-c-system-include", + "inv-feature-c-include-provenance", "inv-feature-list-repos-filter", ] assert skipped == ["cross-module-parse-retry"] - # v1.6.12 predates all four fixes, including the C fix already on main. + assert len(tasks) == 4 + c_task = next(task for task in tasks if task["id"] == "inv-feature-c-include-provenance") + assert c_task["class"] == "investigation-feature" + assert "npx tsc --noEmit" in c_task["verify"] + assert "test/unit/scope-resolution/c/c-include-provenance.test.ts" in c_task["verify"] + assert "test/unit/scope-resolution/external-import-conformance.test.ts" in c_task["verify"] + assert not c_task.get("setup") + # v1.6.12 lacks all four requested behaviors, including retained C include provenance. assert {task["ref"] for task in tasks} == {"c4ecf398de637cf779ab2238e69958cfd5d601ce"} assert all(not task.get("sandbox_copy") for task in tasks) assert all(task["sandbox_dependencies"] for task in tasks) diff --git a/eval/workflow_bench/README.md b/eval/workflow_bench/README.md index e8002e219..a972721d0 100644 --- a/eval/workflow_bench/README.md +++ b/eval/workflow_bench/README.md @@ -786,6 +786,30 @@ the shared `vitest.config.mts` as an oracle file and pass it explicitly with `--config`. Prefer `verify` commands that use the repo's own npm scripts (they carry build pre-hooks). +The C scenario is `inv-feature-c-include-provenance`. At its pinned task +commit, ingestion drops literal system includes before resolution. The task +requires retaining the header identity and enough provenance to prevent +unconfigured system includes from matching unrelated repository headers, +while keeping quoted local and relative includes resolvable. Computed macro +includes remain outside this task's scope. Its hidden oracle parses real C +source and passes the resulting import through the decomposer, interpreter, +and resolver; it does not require a particular metadata field or encoding. + +Offline controls must show that the untouched task base fails the intended +behavioral assertion and that two independent, type-correct repairs pass both +the hidden oracle and the complete authored verification command. Negative +controls also reject dropped imports, lost header identity, and false system +header matches. These controls run with the pinned task dependencies through +`test_shipped_oracles_reject_task_base_and_accept_reference_fixes`. + +This replaces `inv-bug-c-system-include`, whose oracle constructed an import +state the pinned parser did not produce and constrained the repair's internal +representation. Historical scores for that task are not valid evidence of C +ingestion quality and are not comparable with the replacement task. The new +task ID, prompt, verification command, and oracle bytes produce a new task +digest. Old agent patches are not available for rescoring; keep historical +measurements unchanged and compare only runs with matching task contracts. + ## Relation to the SWE-bench harness The rest of `eval/` benchmarks GitNexus _tools_ inside a litellm agent loop diff --git a/eval/workflow_bench/oracles/inv-bug-c-system-include.oracle.test.ts b/eval/workflow_bench/oracles/inv-bug-c-system-include.oracle.test.ts deleted file mode 100644 index 082dc4990..000000000 --- a/eval/workflow_bench/oracles/inv-bug-c-system-include.oracle.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { cScopeResolver } from '../gitnexus/src/core/ingestion/languages/c/scope-resolver.js'; - -describe('hidden oracle: C system headers do not bind to repository decoys', () => { - it.each([ - ['stdio.h', true, null], - ['project.h', true, null], - ['stdio.h', false, 'src/stdio.h'], - ['util.h', false, 'include/util.h'], - ] as const)( - 'resolves %s with isSystem=%s from its include form', - (targetRaw, isSystem, expected) => { - const files = new Set(['src/stdio.h', 'include/project.h', 'include/util.h', 'src/main.c']); - const parsedImport = { kind: 'wildcard' as const, targetRaw, isSystem }; - - // Both spellings name stdio.h. Preserve the parsed include form; - // refusing every header by name would break the quoted local include. - const resolved = cScopeResolver.resolveImportTarget( - parsedImport.targetRaw, - 'src/main.c', - files, - undefined, - { parsedFiles: [], parsedImport }, - ); - expect(resolved).toBe(expected); - }, - ); -}); diff --git a/eval/workflow_bench/oracles/inv-feature-c-include-provenance.oracle.test.ts b/eval/workflow_bench/oracles/inv-feature-c-include-provenance.oracle.test.ts new file mode 100644 index 000000000..bdab94bb9 --- /dev/null +++ b/eval/workflow_bench/oracles/inv-feature-c-include-provenance.oracle.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; +import { getCParser } from '../gitnexus/src/core/ingestion/languages/c/query.js'; +import { splitCInclude } from '../gitnexus/src/core/ingestion/languages/c/import-decomposer.js'; +import { interpretCImport } from '../gitnexus/src/core/ingestion/languages/c/interpret.js'; +import { cScopeResolver } from '../gitnexus/src/core/ingestion/languages/c/scope-resolver.js'; + +function parseInclude(spelling: string) { + const tree = getCParser().parse(`#include ${spelling}\n`); + const node = tree.rootNode.namedChildren.find((child) => child.type === 'preproc_include'); + if (node === undefined) throw new Error('native parser must produce an include node'); + const captures = splitCInclude(node); + return captures === null ? null : interpretCImport(captures); +} + +// Header identity is public behavior. Bare targets or targets carrying their +// literal delimiters are both valid; no private marker field is prescribed. +function headerIdentity(target: string): string { + if ( + (target.startsWith('<') && target.endsWith('>')) || + (target.startsWith('"') && target.endsWith('"')) + ) + return target.slice(1, -1); + return target; +} + +const workspaceFiles = new Set([ + 'src/main.c', + 'src/stdio.h', + 'include/project.h', + 'src/local.h', + 'include/util.h', +]); + +describe.each([ + { name: 'workspace headers', files: workspaceFiles, resolutionConfig: undefined }, + { + name: 'discovered headers', + files: new Set(['src/main.c']), + resolutionConfig: new Set([...workspaceFiles].filter((file) => file.endsWith('.h'))), + }, +])('hidden oracle: retained C include provenance ($name)', ({ files, resolutionConfig }) => { + it.each([ + ['', 'stdio.h', null], + ['', 'project.h', null], + ['"stdio.h"', 'stdio.h', 'src/stdio.h'], + ['"local.h"', 'local.h', 'src/local.h'], + ['"../include/util.h"', '../include/util.h', 'include/util.h'], + ] as const)('retains and resolves literal %s', (spelling, header, expected) => { + const parsedImport = parseInclude(spelling); + expect(parsedImport, `literal include ${spelling} must be retained`).not.toBeNull(); + expect(parsedImport!.kind).toBe('wildcard'); + expect(headerIdentity(parsedImport!.targetRaw), 'retained header identity').toBe(header); + const resolved = cScopeResolver.resolveImportTarget( + parsedImport!.targetRaw, + 'src/main.c', + files, + resolutionConfig, + { parsedFiles: [], parsedImport: parsedImport! }, + ); + const targets = resolved === null ? [] : typeof resolved === 'string' ? [resolved] : resolved; + expect(targets, `include ${spelling} resolution`).toEqual(expected === null ? [] : [expected]); + }); + + it('does not invent a literal import for a computed macro include', () => { + expect(parseInclude('HEADER_MACRO')).toBeNull(); + }); +}); diff --git a/eval/workflow_bench/oracles/reference/inv-bug-c-system-include.patch b/eval/workflow_bench/oracles/reference/inv-bug-c-system-include.patch deleted file mode 100644 index 49ac341fc..000000000 --- a/eval/workflow_bench/oracles/reference/inv-bug-c-system-include.patch +++ /dev/null @@ -1,12 +0,0 @@ ---- a/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts -+++ b/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts -@@ -81,7 +81,8 @@ - // this process. Runs BEFORE `populateOwners`. - applyCaptureSideChannel: applyCStaticLinkageSideChannel, - -- resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => { -+ resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig, context) => { -+ if (context?.parsedImport?.kind === 'wildcard' && context.parsedImport.isSystem) return null; - // Augment allFilePaths with .h files discovered via loadResolutionConfig - // since the phase only passes .c files to the C resolver but #include - // targets .h files classified as C++ in language detection. diff --git a/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-delimiter.patch b/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-delimiter.patch new file mode 100644 index 000000000..be802e723 --- /dev/null +++ b/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-delimiter.patch @@ -0,0 +1,146 @@ +--- a/gitnexus/src/core/ingestion/languages/c/interpret.ts ++++ b/gitnexus/src/core/ingestion/languages/c/interpret.ts +@@ -8,10 +8,9 @@ + const source = captures['@import.source']?.text; + if (source === undefined) return null; + +- // System headers (e.g. ) are not resolved to local files +- if (captures['@import.system'] !== undefined) return null; +- +- return { kind: 'wildcard', targetRaw: source }; ++ // Preserve the literal spelling in the existing target field. ++ const targetRaw = captures['@import.system'] === undefined ? source : `<${source}>`; ++ return { kind: 'wildcard', targetRaw }; + } + + /** +--- a/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts ++++ b/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts +@@ -82,6 +82,7 @@ + applyCaptureSideChannel: applyCStaticLinkageSideChannel, + + resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => { ++ if (targetRaw.startsWith('<') && targetRaw.endsWith('>')) return null; + // Augment allFilePaths with .h files discovered via loadResolutionConfig + // since the phase only passes .c files to the C resolver but #include + // targets .h files classified as C++ in language detection. +--- a/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts ++++ b/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts +@@ -38,6 +38,10 @@ + import { describe, expect, it } from 'vitest'; + import type { ParsedFile, ParsedImport, ScopeId, SymbolDefinition } from 'gitnexus-shared'; + import { SupportedLanguages } from 'gitnexus-shared'; ++import { getCParser } from '../../../src/core/ingestion/languages/c/query.js'; ++import { splitCInclude } from '../../../src/core/ingestion/languages/c/import-decomposer.js'; ++import { interpretCImport } from '../../../src/core/ingestion/languages/c/interpret.js'; ++import type { SyntaxNode } from '../../../src/core/ingestion/utils/ast-helpers.js'; + import { SCOPE_RESOLVERS } from '../../../src/core/ingestion/scope-resolution/pipeline/registry.js'; + import type { ComposerConfig } from '../../../src/core/ingestion/language-config.js'; + import { +@@ -143,6 +147,16 @@ + }; + } + ++function parseCInclude(spelling: string): ParsedImport { ++ const tree = getCParser().parse(`#include ${spelling}\n`); ++ const node = tree.rootNode.namedChildren.find((child) => child.type === 'preproc_include'); ++ if (!node) throw new Error('C conformance fixture did not parse'); ++ const captures = splitCInclude(node as SyntaxNode); ++ const parsedImport = captures === null ? null : interpretCImport(captures); ++ if (parsedImport === null) throw new Error('C literal include was not retained'); ++ return parsedImport; ++} ++ + const PHP_FUNCTION_IMPORT = (targetRaw: string): ParsedImport => ({ + kind: 'named', + localName: 'imported', +@@ -345,9 +359,10 @@ + files: ['src/stdio.h', 'include/util.h', 'src/main.c'], + fromFile: 'src/main.c', + resolutionConfig: undefined, +- external: 'stdio.h', ++ external: '', + decoy: 'src/stdio.h', +- reachesDecoy: 'util.h', ++ reachesDecoy: '"stdio.h"', ++ parsedImport: parseCInclude, + }, + ], + [ +@@ -418,7 +433,6 @@ + const KNOWN_GAPS: ReadonlyMap = new Map([ + [SupportedLanguages.Dart, '`package:http/http.dart` -> `lib/http.dart`'], + [SupportedLanguages.Swift, '`Foundation` -> `Sources/Foundation/Thing.swift`'], +- [SupportedLanguages.C, '`stdio.h` -> `src/stdio.h`'], + [SupportedLanguages.CPlusPlus, '`cstdio.h` -> `src/cstdio.h`'], + ]); + +@@ -463,14 +477,15 @@ + : testCase.files.map( + testCase.parsedFile ?? ((filePath) => ({ filePath }) as unknown as ParsedFile), + ); ++ const parsedImport = testCase.parsedImport?.(targetRaw); + return resolver.resolveImportTarget( +- targetRaw, ++ parsedImport?.targetRaw ?? targetRaw, + testCase.fromFile, + files, + testCase.resolutionConfig, + { + parsedFiles, +- parsedImport: testCase.parsedImport?.(targetRaw), ++ parsedImport, + }, + ); + } +--- /dev/null ++++ b/gitnexus/test/unit/scope-resolution/c/c-include-provenance.test.ts +@@ -0,0 +1,48 @@ ++import { describe, expect, it } from 'vitest'; ++import { getCParser } from '../../../../src/core/ingestion/languages/c/query.js'; ++import { splitCInclude } from '../../../../src/core/ingestion/languages/c/import-decomposer.js'; ++import { interpretCImport } from '../../../../src/core/ingestion/languages/c/interpret.js'; ++import { cScopeResolver } from '../../../../src/core/ingestion/languages/c/scope-resolver.js'; ++import type { SyntaxNode } from '../../../../src/core/ingestion/utils/ast-helpers.js'; ++ ++function parseIncludes(source: string) { ++ return getCParser().parse(source).rootNode.namedChildren ++ .filter((node) => node.type === 'preproc_include') ++ .map((node) => { ++ const captures = splitCInclude(node as SyntaxNode); ++ return captures === null ? null : interpretCImport(captures); ++ }); ++} ++ ++function headerIdentity(target: string): string { ++ if ((target.startsWith('<') && target.endsWith('>')) || ++ (target.startsWith('"') && target.endsWith('"'))) return target.slice(1, -1); ++ return target; ++} ++ ++describe('C literal include provenance through parsing and resolution', () => { ++ it('retains system and quoted headers, resolving only local includes', () => { ++ const imports = parseIncludes([ ++ '#include ', ++ '#include "api.h"', ++ '#include "../include/common.h"', ++ ].join('\n') + '\n'); ++ expect(imports).toHaveLength(3); ++ expect(imports.every((item) => item !== null)).toBe(true); ++ expect(imports.map((item) => headerIdentity(item!.targetRaw))) ++ .toEqual(['sys/net.h', 'api.h', '../include/common.h']); ++ const files = new Set(['src/main.c', 'include/sys/net.h', 'src/api.h', 'include/common.h']); ++ const targets = imports.map((parsedImport) => { ++ const resolved = cScopeResolver.resolveImportTarget( ++ parsedImport!.targetRaw, 'src/main.c', files, undefined, ++ { parsedFiles: [], parsedImport: parsedImport! }, ++ ); ++ return resolved === null ? [] : typeof resolved === 'string' ? [resolved] : resolved; ++ }); ++ expect(targets).toEqual([[], ['src/api.h'], ['include/common.h']]); ++ }); ++ ++ it('does not treat a computed macro as a literal header path', () => { ++ expect(parseIncludes('#include HEADER_MACRO\n')).toEqual([null]); ++ }); ++}); diff --git a/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-flag.patch b/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-flag.patch new file mode 100644 index 000000000..726aebe18 --- /dev/null +++ b/eval/workflow_bench/oracles/reference/inv-feature-c-include-provenance-flag.patch @@ -0,0 +1,155 @@ +--- a/gitnexus/src/core/ingestion/languages/c/interpret.ts ++++ b/gitnexus/src/core/ingestion/languages/c/interpret.ts +@@ -8,10 +8,13 @@ + const source = captures['@import.source']?.text; + if (source === undefined) return null; + +- // System headers (e.g. ) are not resolved to local files +- if (captures['@import.system'] !== undefined) return null; +- +- return { kind: 'wildcard', targetRaw: source }; ++ // Keep provenance locally without changing immutable shared declarations. ++ const parsedImport: ParsedImport & { isSystem: boolean } = { ++ kind: 'wildcard', ++ targetRaw: source, ++ isSystem: captures['@import.system'] !== undefined, ++ }; ++ return parsedImport; + } + + /** +--- a/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts ++++ b/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts +@@ -81,7 +81,11 @@ + // this process. Runs BEFORE `populateOwners`. + applyCaptureSideChannel: applyCStaticLinkageSideChannel, + +- resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => { ++ resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig, context) => { ++ const parsedImport = context?.parsedImport; ++ if (parsedImport?.kind === 'wildcard' && 'isSystem' in parsedImport && parsedImport.isSystem === true) { ++ return null; ++ } + // Augment allFilePaths with .h files discovered via loadResolutionConfig + // since the phase only passes .c files to the C resolver but #include + // targets .h files classified as C++ in language detection. +--- a/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts ++++ b/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts +@@ -38,6 +38,10 @@ + import { describe, expect, it } from 'vitest'; + import type { ParsedFile, ParsedImport, ScopeId, SymbolDefinition } from 'gitnexus-shared'; + import { SupportedLanguages } from 'gitnexus-shared'; ++import { getCParser } from '../../../src/core/ingestion/languages/c/query.js'; ++import { splitCInclude } from '../../../src/core/ingestion/languages/c/import-decomposer.js'; ++import { interpretCImport } from '../../../src/core/ingestion/languages/c/interpret.js'; ++import type { SyntaxNode } from '../../../src/core/ingestion/utils/ast-helpers.js'; + import { SCOPE_RESOLVERS } from '../../../src/core/ingestion/scope-resolution/pipeline/registry.js'; + import type { ComposerConfig } from '../../../src/core/ingestion/language-config.js'; + import { +@@ -143,6 +147,16 @@ + }; + } + ++function parseCInclude(spelling: string): ParsedImport { ++ const tree = getCParser().parse(`#include ${spelling}\n`); ++ const node = tree.rootNode.namedChildren.find((child) => child.type === 'preproc_include'); ++ if (!node) throw new Error('C conformance fixture did not parse'); ++ const captures = splitCInclude(node as SyntaxNode); ++ const parsedImport = captures === null ? null : interpretCImport(captures); ++ if (parsedImport === null) throw new Error('C literal include was not retained'); ++ return parsedImport; ++} ++ + const PHP_FUNCTION_IMPORT = (targetRaw: string): ParsedImport => ({ + kind: 'named', + localName: 'imported', +@@ -345,9 +359,10 @@ + files: ['src/stdio.h', 'include/util.h', 'src/main.c'], + fromFile: 'src/main.c', + resolutionConfig: undefined, +- external: 'stdio.h', ++ external: '', + decoy: 'src/stdio.h', +- reachesDecoy: 'util.h', ++ reachesDecoy: '"stdio.h"', ++ parsedImport: parseCInclude, + }, + ], + [ +@@ -418,7 +433,6 @@ + const KNOWN_GAPS: ReadonlyMap = new Map([ + [SupportedLanguages.Dart, '`package:http/http.dart` -> `lib/http.dart`'], + [SupportedLanguages.Swift, '`Foundation` -> `Sources/Foundation/Thing.swift`'], +- [SupportedLanguages.C, '`stdio.h` -> `src/stdio.h`'], + [SupportedLanguages.CPlusPlus, '`cstdio.h` -> `src/cstdio.h`'], + ]); + +@@ -463,14 +477,15 @@ + : testCase.files.map( + testCase.parsedFile ?? ((filePath) => ({ filePath }) as unknown as ParsedFile), + ); ++ const parsedImport = testCase.parsedImport?.(targetRaw); + return resolver.resolveImportTarget( +- targetRaw, ++ parsedImport?.targetRaw ?? targetRaw, + testCase.fromFile, + files, + testCase.resolutionConfig, + { + parsedFiles, +- parsedImport: testCase.parsedImport?.(targetRaw), ++ parsedImport, + }, + ); + } +--- /dev/null ++++ b/gitnexus/test/unit/scope-resolution/c/c-include-provenance.test.ts +@@ -0,0 +1,48 @@ ++import { describe, expect, it } from 'vitest'; ++import { getCParser } from '../../../../src/core/ingestion/languages/c/query.js'; ++import { splitCInclude } from '../../../../src/core/ingestion/languages/c/import-decomposer.js'; ++import { interpretCImport } from '../../../../src/core/ingestion/languages/c/interpret.js'; ++import { cScopeResolver } from '../../../../src/core/ingestion/languages/c/scope-resolver.js'; ++import type { SyntaxNode } from '../../../../src/core/ingestion/utils/ast-helpers.js'; ++ ++function parseIncludes(source: string) { ++ return getCParser().parse(source).rootNode.namedChildren ++ .filter((node) => node.type === 'preproc_include') ++ .map((node) => { ++ const captures = splitCInclude(node as SyntaxNode); ++ return captures === null ? null : interpretCImport(captures); ++ }); ++} ++ ++function headerIdentity(target: string): string { ++ if ((target.startsWith('<') && target.endsWith('>')) || ++ (target.startsWith('"') && target.endsWith('"'))) return target.slice(1, -1); ++ return target; ++} ++ ++describe('C literal include provenance through parsing and resolution', () => { ++ it('retains system and quoted headers, resolving only local includes', () => { ++ const imports = parseIncludes([ ++ '#include ', ++ '#include "api.h"', ++ '#include "../include/common.h"', ++ ].join('\n') + '\n'); ++ expect(imports).toHaveLength(3); ++ expect(imports.every((item) => item !== null)).toBe(true); ++ expect(imports.map((item) => headerIdentity(item!.targetRaw))) ++ .toEqual(['sys/net.h', 'api.h', '../include/common.h']); ++ const files = new Set(['src/main.c', 'include/sys/net.h', 'src/api.h', 'include/common.h']); ++ const targets = imports.map((parsedImport) => { ++ const resolved = cScopeResolver.resolveImportTarget( ++ parsedImport!.targetRaw, 'src/main.c', files, undefined, ++ { parsedFiles: [], parsedImport: parsedImport! }, ++ ); ++ return resolved === null ? [] : typeof resolved === 'string' ? [resolved] : resolved; ++ }); ++ expect(targets).toEqual([[], ['src/api.h'], ['include/common.h']]); ++ }); ++ ++ it('does not treat a computed macro as a literal header path', () => { ++ expect(parseIncludes('#include HEADER_MACRO\n')).toEqual([null]); ++ }); ++}); diff --git a/eval/workflow_bench/tasks.scenarios.yaml b/eval/workflow_bench/tasks.scenarios.yaml index 08ec59699..98f532aed 100644 --- a/eval/workflow_bench/tasks.scenarios.yaml +++ b/eval/workflow_bench/tasks.scenarios.yaml @@ -3,7 +3,7 @@ # Task fields: # id: short slug used in reports # class: task class for cost/quality routing analysis. The suite spans: -# trivial → investigation-bug → investigation-feature → cross-module +# trivial → investigation-feature → cross-module # repo: path to a git repo that is GitNexus-indexed # ref: git ref benchmarked (fresh detached worktree per arm per run) # setup: optional shell command run in the fresh worktree first @@ -29,7 +29,7 @@ # prebuilt .gitnexus directory is never copied. Three declared dependency trees # are mounted read-only; missing assets fail before model execution. # Every task starts at the exact v1.6.12 commit, where all four tasks remain -# unsolved. In particular, moving main already contains the C include fix. +# unsolved. The C provenance feature replaces the invalid historical system-include task. tasks: # Overhead floor — measured 2026-07-11 (see README calibration): the @@ -67,30 +67,39 @@ tasks: - source: trivial-status-json-alias.oracle.test.ts target: trivial-status-json-alias.oracle.test.ts - # Investigation-heavy bug (#2965): requires following the C import marker - # through the shared scope-resolution pipeline without breaking local - # quoted-header resolution. - - id: inv-bug-c-system-include - class: investigation-bug + # Investigation-heavy feature: retain literal include provenance through + # parsing and resolution without binding system headers to repository decoys. + - id: inv-feature-c-include-provenance + class: investigation-feature repo: ~/GitNexus ref: c4ecf398de637cf779ab2238e69958cfd5d601ce sandbox_dependencies: *gitnexus_sandbox_dependencies prompt: > - Fix the C scope resolver so a system include such as never - resolves to an unrelated same-named header inside the repository, while - quoted/local includes still resolve. Remove C from the conformance - suite's KNOWN_GAPS and cover the negative and positive cases there. - verify: cd gitnexus && npx tsc --noEmit && npx vitest run test/unit/scope-resolution/external-import-conformance.test.ts + Retain literal C #include records after parsing and interpretation, including + system includes such as . Preserve each header's identity and enough + provenance to distinguish system from quoted includes. With no configured + system include roots, system includes must never resolve to unrelated + same-named repository headers; quoted sibling and relative includes must + still resolve. Computed macro includes remain unsupported and produce no + parsed import. Cover the real parser, include decomposition, interpretation + and resolver in test/unit/scope-resolution/c/c-include-provenance.test.ts. + Correct the C conformance fixture to distinguish system and quoted forms of + the same header, and remove C from KNOWN_GAPS. Choose the representation; + shared dependency declarations are immutable. + verify: >- + cd gitnexus && npx tsc --noEmit && npx vitest run + test/unit/scope-resolution/c/c-include-provenance.test.ts + test/unit/scope-resolution/external-import-conformance.test.ts oracle: command: >- cd gitnexus && ./node_modules/.bin/vitest run --config "$GITNEXUS_BENCH_ORACLE_ROOT/vitest.config.mts" - "$GITNEXUS_BENCH_ORACLE_ROOT/inv-bug-c-system-include.oracle.test.ts" + "$GITNEXUS_BENCH_ORACLE_ROOT/inv-feature-c-include-provenance.oracle.test.ts" files: - source: vitest.config.mts target: vitest.config.mts - - source: inv-bug-c-system-include.oracle.test.ts - target: inv-bug-c-system-include.oracle.test.ts + - source: inv-feature-c-include-provenance.oracle.test.ts + target: inv-feature-c-include-provenance.oracle.test.ts # Investigation-heavy feature: touches tool schema, backend filtering, and # pagination totals — three seams that must stay consistent.