GitNexus/eval/workflow_bench/tasks.scenarios.yaml

# Scenario suite for the gitnexus workflow benchmark — the ground-base matrix.
#
# Task fields:
#   id:      short slug used in reports
#   class:   task class for cost/quality routing analysis. The suite spans:
#            trivial → investigation-bug → investigation-feature → cross-module
#   repo:    path to a git repo that is GitNexus-indexed
#   ref:     git ref benchmarked (fresh detached worktree per arm per run)
#   setup:   optional shell command run in the fresh worktree first
#            (task-specific preparation; dependencies are immutable mounts)
#   prompt:  the engineering task, phrased once, given verbatim to every arm.
#            Prescribe the test file path — that keeps `verify` deterministic.
#   verify:  model-visible authored-test command (recorded as a separate signal)
#   oracle:  harness-owned hidden files + command; oracle success is also
#            required for resolved and files are staged only after the session
#   expensive: optional boolean; true scenarios require --include-expensive
#   sandbox_dependencies: repository-local dependencies mounted read-only
#
# Arms (runner --arms): workflow (plan→work), workflow_direct (work skill,
# no plan), baseline (no skills, MCP allowed), baseline_nomcp (no skills, no
# graph tools). candidate_workflow and candidate_workflow_direct apply a
# skill-only --candidate-overlay and must be paired with their incumbent arm.
# Comparing workflow vs workflow_direct vs baseline locates the task-complexity
# boundary where each mode pays for itself — that boundary is the routing rule
# lfg's gate and work's direct-mode triage encode.
#
# GitNexus scenarios build one graph from a history-pruned, parentless clone and
# cache only that sanitized graph for every arm. The target repository's
# prebuilt .gitnexus directory is never copied. Three declared dependency trees
# are mounted read-only; missing assets fail before model execution.

tasks:
  # Overhead floor — measured 2026-07-11 (see README calibration): the
  # workflow is EXPECTED to lose here. Kept in the suite so regressions in
  # the overhead floor stay visible.
  - id: trivial-status-json-alias
    class: trivial
    repo: ~/GitNexus
    ref: main
    sandbox_dependencies: &gitnexus_sandbox_dependencies
      - source: node_modules
        target: node_modules
      - source: gitnexus/node_modules
        target: gitnexus/node_modules
      - source: gitnexus-shared/node_modules
        target: gitnexus-shared/node_modules
    prompt: >
      Add -j as a short alias for --json on the gitnexus status command
      (gitnexus/src/cli/index.ts), and cover the alias with a unit test in
      gitnexus/test/unit/status-json-alias.test.ts.
    verify: cd gitnexus && npx tsc --noEmit && npx vitest run test/unit/status-json-alias.test.ts
    oracle:
      command: >-
        cd gitnexus && ./node_modules/.bin/vitest run
        --config "$GITNEXUS_BENCH_ORACLE_ROOT/vitest.config.mts"
        "$GITNEXUS_BENCH_ORACLE_ROOT/trivial-status-json-alias.oracle.test.ts"
      files:
        - source: vitest.config.mts
          target: vitest.config.mts
        - source: trivial-status-json-alias.oracle.test.ts
          target: trivial-status-json-alias.oracle.test.ts

  # Investigation-heavy bug (#2965): requires following the C import marker
  # through the shared scope-resolution pipeline without breaking local
  # quoted-header resolution.
  - id: inv-bug-c-system-include
    class: investigation-bug
    repo: ~/GitNexus
    ref: main
    sandbox_dependencies: *gitnexus_sandbox_dependencies
    prompt: >
      Fix the C scope resolver so a system include such as <stdio.h> never
      resolves to an unrelated same-named header inside the repository, while
      quoted/local includes still resolve. Remove C from the conformance
      suite's KNOWN_GAPS and cover the negative and positive cases there.
    verify: cd gitnexus && npx tsc --noEmit && npx vitest run test/unit/scope-resolution/external-import-conformance.test.ts
    oracle:
      command: >-
        cd gitnexus && ./node_modules/.bin/vitest run
        --config "$GITNEXUS_BENCH_ORACLE_ROOT/vitest.config.mts"
        "$GITNEXUS_BENCH_ORACLE_ROOT/inv-bug-c-system-include.oracle.test.ts"
      files:
        - source: vitest.config.mts
          target: vitest.config.mts
        - source: inv-bug-c-system-include.oracle.test.ts
          target: inv-bug-c-system-include.oracle.test.ts

  # Investigation-heavy feature: touches tool schema, backend filtering, and
  # pagination totals — three seams that must stay consistent.
  - id: inv-feature-list-repos-filter
    class: investigation-feature
    repo: ~/GitNexus
    ref: main
    sandbox_dependencies: *gitnexus_sandbox_dependencies
    prompt: >
      Add an optional "name_contains" filter parameter to the list_repos MCP
      tool: case-insensitive substring match on the repo name, with the
      pagination object (total/hasMore/nextOffset) reflecting the FILTERED
      set. Cover with unit tests in
      gitnexus/test/unit/list-repos-name-filter.test.ts.
    verify: cd gitnexus && npx tsc --noEmit && npx vitest run test/unit/list-repos-name-filter.test.ts
    oracle:
      command: >-
        cd gitnexus && ./node_modules/.bin/vitest run
        --config "$GITNEXUS_BENCH_ORACLE_ROOT/vitest.config.mts"
        "$GITNEXUS_BENCH_ORACLE_ROOT/inv-feature-list-repos-filter.oracle.test.ts"
      files:
        - source: vitest.config.mts
          target: vitest.config.mts
        - source: inv-feature-list-repos-filter.oracle.test.ts
          target: inv-feature-list-repos-filter.oracle.test.ts

  # Cross-module: worker-pool + pipeline seams, concurrency-sensitive.
  # The most expensive scenario — run deliberately, not by default.
  - id: cross-module-parse-retry
    class: cross-module
    expensive: true
    repo: ~/GitNexus
    ref: main
    sandbox_dependencies: *gitnexus_sandbox_dependencies
    prompt: >
      Add bounded retry with backoff to the ingestion pipeline so a transient
      parse-worker failure on a file is retried up to 2 times before the file
      is marked failed, without retrying deterministic parse errors. Cover
      the retry/no-retry decision with unit tests in
      gitnexus/test/unit/parse-retry.test.ts.
    verify: cd gitnexus && npx tsc --noEmit && npx vitest run test/unit/parse-retry.test.ts
    oracle:
      command: >-
        cd gitnexus && ./node_modules/.bin/vitest run
        --config "$GITNEXUS_BENCH_ORACLE_ROOT/vitest.config.mts"
        "$GITNEXUS_BENCH_ORACLE_ROOT/cross-module-parse-retry.oracle.test.ts"
      files:
        - source: vitest.config.mts
          target: vitest.config.mts
        - source: cross-module-parse-retry.oracle.test.ts
          target: cross-module-parse-retry.oracle.test.ts