From 13d06d41c4b414aad3794da2dab569e98469429e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Wed, 28 Oct 2026 11:58:29 +0000 Subject: [PATCH] feat(ci): rebalance test shards from recorded timings (#3748) * feat(ci): rebalance test shards from recorded timings * fix(ci): preserve artifact retries for shard timings Find shard-plan generation by its stable step ID and clarify the native runner cohort count (#3748). Note: pre-existing reference-oracle parser type failures in two test_oracle_assets cases remain outside this change. * fix(ci): keep retry timing snapshots immutable Require the existing snapshot on reruns so a failed download cannot refresh history and change shard assignments. Cover the workflow contract and document recovery when the artifact is unavailable. * docs(ci): document shard timing profile selection * fix(ci): balance cold shard estimates consistently --------- Co-authored-by: Gergo Magyar --- .github/workflows/ci-tests.yml | 142 +++++++++- TESTING.md | 30 +++ docs/cli-reference.md | 1 + eval/tests/test_workflow_bench.py | 20 +- gitnexus/package-lock.json | 16 +- gitnexus/package.json | 3 +- gitnexus/scripts/cross-platform-shard.ts | 211 --------------- gitnexus/scripts/cross-platform-tests.ts | 7 +- gitnexus/scripts/run-cross-platform.ts | 51 ++-- gitnexus/scripts/shard-timings-reporter.ts | 57 ++++ gitnexus/scripts/shard-timings.ts | 16 ++ gitnexus/test/helpers/perf-sequencer.ts | 59 +++-- .../test/unit/cross-platform-shard.test.ts | 250 ------------------ gitnexus/test/unit/perf-sequencer.test.ts | 113 ++++++++ .../test/unit/shard-timings-reporter.test.ts | 87 ++++++ .../test/unit/shard-timings-roundtrip.test.ts | 96 +++++++ gitnexus/vitest.config.ts | 8 +- 17 files changed, 629 insertions(+), 538 deletions(-) delete mode 100644 gitnexus/scripts/cross-platform-shard.ts create mode 100644 gitnexus/scripts/shard-timings-reporter.ts create mode 100644 gitnexus/scripts/shard-timings.ts delete mode 100644 gitnexus/test/unit/cross-platform-shard.test.ts create mode 100644 gitnexus/test/unit/perf-sequencer.test.ts create mode 100644 gitnexus/test/unit/shard-timings-reporter.test.ts create mode 100644 gitnexus/test/unit/shard-timings-roundtrip.test.ts diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index d246ab31c..f8ec70bd5 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -139,6 +139,11 @@ jobs: - name: Ensure FTS + VECTOR extensions installed run: npx tsx scripts/ensure-fts.ts working-directory: gitnexus + - name: Download frozen timing snapshot + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: shard-timing-input + path: gitnexus/.vitest/timings - name: Run sharded tests with coverage (blob) # Shard via env var (not `${{ }}` inlined into the shell) so it isn't a # template-injection sink; shell: bash makes "$SHARD" expand uniformly. @@ -195,11 +200,17 @@ jobs: pattern: coverage-blob-* path: gitnexus/.vitest/blob merge-multiple: true + - name: Download frozen timing snapshot + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: shard-timing-input + path: gitnexus/.vitest/timings - name: Merge coverage + enforce thresholds run: >- npx vitest --mergeReports=.vitest/blob --reporter=default --reporter=./scripts/execution-reporter.ts + --reporter=./scripts/shard-timings-reporter.ts --outputFile=test-results.json --coverage --coverage.reporter=json-summary @@ -207,6 +218,12 @@ jobs: --coverage.reporter=text --coverage.thresholds.autoUpdate=false working-directory: gitnexus + - name: Upload learned coverage timings + uses: ./.github/actions/upload-test-artifact + with: + name: timings-coverage + path: gitnexus/.vitest/timings/coverage.json + include-hidden-files: true # gitnexus-shared already built by setup-gitnexus above - name: Install gitnexus-web dependencies run: npm ci @@ -236,7 +253,8 @@ jobs: # Single source of truth for the platform-sensitive shard count. TOTAL below # generates both the shard index list (the matrix) and the /N denominator (job # name + --shard arg), so they can't drift — bump the shard count by editing - # TOTAL alone. Checkout-free (ubuntu ships jq), so no credential surface. + # TOTAL alone. Restore timing history once: independent restores in each shard + # could select different histories while another run publishes an update. shard-plan: runs-on: ubuntu-latest outputs: @@ -245,6 +263,41 @@ jobs: cov_shards: ${{ steps.gen.outputs.cov_shards }} cov_total: ${{ steps.gen.outputs.cov_total }} steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Reuse this run's timing input on a retry + id: previous-input + if: github.run_attempt > 1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: shard-timing-input + path: .vitest/timings + - name: Restore last trusted timing history + if: github.run_attempt == 1 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 + with: + path: .vitest/timings + key: test-timings-v1-node22-vitest5-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: test-timings-v1-node22-vitest5- + - name: Prepare timing snapshot + if: github.run_attempt == 1 + shell: bash + run: | + mkdir -p .vitest/timings + for profile in coverage windows-latest macos-latest; do + file=".vitest/timings/$profile.json" + if [ ! -f "$file" ]; then printf '{}\n' > "$file"; fi + done + sha256sum .vitest/timings/*.json >> "$GITHUB_STEP_SUMMARY" + - name: Freeze timing input for every shard and retry + if: github.run_attempt == 1 + uses: ./.github/actions/upload-test-artifact + with: + name: shard-timing-input + path: .vitest/timings/*.json + include-hidden-files: true + retention-days: 30 - id: gen run: | # Keep the recorded native workload below 17 minutes, leaving @@ -275,17 +328,8 @@ jobs: matrix: # Ubuntu already covered by the coverage job above os: [windows-latest, macos-latest] - # Shard the fixed file list across N runners per OS (N = TOTAL in the - # shard-plan job). The suite is dominated by ~50 CLI/worker process - # spawns and Windows is ~5x slower than macOS at those, so the unsharded - # run crept past the 15-min watchdog in run-cross-platform.ts. vitest - # shards by file COUNT, not runtime, so the heaviest spawn suites can - # cluster on one shard. The busiest Windows shard has grown to the old - # 15-minute watchdog (14m57s on the v1.6.10-rc.19 green run, one - # observed timeout since — #2449), so the job env below raises the - # per-shard watchdog to 20 minutes, still bounded by timeout-minutes. - # Shard indices come from the shard-plan job (single source of truth): - # its TOTAL drives this list and the /N in the job name + --shard arg. + # Timing-based assignment uses the same frozen input for all N runners. + # Keep the existing per-shard watchdog and matrix count. shard: ${{ fromJSON(needs.shard-plan.outputs.shards) }} runs-on: ${{ matrix.os }} timeout-minutes: 25 @@ -306,6 +350,7 @@ jobs: # failure, never a silent skip (same contract as GITNEXUS_REQUIRE_FTS). GITNEXUS_REQUIRE_VECTOR: '1' GITNEXUS_E2E_CLI: dist + GITNEXUS_SHARD_PROFILE: ${{ matrix.os }} # #2449: hosted Windows runners intermittently push the busiest shard past # the default 15-minute watchdog. 20 minutes restores real headroom while # the 25-minute job timeout above still bounds a genuine hang. @@ -336,6 +381,11 @@ jobs: - name: Ensure FTS + VECTOR extensions installed run: npx tsx scripts/ensure-fts.ts working-directory: gitnexus + - name: Download frozen timing snapshot + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: shard-timing-input + path: gitnexus/.vitest/timings - name: Run platform-sensitive tests # Pass the shard through an env var (not `${{ }}` inlined into the shell) # so it isn't a template-injection sink (zizmor). shell: bash makes the @@ -354,6 +404,74 @@ jobs: name: execution-${{ matrix.os }}-${{ matrix.shard }} path: gitnexus/${{ matrix.os }}-${{ matrix.shard }}.json retention-days: 5 + - name: Upload platform timing blob + if: always() + uses: ./.github/actions/upload-test-artifact + with: + name: platform-blob-${{ matrix.os }}-${{ matrix.shard }} + path: gitnexus/.vitest/blob/ + include-hidden-files: true + + # Merge on the originating OS so absolute paths in Vitest blobs keep their + # meaning. No tests are re-executed; the reporter validates the native subset. + native-timings: + name: ${{ matrix.os }} / shard timings + needs: cross-platform + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + env: + GITNEXUS_SHARD_PROFILE: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/setup-gitnexus + with: + lifecycle-scripts: 'false' + - name: Download frozen timing snapshot + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: shard-timing-input + path: gitnexus/.vitest/timings + - name: Download native blobs + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + pattern: platform-blob-${{ matrix.os }}-* + path: gitnexus/.vitest/blob + merge-multiple: true + - name: Check native partition and learn timings + run: npx vitest --mergeReports=.vitest/blob --reporter=default --reporter=./scripts/shard-timings-reporter.ts + working-directory: gitnexus + - name: Upload learned native timings + uses: ./.github/actions/upload-test-artifact + with: + name: timings-${{ matrix.os }} + path: gitnexus/.vitest/timings/${{ matrix.os }}.json + include-hidden-files: true + + # Only successful trusted main runs publish history. PRs consume it through + # the planner; they never replace the baseline used by other PRs. + save-shard-timings: + needs: [coverage-merge, native-timings, test-completeness] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Download learned timings + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + pattern: timings-* + path: .vitest/timings + merge-multiple: true + - name: Save timing history for later runs + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 + with: + path: .vitest/timings + key: test-timings-v1-node22-vitest5-${{ github.run_id }}-${{ github.run_attempt }} # Branch protection still requires these six names from the three-shard # matrix. Keep them as aggregate gates as the real matrix grows: every native diff --git a/TESTING.md b/TESTING.md index cbad267a9..9b2a130e4 100644 --- a/TESTING.md +++ b/TESTING.md @@ -104,6 +104,36 @@ The subset is defined in `gitnexus/scripts/cross-platform-tests.ts` and includes When adding a platform-sensitive test, add it to the appropriate section in `scripts/cross-platform-tests.ts`. +### Automatic shard balancing + +CI uses `vitest-shard-balancer` to distribute test files by measured duration. +Timing history lives in GitHub Actions storage; adding a test does not require +editing a weight table. Windows, macOS, and Ubuntu coverage each have their own +profile because native startup and coverage costs differ. + +The `shard-plan` job restores history once and uploads `shard-timing-input`. +Every shard and report merge downloads that same snapshot. Retries reuse it, +so a newly published history cannot change the partition halfway through a run. +If the snapshot cannot be restored, the retry fails. If it was never uploaded, +deleted, or expired, trigger a fresh workflow run with a new commit; rerunning +the old run must not replace its timing input. +With no usable history, the sequencer spreads sequential/native work using +generic estimates; new files in an established profile use the package's median. + +The merge reporter includes setup, collection, and test execution costs and +smooths new measurements with previous ones. It rejects missing, duplicate, or +unexpected modules and unsuccessful runs, and retains prior costs for skipped +modules. Native blobs are merged on their originating OS. Only main-branch push +runs that pass the execution audit publish the updated history to the Actions +cache. Cache expiry affects balance, not test discovery or completeness. + +For a local reproduction, download the run's `shard-timing-input` artifact into +`gitnexus/.vitest/timings/` and set `GITNEXUS_SHARD_PROFILE` to `windows-latest`, +`macos-latest`, or `coverage` (the default). Then use the same `--shard=i/n` +argument with `npm run test:cross-platform -- --shard=i/n` or `npm test -- --shard=i/n`. +Keep `.vitest/` untracked. Bump the workflow's timing-cache schema key when a +runner/measurement change makes the stored profiles incompatible. + ### Confirming no tests are orphaned Every test file matches one of the three vitest projects. To verify: diff --git a/docs/cli-reference.md b/docs/cli-reference.md index dd4885cd5..a387b7241 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -223,6 +223,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | Variable | Default | Effect | Tune when… | | ----------------------------------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `GITNEXUS_TEST_REPORT` | unset | Write a JSON execution receipt from `scripts/run-cross-platform.ts`; accepts a basename ending in `.json`. | CI platform shards use distinct filenames so the completeness check can require every report. | +| `GITNEXUS_SHARD_PROFILE` | `coverage` | Selects the timing profile for Vitest shard balancing and merged timing reports: `coverage`, `windows-latest`, or `macos-latest`. | CI selects its native OS; set the matching profile when reproducing a shard locally. See [Automatic shard balancing](../TESTING.md#automatic-shard-balancing). | | `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers `. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. | | `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | | `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | diff --git a/eval/tests/test_workflow_bench.py b/eval/tests/test_workflow_bench.py index 4ebd8bc52..98248b50d 100644 --- a/eval/tests/test_workflow_bench.py +++ b/eval/tests/test_workflow_bench.py @@ -321,7 +321,7 @@ def test_platform_ci_preserves_required_check_names_with_complete_execution_gate for platform in ("windows-latest", "macos-latest") for check in (1, 2, 3) } - plan = jobs["shard-plan"]["steps"][0]["run"] + plan = next(step["run"] for step in jobs["shard-plan"]["steps"] if step.get("id") == "gen") total = int(re.search(r"^TOTAL=(\d+)\b", plan, re.MULTILINE)[1]) native_names = { "tests / " @@ -408,6 +408,24 @@ def test_test_artifact_retries_cannot_hide_missing_evidence(ci_test_jobs): ) +def test_shard_plan_retry_cannot_replace_frozen_timing_input(ci_test_jobs): + steps = ci_test_jobs["shard-plan"]["steps"] + restore = next(step for step in steps if step.get("id") == "previous-input") + assert restore["if"] == "github.run_attempt > 1" + assert not restore.get("continue-on-error", False) + assert restore["with"]["name"] == "shard-timing-input" + + # A failed restore must never fall through to newer history, including + # upload retries that may overwrite an artifact left by a previous attempt. + for name in ( + "Restore last trusted timing history", + "Prepare timing snapshot", + "Freeze timing input for every shard and retry", + ): + step = next(step for step in steps if step.get("name") == name) + assert step["if"] == "github.run_attempt == 1", name + + @pytest.mark.parametrize("native_result", ["success", "failure", "cancelled", "skipped", ""]) @pytest.mark.parametrize("execution_result", ["success", "failure", "cancelled", "skipped", ""]) @pytest.mark.parametrize("scope_bench_result", ["success", "failure", "cancelled", "skipped", ""]) diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 10de4ce75..47d81beb2 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -86,7 +86,8 @@ "jsdom": "30.1.1", "tsx": "^4.0.0", "typescript": "^7.0.2", - "vitest": "^5.0.0" + "vitest": "^5.0.0", + "vitest-shard-balancer": "0.1.0" }, "engines": { "node": "^22.18.0 || >=24.11.0" @@ -7349,6 +7350,19 @@ } } }, + "node_modules/vitest-shard-balancer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/vitest-shard-balancer/-/vitest-shard-balancer-0.1.0.tgz", + "integrity": "sha512-pUMR4vaEKKMnfwvCfLKSbYhbb3mPxKzii69toipqmupvO48T4jl2jx57qK/nnGknImkjO9i+fTfWQv/X4096RA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^22.12.0 || >=24.0.0" + }, + "peerDependencies": { + "vitest": ">=3.2.0" + } + }, "node_modules/w3c-xmlserializer": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-6.0.0.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 36b4c8d5b..c81ba4bd9 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -150,7 +150,8 @@ "jsdom": "30.1.1", "tsx": "^4.0.0", "typescript": "^7.0.2", - "vitest": "^5.0.0" + "vitest": "^5.0.0", + "vitest-shard-balancer": "0.1.0" }, "overrides": { "adm-zip": ">=0.6.0", diff --git a/gitnexus/scripts/cross-platform-shard.ts b/gitnexus/scripts/cross-platform-shard.ts deleted file mode 100644 index 2f1d26676..000000000 --- a/gitnexus/scripts/cross-platform-shard.ts +++ /dev/null @@ -1,211 +0,0 @@ -/** - * Weight-aware partitioning for the cross-platform test matrix. - * - * WHY THIS EXISTS. `run-cross-platform.ts` used to hand vitest the whole file - * list plus `--shard=i/n`, and vitest partitions by file COUNT. Runtime on this - * suite is wildly uneven — measured on the Windows runner, `cli-e2e` is 621 s - * and `worker-pool` 221 s, while most files are under a second — so a - * count-split routinely put several of the heaviest suites on one shard. That - * is #2449, and this file's sibling header has documented the symptom ("the - * heaviest spawn suites can cluster on one shard") since the watchdog was first - * raised from 15 to 20 minutes. - * - * It went from a latent hazard to a red matrix when three CHEAP files (the - * `dist/` module-load closure guards: 448 ms, 53 ms, sub-second) were added to - * `SPAWN_CLI`. They cost nothing to run, but a count-split re-partitions on - * every insertion, and the reshuffle happened to land `cli-e2e` + `cli-limit-e2e` - * + `analyze-heap-oom-e2e` together on shard 1/3 — 32 files against 26 and 29 — - * which blew the 20-minute budget with four files still queued. Nothing about - * the added files caused it; they were simply the perturbation. - * - * So the split is done HERE, by weight, and only the chosen shard's files are - * handed to vitest. Two properties follow, and both are pinned in - * `test/unit/cross-platform-shard.test.ts`: - * - * - the heaviest suites are spread across shards by construction, so the - * busiest shard tracks the ideal rather than the luck of the sort order; - * - adding or removing a CHEAP file cannot move a heavy one, so registering a - * new platform-sensitive test is no longer a CI-stability gamble. That is the - * property whose absence caused this. - */ - -/** - * Measured wall-clock on the WINDOWS runner (the slowest platform, so it is the - * one that decides the budget), in seconds, from the last fully-green matrix run - * plus the timed files of the run that failed. - * - * Only files heavy enough to matter are listed; everything else is carried by - * {@link PER_FILE_OVERHEAD_SEC} alone. These are load-balancing hints, NOT - * assertions — no test measures elapsed time against this table. Missing or - * stale heavy entries can still overload a shard; refresh them from failed - * CI logs and replay that profile in the partition tests. Deliberately not - * auto-generated: a committed table is reviewable and works offline, and the - * alternative (timing files at CI runtime to decide the split) would make the - * partition depend on the very machine load it is trying to protect against. - */ -export const WINDOWS_WEIGHTS_SEC: Readonly> = { - // Provisional Windows estimates, not Windows measurements: these native - // inline-handler proofs took 7.25/6.86/6.52s on Node24 Linux. Apply the - // runner's documented ~5x slowdown and round up; keep the watchdog unchanged. - 'test/integration/cfg/interproc-taint.test.ts': 40, - 'test/integration/function-local-identity.test.ts': 35, - 'test/integration/taint-explain.test.ts': 35, - // Conservative estimates from Windows run 38144297115: persisted - // rename cases include native indexing and take up to ~25s each. Keep - // each language together while charging its complete fixture inventory. - // Refresh underestimates from run 38697469283; Zig's higher measurement - // comes from 38693066214. Stale weights packed too much native work ahead - // of cli-e2e and exhausted its shard's 20-minute watchdog. - 'test/integration/rename-provider-c.test.ts': 202, - 'test/integration/rename-provider-cobol.test.ts': 75, - 'test/integration/rename-provider-cpp.test.ts': 250, - 'test/integration/rename-provider-csharp.test.ts': 266, - 'test/integration/rename-provider-dart.test.ts': 425, - 'test/integration/rename-provider-go.test.ts': 221, - 'test/integration/rename-provider-java.test.ts': 300, - 'test/integration/rename-provider-javascript.test.ts': 225, - 'test/integration/rename-provider-kotlin.test.ts': 300, - 'test/integration/rename-provider-objective-c.test.ts': 386, - 'test/integration/rename-provider-php.test.ts': 100, - 'test/integration/rename-provider-python.test.ts': 375, - 'test/integration/rename-provider-ruby.test.ts': 100, - 'test/integration/rename-provider-rust.test.ts': 75, - 'test/integration/rename-provider-swift.test.ts': 300, - 'test/integration/rename-provider-typescript.test.ts': 625, - 'test/integration/rename-provider-vue.test.ts': 200, - 'test/integration/rename-provider-zig.test.ts': 183, - // Provisional: seven real-indexing child cases, using the existing ~25s - // Windows persisted-fixture profile above. Replace with observed CI timing. - 'test/integration/rename-preview-plan.test.ts': 175, - 'test/integration/mcp-index-generation.test.ts': 201, - // Windows run 39934695864 timed out after undercharged native suites; - // 40172490240 supplies the query-symbol and dirty-recovery measurements. - 'test/integration/incremental-method-integrity.test.ts': 145, - 'test/integration/fts-repair-warm-session.test.ts': 62, - 'test/integration/query-symbol-integrity.test.ts': 34, - 'test/unit/incremental-dirty-recovery.test.ts': 128, - // Re-measured after the analyze --watch e2e landed in #3072. The previous - // 361 s entry undercharged this suite and left shard 1 close to the watchdog. - 'test/integration/cli-e2e.test.ts': 621, - 'test/integration/worker-pool.test.ts': 254, - 'test/unit/incremental-vector-extension-ordering.test.ts': 87, - // Measured on Windows in run 34014266125 (#3190, 2026-09-06). These DB - // suites landed together on shard 3: the old 180s estimate and missing - // entries made a ~27-minute recorded load look like an ~12-minute shard. - // Upstream speedups may reduce these figures; retaining conservative weights - // keeps the expensive suites distributed without changing the watchdog. - 'test/unit/incremental-index-extension-dml-gate.test.ts': 414, - // Re-measured on windows-latest run 34815870795 after vendored-first FTS - // rewrote the HOME-layout e2e (373s) and skills-e2e grew to 542s. The old - // 146s/444s entries packed both onto shard 2/3 and blew the 20-minute - // watchdog with one file still queued. - 'test/integration/skills-e2e.test.ts': 550, - 'test/integration/fts-extension-e2e.test.ts': 380, - 'test/integration/skip-fts.test.ts': 110, - // Provisional: twice the local native-suite duration, pending Windows CI. - // These scheduling hints do not change test timeouts or pass/fail gates. - 'test/integration/embedding-identity-integrity.test.ts': 180, - 'test/integration/shared-store-embedding-reuse.test.ts': 140, - // Provisional: twice the 88.42s local native run, pending Windows CI timing. - 'test/integration/embedding-input-isolation.test.ts': 180, - 'test/integration/analyze-wal-checkpoint-failure.test.ts': 86, - // Windows run 39046807237 measured 308s before two cases reached recovery. - // Allow the measured ~19s recovery cost for each corrected in-place case. - 'test/unit/incremental-write-integrity.test.ts': 360, - 'test/integration/cli-limit-e2e.test.ts': 75, - 'test/unit/hooks.test.ts': 26, - 'test/integration/analyze-heap-oom-e2e.test.ts': 23, - 'test/unit/git-utils.test.ts': 18, - 'test/integration/hooks-e2e.test.ts': 15, - 'test/integration/tree-sitter-languages.test.ts': 9, - 'test/unit/repo-manager.test.ts': 9, - 'test/unit/detect-changes-worktree.test.ts': 9, - 'test/integration/antigravity-hook-e2e.test.ts': 7, - 'test/unit/index-lock.test.ts': 5, - 'test/unit/setup.test.ts': 5, - // ESTIMATE, not a measurement. This file asserts almost nothing; it READS — - // one 4893-file pass over every tracked text file, plus an 830-file pass over - // `src/`. Measured at 2.3 s and 0.3 s per pass on a virtualised and a local - // Linux filesystem respectively, so the cost is entirely per-file open - // latency, which is the term Windows inflates most (NTFS plus Defender on - // every read). Scaled from the slower Linux figure to keep the split - // conservative rather than let the 8 s PER_FILE_OVERHEAD floor under-charge - // a file that touches more paths than anything else here. Replace with a real - // figure after the first green Windows matrix run. - 'test/unit/source-control-bytes.test.ts': 15, -}; - -/** - * Fixed cost every file pays regardless of what it asserts: a pool worker start, - * module graph evaluation, and (for most of this list) a native addon load. - * - * Added to EVERY file's weight, not just unmeasured ones, and that is the point. - * Calibrated against the last green Windows matrix: its busiest shard ran 736 s - * of wall clock over ~511 s of measured file time, so roughly 8 s per file is - * unattributed setup. Without this term the balancer treats a light file as - * nearly free and, having isolated the two monsters, piles every remaining file - * onto the other shards — trading a runtime imbalance for a file-count one that - * costs just as much. With it, the split balances runtime AND count together. - */ -const PER_FILE_OVERHEAD_SEC = 8; - -/** - * Scheduling weight for `file`: its measured runtime (0 if it was fast enough - * that vitest printed no duration) plus the per-file floor above. - */ -export function weightOf(file: string): number { - return (WINDOWS_WEIGHTS_SEC[file] ?? 0) + PER_FILE_OVERHEAD_SEC; -} - -/** - * Partition `files` into `total` shards and return the 1-based `index` one. - * - * Longest-processing-time first: sort by weight descending, then repeatedly give - * the next file to the lightest shard so far. LPT is the standard greedy for - * multiprocessor scheduling and is guaranteed within 4/3 of optimal — far more - * than enough here, where the goal is only "no shard gets two monsters". - * - * Ties break on the file path so the partition is DETERMINISTIC: every shard - * computes the same split independently, on a different machine, with no - * coordination — which is what lets each runner select its own slice. - * - * Returns files in the input list's original order, not weight order, so failure - * output and reruns stay readable. - */ -export function shardFiles( - files: readonly string[], - index: number, - total: number, -): readonly string[] { - if (!Number.isInteger(total) || total < 1) { - throw new Error(`shard total must be a positive integer, got ${total}`); - } - if (!Number.isInteger(index) || index < 1 || index > total) { - throw new Error(`shard index must be in 1..${total}, got ${index}`); - } - if (total === 1) return [...files]; - - const byWeightDesc = [...files].sort((a, b) => { - const diff = weightOf(b) - weightOf(a); - return diff !== 0 ? diff : a.localeCompare(b); - }); - - const loads = Array.from({ length: total }, () => 0); - const assigned = Array.from({ length: total }, () => new Set()); - for (const file of byWeightDesc) { - let lightest = 0; - for (let i = 1; i < total; i++) { - if (loads[i]! < loads[lightest]!) lightest = i; - } - assigned[lightest]!.add(file); - loads[lightest]! += weightOf(file); - } - - const mine = assigned[index - 1]!; - return files.filter((f) => mine.has(f)); -} - -/** Total weight of a file set — the shard cost this balancer is minimising. */ -export function shardWeight(files: readonly string[]): number { - return files.reduce((sum, f) => sum + weightOf(f), 0); -} diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index febc8cf99..3b9c0c14b 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -267,6 +267,8 @@ const LBUG_NATIVE = [ // process spawning, which behaves differently across platforms (shell // quoting, path resolution, signal handling) const SPAWN_CLI = [ + // Real shard/blob/JSON roundtrip: path identities and child invocation vary by OS. + 'test/unit/shard-timings-roundtrip.test.ts', 'test/integration/cli-e2e.test.ts', 'test/integration/cli-limit-e2e.test.ts', 'test/integration/hooks-e2e.test.ts', @@ -321,9 +323,8 @@ const SPAWN_CLI = [ // earlier attempt to register them still turned the matrix red — not from // their own cost, but because vitest sharded by file COUNT, so inserting any // file re-partitioned the list and happened to cluster `cli-e2e` (621 s) with - // `cli-limit-e2e` (75 s) on one shard. The split is weight-aware now - // (`scripts/cross-platform-shard.ts`), so a cheap file can no longer move a - // heavy one. + // `cli-limit-e2e` (75 s) on one shard. PerfSequencer now uses recorded timings + // through vitest-shard-balancer, shared by all shards in each workflow run. // // #2802: MCP startup must not eagerly load the analyze-only language // provider registry or the group contract extractors. diff --git a/gitnexus/scripts/run-cross-platform.ts b/gitnexus/scripts/run-cross-platform.ts index 630fd0b93..952c8b004 100644 --- a/gitnexus/scripts/run-cross-platform.ts +++ b/gitnexus/scripts/run-cross-platform.ts @@ -15,7 +15,6 @@ import path from 'path'; import { fileURLToPath } from 'url'; import { ALL_CROSS_PLATFORM } from './cross-platform-tests.js'; import { parseShardArg } from './shard-arg.js'; -import { shardFiles, shardWeight } from './cross-platform-shard.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '..'); @@ -29,14 +28,8 @@ if (missing.length > 0) { process.exit(1); } -// Optional sharding (CI): `--shard=/` splits the fixed file list across -// parallel matrix shards. The split is computed HERE, by measured weight, and -// only this shard's files are handed to vitest — it is NOT passed through, -// because vitest partitions by file COUNT and this suite's runtimes span three -// orders of magnitude (see cross-platform-shard.ts). The -// Windows runner is ~5x slower than macOS/Linux on this spawn-heavy suite (~50 -// CLI/worker process spawns), so a single shard was creeping past the watchdog -// below; sharding keeps each runner well under it (see ci-tests.yml matrix). +// Pass the complete subset and --shard to the configured timing sequencer. +// Every runner reads the same snapshot, selected once by CI's shard-plan job. // Fail loud on a malformed --shard arg (mirrors the missing-files check above): // a silently-dropped shard flag would run the full unsharded suite and re-trip // the watchdog. Kept outside the execFileSync try/catch below so the message @@ -49,13 +42,7 @@ try { process.exit(1); } -// Per-shard watchdog, default 15 min. Sharding splits the file list by COUNT, not -// runtime, so the heaviest spawn suites can cluster on one shard — what this -// bounds is the *busiest* shard, not an even 1/n of wall-clock. The busiest -// Windows shard has grown to the default (14m57s on the v1.6.10-rc.19 green -// run, one observed timeout since — #2449), so CI raises the budget to 20 -// minutes via GITNEXUS_CROSS_PLATFORM_TIMEOUT_MINUTES; the default stays 15 -// for local runs. +// Preserve the 15-minute local watchdog and the 20-minute CI override. const DEFAULT_TIMEOUT_MIN = 15; const timeoutMinutes = Number.parseInt( process.env.GITNEXUS_CROSS_PLATFORM_TIMEOUT_MINUTES ?? String(DEFAULT_TIMEOUT_MIN), @@ -66,17 +53,9 @@ const timeoutMs = ? timeoutMinutes * 60 * 1000 : DEFAULT_TIMEOUT_MIN * 60 * 1000; -// Resolve the shard to an explicit file list. `--shard=i/n` is consumed here, -// never forwarded: forwarding it as well would re-partition this slice a second -// time and silently drop most of it. -const shardParts = shardArg?.replace('--shard=', '').split('/'); -const shardIndex = shardParts ? Number(shardParts[0]) : 1; -const shardTotal = shardParts ? Number(shardParts[1]) : 1; -const files = shardFiles(ALL_CROSS_PLATFORM, shardIndex, shardTotal); - console.log( - `Running ${files.length} of ${ALL_CROSS_PLATFORM.length} platform-sensitive tests` + - `${shardArg ? ` (shard ${shardIndex}/${shardTotal}, ~${shardWeight(files)}s measured weight)` : ''}...\n`, + `Running platform-sensitive test suite (${ALL_CROSS_PLATFORM.length} files total)` + + `${shardArg ? ` (${shardArg}, balanced by timing history)` : ''}...\n`, ); const startedAt = Date.now(); @@ -88,16 +67,22 @@ const reportArgs = reportName ? [ '--reporter=default', '--reporter=./scripts/execution-reporter.ts', - `--outputFile=${reportName}`, + '--reporter=blob', + `--outputFile.json=${reportName}`, + `--outputFile.blob=.vitest/blob/${reportName}`, ] : []; try { - execFileSync('npx', ['vitest', 'run', ...files, ...reportArgs], { - cwd: ROOT, - stdio: 'inherit', - timeout: timeoutMs, - shell: true, - }); + execFileSync( + 'npx', + ['vitest', 'run', ...ALL_CROSS_PLATFORM, ...(shardArg ? [shardArg] : []), ...reportArgs], + { + cwd: ROOT, + stdio: 'inherit', + timeout: timeoutMs, + shell: true, + }, + ); } catch (err) { // execFileSync sets `killed`/`signal` when the watchdog above kills vitest. const e = err as { diff --git a/gitnexus/scripts/shard-timings-reporter.ts b/gitnexus/scripts/shard-timings-reporter.ts new file mode 100644 index 000000000..75c61f17c --- /dev/null +++ b/gitnexus/scripts/shard-timings-reporter.ts @@ -0,0 +1,57 @@ +import path from 'node:path'; +import type { TestModule, TestRunEndReason, Vitest } from 'vitest/node'; +import { TimingsReporter } from 'vitest-shard-balancer'; +import { ALL_CROSS_PLATFORM } from './cross-platform-tests.js'; +import { shardTimingKey, shardTimingsFile } from './shard-timings.js'; + +export default class ShardTimingsReporter extends TimingsReporter { + private context!: Vitest; + + constructor() { + // The package's whole-config check cannot describe our native subset. + // Check the exact cohort below, before excluding skipped modules from learning. + super({ timingsFile: shardTimingsFile(), requireAllFiles: false }); + } + + override onInit(context: Vitest): void { + this.context = context; + super.onInit(context); + } + + override async onTestRunEnd( + modules: readonly TestModule[], + errors: readonly unknown[] = [], + reason: TestRunEndReason = 'interrupted', + ): Promise { + if (!this.context.config.mergeReports) return; + if (errors.length || reason !== 'passed') { + throw new Error('Refusing to learn shard timings from an unsuccessful test run'); + } + const root = this.context.config.root; + const native = (process.env.GITNEXUS_SHARD_PROFILE ?? 'coverage') !== 'coverage'; + const nativeFiles = new Set(ALL_CROSS_PLATFORM); + const specs = (await this.context.globTestSpecifications()).filter( + (spec) => + !native || nativeFiles.has(path.relative(root, spec.moduleId).split(path.sep).join('/')), + ); + const expected = new Set( + specs.map((spec) => shardTimingKey(root, spec.moduleId, spec.project.name)), + ); + const actual = new Set(); + for (const module of modules) { + const key = shardTimingKey(root, module.moduleId, module.project.name); + if (actual.has(key) || !expected.has(key)) { + throw new Error(`Duplicate or unexpected test module in shard reports: ${key}`); + } + actual.add(key); + } + const missing = [...expected].filter((key) => !actual.has(key)); + if (!actual.size || missing.length) { + throw new Error( + `Incomplete shard reports; missing ${missing.length} test modules: ${missing.join(', ')}`, + ); + } + // A platform-specific skip must not turn a previously expensive file into zero work. + await super.onTestRunEnd(modules.filter((module) => module.state() === 'passed')); + } +} diff --git a/gitnexus/scripts/shard-timings.ts b/gitnexus/scripts/shard-timings.ts new file mode 100644 index 000000000..7ab44d316 --- /dev/null +++ b/gitnexus/scripts/shard-timings.ts @@ -0,0 +1,16 @@ +import path from 'node:path'; + +/** Separate native costs from coverage and from the other operating system. */ +export function shardTimingsFile(): string { + const profile = process.env.GITNEXUS_SHARD_PROFILE ?? 'coverage'; + if (!['coverage', 'windows-latest', 'macos-latest'].includes(profile)) { + throw new Error(`Unknown shard timing profile: ${profile}`); + } + return `.vitest/timings/${profile}.json`; +} + +/** Matches vitest-shard-balancer's project-qualified timing identity. */ +export function shardTimingKey(root: string, moduleId: string, projectName: string): string { + const relative = path.relative(root, moduleId).split(path.sep).join('/'); + return projectName ? `${projectName}:${relative}` : relative; +} diff --git a/gitnexus/test/helpers/perf-sequencer.ts b/gitnexus/test/helpers/perf-sequencer.ts index 9dd6d1fb3..e100cd480 100644 --- a/gitnexus/test/helpers/perf-sequencer.ts +++ b/gitnexus/test/helpers/perf-sequencer.ts @@ -1,35 +1,52 @@ +import { statSync } from 'node:fs'; +import path from 'node:path'; import { BaseSequencer, type TestSpecification } from 'vitest/node'; -import { assignShards, specWeight } from './shard-balance.js'; +import { partition, readTimings } from 'vitest-shard-balancer'; +import { shardTimingKey, shardTimingsFile } from '../../scripts/shard-timings.js'; /** * Cost-balanced shard sequencer (wired via `sequence.sequencer` in * vitest.config.ts). * - * Overrides only `shard()` — `sort()` keeps the base behaviour so the projects' - * `groupOrder` and duration-cache ordering are untouched. Instead of vitest's - * default hash split (balanced by file COUNT, which piled the spawn-heavy suites - * onto one runner), it balances by estimated WORK (see `specWeight`): the - * fileParallelism:false spawn-heavy files are spread evenly across shards, so the - * slowest shard's wall-clock drops and no single runner carries all the - * contention. The partition stays complete and disjoint (see `assignShards`). + * Uses the package's duration partitioner with the frozen CI timing snapshot. + * Only shard() changes; BaseSequencer keeps groupOrder and local sorting. + * On a cold start, spread sequential/native work using a generic size estimate + * instead of the count split that previously clustered the slowest suites. */ export default class PerfSequencer extends BaseSequencer { override async shard(specs: TestSpecification[]): Promise { const shard = this.ctx.config.shard; if (!shard) return specs; - const groups = assignShards( - specs, - shard.count, - (spec) => { - const config = spec.project.config; - const fileParallelism = - 'fileParallelism' in config && typeof config.fileParallelism === 'boolean' - ? config.fileParallelism - : undefined; - return specWeight({ moduleId: spec.moduleId, project: { config: { fileParallelism } } }); - }, - (spec) => spec.moduleId, + const root = this.ctx.config.root; + const keyed = new Map( + specs.map((spec) => [shardTimingKey(root, spec.moduleId, spec.project.name), spec]), + ); + if (keyed.size !== specs.length) throw new Error('Duplicate test specification in shard input'); + const keys = [...keyed.keys()]; + const timings = readTimings(path.resolve(root, shardTimingsFile())); + let groups = partition(keys, timings, shard.count); + if (!groups) { + const estimates = Object.fromEntries( + [...keyed].map(([key, spec]) => { + let sizeKb = 0; + try { + sizeKb = statSync(spec.moduleId).size / 1024; + } catch { + /* Virtual test module. */ + } + const config = spec.project.config; + const sequential = 'fileParallelism' in config && config.fileParallelism === false; + // Keep all cold weights on the same scale: 1,000 points represents + // 20 seconds of sequential work. A millisecond value here would + // dwarf file size and pile the parallel cohort onto fewer shards. + return [key, (sequential ? 1_000 : 1) + sizeKb]; + }), + ); + groups = partition(keys, estimates, shard.count); + } + const selected = new Set(groups?.[shard.index - 1] ?? []); + return specs.filter((spec) => + selected.has(shardTimingKey(root, spec.moduleId, spec.project.name)), ); - return groups[shard.index - 1] ?? []; } } diff --git a/gitnexus/test/unit/cross-platform-shard.test.ts b/gitnexus/test/unit/cross-platform-shard.test.ts deleted file mode 100644 index caf8cca2b..000000000 --- a/gitnexus/test/unit/cross-platform-shard.test.ts +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Pins the weight-aware split behind the cross-platform matrix (#2449). - * - * The regression this guards is specific and was expensive: three CHEAP files - * were registered in `SPAWN_CLI`, vitest re-partitioned the list by file COUNT, - * and the reshuffle clustered `cli-e2e` (now 621 s on Windows) with `cli-limit-e2e` - * (75 s) and `analyze-heap-oom-e2e` (23 s) on one shard, which then blew the - * 20-minute watchdog. The added files cost nothing; the COUNT-split did it. - * - * So the load-bearing case here is not "the split is even" — it is - * "adding a cheap file does not move a heavy one". A partition that merely - * balanced totals could still reshuffle everything on every insertion and would - * reproduce the outage exactly. - */ - -import { readFileSync } from 'node:fs'; -import { describe, it, expect } from 'vitest'; -import { - shardFiles, - shardWeight, - weightOf, - WINDOWS_WEIGHTS_SEC, -} from '../../scripts/cross-platform-shard.js'; -import { - ALL_CROSS_PLATFORM, - LBUG_NATIVE, - WORKER_THREADS, -} from '../../scripts/cross-platform-tests.js'; - -// Replay the actual CI partition, including shard-count changes as the suite grows. -const workflow = readFileSync( - new URL('../../../.github/workflows/ci-tests.yml', import.meta.url), - 'utf8', -); -const SHARD_TOTAL = Number(workflow.match(/^\s+TOTAL=(\d+)\b/m)?.[1]); - -/** Every shard of a split, as file lists. */ -const allShards = (files: readonly string[], total: number): readonly (readonly string[])[] => - Array.from({ length: total }, (_unused, i) => shardFiles(files, i + 1, total)); - -describe('cross-platform shard partition', () => { - it('selects the native inline-handler proofs exactly once across the CI shards', () => { - const proofs = [ - [LBUG_NATIVE, 'test/integration/taint-explain.test.ts'], - [WORKER_THREADS, 'test/integration/cfg/interproc-taint.test.ts'], - [WORKER_THREADS, 'test/integration/function-local-identity.test.ts'], - ] as const; - const selected = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).flat(); - for (const [category, file] of proofs) { - expect(category.filter((candidate) => candidate === file)).toEqual([file]); - expect(selected.filter((candidate) => candidate === file)).toEqual([file]); - } - }); - - it('selects both real-worker local-import proofs exactly once across the CI shards', () => { - const proofs = [ - 'test/integration/local-import-worker-parity.test.ts', - 'test/integration/resolvers/local-import-persistence.test.ts', - ]; - const selected = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).flat(); - for (const file of proofs) { - expect(WORKER_THREADS.filter((candidate) => candidate === file)).toEqual([file]); - expect(ALL_CROSS_PLATFORM.filter((candidate) => candidate === file)).toEqual([file]); - expect(selected.filter((candidate) => candidate === file)).toEqual([file]); - } - }); - - it('does not recreate the overloaded Windows shard from the #3190 CI run', () => { - // Run 34014266125: these serialized DB suites were missing or undercharged - // in the scheduling table. Keep the observed profile independent of the - // table so deleting a weight cannot make this regression pass again. - const observed: Readonly> = { - 'test/integration/skills-e2e.test.ts': 550, - 'test/unit/incremental-index-extension-dml-gate.test.ts': 414, - 'test/integration/fts-extension-e2e.test.ts': 380, - 'test/integration/analyze-wal-checkpoint-failure.test.ts': 86, - 'test/integration/skip-fts.test.ts': 110, - }; - const floor = weightOf('test/unmeasured.test.ts'); - const loads = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).map((files) => - files.reduce((sum, file) => sum + Math.max(weightOf(file), (observed[file] ?? 0) + floor), 0), - ); - // This is a deterministic replay of recorded weights, not a timing test. - // The runner's existing watchdog remains 20 minutes. - expect(Math.max(...loads)).toBeLessThan(20 * 60); - const shards = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL); - const heavyweightLocations = [ - 'test/integration/cli-e2e.test.ts', - 'test/integration/skills-e2e.test.ts', - 'test/unit/incremental-index-extension-dml-gate.test.ts', - ].map((file) => shards.findIndex((files) => files.includes(file))); - expect(heavyweightLocations).not.toContain(-1); - expect(new Set(heavyweightLocations).size).toBe(heavyweightLocations.length); - expect( - shards.filter( - (s) => - s.includes('test/integration/skills-e2e.test.ts') && - s.includes('test/integration/fts-extension-e2e.test.ts'), - ), - ).toEqual([]); - }); - - it('covers every file exactly once, with no overlap between shards', () => { - const shards = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL); - const seen = shards.flatMap((s) => [...s]); - - expect(seen.slice().sort()).toEqual([...ALL_CROSS_PLATFORM].sort()); - expect(new Set(seen).size).toBe(ALL_CROSS_PLATFORM.length); - }); - - it('leaves watchdog headroom for the recorded October Windows workload', () => { - // Run 38697469283 exhausted the 20-minute watchdog before cli-e2e - // finished. Its preceding serialized suites were undercharged or absent - // from the weight table. The CLI and Zig timings below come from the - // preceding successful run 38693066214; the other timings are from the - // failing run. Keep this replay independent of the scheduling table. - const observed: Readonly> = { - 'test/integration/cli-e2e.test.ts': 563, - 'test/integration/mcp-index-generation.test.ts': 201, - 'test/integration/rename-provider-objective-c.test.ts': 285, - 'test/integration/rename-provider-zig.test.ts': 183, - 'test/integration/rename-provider-go.test.ts': 221, - 'test/integration/worker-pool.test.ts': 254, - 'test/integration/rename-provider-swift.test.ts': 300, - 'test/integration/rename-provider-c.test.ts': 202, - 'test/unit/incremental-dirty-recovery.test.ts': 76, - }; - const floor = weightOf('test/unmeasured.test.ts'); - const loads = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).map((files) => - files.reduce((sum, file) => sum + Math.max(weightOf(file), (observed[file] ?? 0) + floor), 0), - ); - // Leave at least three minutes for hosted-runner variation. Merely - // fitting under 20 minutes left the previous partition prone to timeout. - expect(Math.max(...loads)).toBeLessThanOrEqual(17 * 60); - }); - - it('leaves watchdog headroom for the October 23 native-suite timeout', () => { - // Windows run 39934695864: shard 10 timed out with Kotlin still pending. - // Use its completed suite times, plus the October 25 green run 40172490240 - // for query-symbol-integrity and incremental-dirty-recovery. These measured - // costs must remain independent of the scheduling hints being tested. - const observed: Readonly> = { - 'test/integration/rename-provider-objective-c.test.ts': 385.174, - 'test/integration/rename-provider-csharp.test.ts': 265.99, - 'test/integration/incremental-method-integrity.test.ts': 144.974, - 'test/integration/fts-repair-warm-session.test.ts': 61.015, - 'test/integration/query-symbol-integrity.test.ts': 33.892, - 'test/unit/incremental-dirty-recovery.test.ts': 127.8, - }; - const floor = weightOf('test/unmeasured.test.ts'); - const loads = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).map((files) => - files.reduce((sum, file) => sum + Math.max(weightOf(file), (observed[file] ?? 0) + floor), 0), - ); - expect(Math.max(...loads)).toBeLessThanOrEqual(17 * 60); - }); - - it('keeps each shard within a shard of the ideal weight', () => { - const shards = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL); - const weights = shards.map(shardWeight); - const ideal = shardWeight(ALL_CROSS_PLATFORM) / SHARD_TOTAL; - - // LPT's guarantee is 4/3 of optimal, and optimal is at least the ideal - // average. A hard 1.34x ceiling on the busiest shard is what keeps the - // matrix inside its watchdog no matter how the list is edited. - expect(Math.max(...weights)).toBeLessThanOrEqual(ideal * 1.34); - }); - - it('keeps the CLI and worker-pool suites on different shards', () => { - // The exact shape of the outage: cli-e2e and worker-pool are 621 s and - // 222 s, so together they are most of a shard's budget before anything else - // is scheduled. - const shards = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL); - const withBoth = shards.filter( - (s) => - s.includes('test/integration/cli-e2e.test.ts') && - s.includes('test/integration/worker-pool.test.ts'), - ); - - expect(withBoth).toEqual([]); - }); - - it('does not move a heavy file when a cheap file is added — the #2449 regression', () => { - const heavy = Object.keys(WINDOWS_WEIGHTS_SEC); - const placementOf = (files: readonly string[]): ReadonlyMap => { - const shards = allShards(files, SHARD_TOTAL); - return new Map( - heavy - .map((f) => [f, shards.findIndex((s) => s.includes(f))] as const) - .filter(([, i]) => i >= 0), - ); - }; - - const before = placementOf(ALL_CROSS_PLATFORM); - - // The inserted names sort EARLY, and there is a case that is NOT a multiple - // of the shard count. Both details are load-bearing, and getting them wrong - // made earlier versions of this test vacuous: - // - names that sort last cannot disturb anything under any scheme; - // - adding exactly `total` files leaves an equal-weight round-robin in the - // same rotation, so a count-split would pass too. - // Under the real weighted split, heavy files are scheduled before every - // light one, so no number of cheap insertions can move them. - const afterOne = placementOf([...ALL_CROSS_PLATFORM, 'test/aaa-new-cheap-a.test.ts']); - const afterTwo = placementOf([ - ...ALL_CROSS_PLATFORM, - 'test/aaa-new-cheap-a.test.ts', - 'test/aaa-new-cheap-b.test.ts', - ]); - - expect(Object.fromEntries(afterOne)).toMatchObject(Object.fromEntries(before)); - expect(Object.fromEntries(afterTwo)).toMatchObject(Object.fromEntries(before)); - }); - - it('is deterministic, so every runner computes the same split independently', () => { - // Each matrix job resolves its own slice on its own machine with no shared - // state, so an unstable sort would silently drop or duplicate files. - const once = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).map((s) => [...s]); - const twice = allShards([...ALL_CROSS_PLATFORM].reverse(), SHARD_TOTAL).map((s) => - [...s].sort(), - ); - - expect(twice).toEqual(once.map((s) => [...s].sort())); - }); - - it('returns every file for a single-shard run, and rejects an out-of-range shard', () => { - expect(shardFiles(ALL_CROSS_PLATFORM, 1, 1)).toEqual([...ALL_CROSS_PLATFORM]); - expect(() => shardFiles(ALL_CROSS_PLATFORM, 0, 3)).toThrow(/shard index/); - expect(() => shardFiles(ALL_CROSS_PLATFORM, 4, 3)).toThrow(/shard index/); - expect(() => shardFiles(ALL_CROSS_PLATFORM, 1, 0)).toThrow(/shard total/); - }); - - it('charges every file the per-file floor, so light files are never free', () => { - // Without this, the balancer isolates the monsters and then piles all the - // light files onto the remaining shards — a count imbalance that costs just - // as much wall clock as the runtime one it just fixed. - expect(weightOf('test/unit/zzz-does-not-exist.test.ts')).toBeGreaterThan(0); - expect(weightOf('test/integration/cli-e2e.test.ts')).toBeGreaterThan( - WINDOWS_WEIGHTS_SEC['test/integration/cli-e2e.test.ts'] ?? 0, - ); - }); - - it('weights only files that are actually registered', () => { - // A weight entry for a file no longer in the list is dead config that the - // balancer silently ignores; catching it here keeps the table honest. - const registered = new Set(ALL_CROSS_PLATFORM); - const stale = Object.keys(WINDOWS_WEIGHTS_SEC).filter((f) => !registered.has(f)); - - expect(stale).toEqual([]); - }); -}); diff --git a/gitnexus/test/unit/perf-sequencer.test.ts b/gitnexus/test/unit/perf-sequencer.test.ts new file mode 100644 index 000000000..da4537486 --- /dev/null +++ b/gitnexus/test/unit/perf-sequencer.test.ts @@ -0,0 +1,113 @@ +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { TestSpecification, Vitest } from 'vitest/node'; +import PerfSequencer from '../helpers/perf-sequencer.js'; + +const roots: string[] = []; +afterEach(() => { + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function fixture() { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-shards-')); + roots.push(root); + mkdirSync(path.join(root, '.vitest/timings'), { recursive: true }); + const specs = ['a', 'b', 'c', 'd'].map((name) => { + const moduleId = path.join(root, `${name}.test.ts`); + writeFileSync(moduleId, '// same size\n'); + return { + moduleId, + project: { name: 'native', config: { fileParallelism: false } }, + } as unknown as TestSpecification; + }); + const history = (profile: string, weights: number[]) => + writeFileSync( + path.join(root, '.vitest/timings', `${profile}.json`), + JSON.stringify( + Object.fromEntries( + specs.map((spec, i) => [`native:${path.basename(spec.moduleId)}`, weights[i]]), + ), + ), + ); + const split = async () => + Promise.all( + [1, 2].map(async (index) => { + const ctx = { config: { root, shard: { index, count: 2 } } } as Vitest; + return new PerfSequencer(ctx).shard(specs); + }), + ); + return { root, specs, history, split }; +} + +describe('timing-based CI sharding', () => { + it('learns from measured durations instead of putting both slow files on one shard', async () => { + const { specs, history, split } = fixture(); + history('coverage', [10000, 10, 9000, 10]); + const bins = await split(); + expect(bins.some((bin) => bin.includes(specs[0]) && bin.includes(specs[2]))).toBe(false); + expect( + bins + .flat() + .map((spec) => spec.moduleId) + .sort(), + ).toEqual(specs.map((spec) => spec.moduleId).sort()); + }); + + it('keeps platform histories separate and gives independent runners identical partitions', async () => { + const { specs, history, split } = fixture(); + history('windows-latest', [10000, 10, 9000, 10]); + history('macos-latest', [9000, 10000, 10, 10]); + vi.stubEnv('GITNEXUS_SHARD_PROFILE', 'windows-latest'); + const windows = await split(); + expect(await split()).toEqual(windows); + expect(windows.some((bin) => bin.includes(specs[0]) && bin.includes(specs[2]))).toBe(false); + vi.stubEnv('GITNEXUS_SHARD_PROFILE', 'macos-latest'); + const mac = await split(); + expect(mac.some((bin) => bin.includes(specs[0]) && bin.includes(specs[1]))).toBe(false); + expect(mac).not.toEqual(windows); + }); + + it('still schedules new files and cold starts as complete disjoint partitions', async () => { + const { specs, history, split } = fixture(); + const cold = await split(); + expect(cold.map((bin) => bin.length)).toEqual([2, 2]); + history('coverage', [10000]); + const warm = await split(); + expect(new Set(warm.flat()).size).toBe(specs.length); + expect(warm.flat().length).toBe(specs.length); + }); + + it('spreads the parallel cohort across all cold shards when sequential counts differ', async () => { + const { root } = fixture(); + const specs = Array.from({ length: 1414 }, (_, i) => { + const moduleId = path.join(root, `${i}.test.ts`); + writeFileSync(moduleId, ' '.repeat(10 * 1024)); + const sequential = i < 94; + return { + moduleId, + project: { + name: sequential ? 'native' : 'default', + config: { fileParallelism: !sequential }, + }, + } as unknown as TestSpecification; + }); + const bins = await Promise.all( + Array.from({ length: 8 }, (_, i) => { + const ctx = { config: { root, shard: { index: i + 1, count: 8 } } } as Vitest; + return new PerfSequencer(ctx).shard(specs); + }), + ); + const parallelCounts = bins.map( + (bin) => bin.filter((spec) => spec.project.name === 'default').length, + ); + // A one-file difference in sequential work must not concentrate the entire + // parallel cohort on two runners (660 files each in the failing CI run). + expect(Math.min(...parallelCounts)).toBeGreaterThan(100); + expect(Math.max(...parallelCounts)).toBeLessThan(250); + expect(new Set(bins.flat()).size).toBe(specs.length); + expect(bins.flat()).toHaveLength(specs.length); + }); +}); diff --git a/gitnexus/test/unit/shard-timings-reporter.test.ts b/gitnexus/test/unit/shard-timings-reporter.test.ts new file mode 100644 index 000000000..4e2c8a064 --- /dev/null +++ b/gitnexus/test/unit/shard-timings-reporter.test.ts @@ -0,0 +1,87 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { TestModule, Vitest } from 'vitest/node'; +import ShardTimingsReporter from '../../scripts/shard-timings-reporter.js'; + +const roots: string[] = []; +afterEach(() => { + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function fixture() { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-timings-')); + roots.push(root); + const file = path.join(root, '.vitest/timings/coverage.json'); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, JSON.stringify({ 'native:a.test.ts': 1000, 'native:b.test.ts': 500 })); + const modules = ['a', 'b'].map((name) => ({ + moduleId: path.join(root, `${name}.test.ts`), + project: { name: 'native' }, + state: () => 'passed', + diagnostic: () => ({ + environmentSetupDuration: 10, + prepareDuration: 20, + setupDuration: 30, + collectDuration: 40, + duration: 100, + }), + })) as unknown as TestModule[]; + const reporter = new ShardTimingsReporter(); + reporter.onInit({ + config: { root, mergeReports: 'blobs' }, + globTestSpecifications: async () => modules, + } as unknown as Vitest); + return { file, modules, reporter }; +} + +describe('learning shard timings', () => { + it('includes setup and collection, smooths measurements, and retains skipped-file history', async () => { + const { file, modules, reporter } = fixture(); + modules[1].state = () => 'skipped'; + await reporter.onTestRunEnd(modules, [], 'passed'); + expect(JSON.parse(readFileSync(file, 'utf8'))).toEqual({ + 'native:a.test.ts': 600, + 'native:b.test.ts': 500, + }); + }); + + it.each(['missing', 'duplicate', 'unexpected'] as const)( + 'rejects %s modules without replacing history', + async (kind) => { + const { file, modules, reporter } = fixture(); + const before = readFileSync(file, 'utf8'); + const inputs = + kind === 'missing' + ? modules.slice(0, 1) + : [ + ...modules, + kind === 'duplicate' + ? modules[0] + : ({ ...modules[0], moduleId: '/unexpected.test.ts' } as unknown as TestModule), + ]; + await expect(reporter.onTestRunEnd(inputs, [], 'passed')).rejects.toThrow( + /Incomplete|Duplicate or unexpected/, + ); + expect(readFileSync(file, 'utf8')).toBe(before); + }, + ); + + it.each(['failed', 'interrupted'] as const)('does not learn from a %s run', async (reason) => { + const { file, modules, reporter } = fixture(); + const before = readFileSync(file, 'utf8'); + await expect(reporter.onTestRunEnd(modules, [], reason)).rejects.toThrow(/unsuccessful/); + expect(readFileSync(file, 'utf8')).toBe(before); + }); + + it('does not learn when the runner reports an unhandled error', async () => { + const { file, modules, reporter } = fixture(); + const before = readFileSync(file, 'utf8'); + await expect( + reporter.onTestRunEnd(modules, [new Error('worker crashed')], 'passed'), + ).rejects.toThrow(/unsuccessful/); + expect(readFileSync(file, 'utf8')).toBe(before); + }); +}); diff --git a/gitnexus/test/unit/shard-timings-roundtrip.test.ts b/gitnexus/test/unit/shard-timings-roundtrip.test.ts new file mode 100644 index 000000000..de8d72ed3 --- /dev/null +++ b/gitnexus/test/unit/shard-timings-roundtrip.test.ts @@ -0,0 +1,96 @@ +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { expect, it } from 'vitest'; + +it('runs disjoint Vitest shards, preserves JSON receipts, learns from blobs, and rejects a missing shard', () => { + const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-shard-roundtrip-')); + const vitest = fileURLToPath(new URL('../../node_modules/vitest/vitest.mjs', import.meta.url)); + const sequencer = fileURLToPath(new URL('../helpers/perf-sequencer.ts', import.meta.url)); + const reporter = fileURLToPath( + new URL('../../scripts/shard-timings-reporter.ts', import.meta.url), + ); + const execution = fileURLToPath(new URL('../../scripts/execution-reporter.ts', import.meta.url)); + const timingFile = path.join(root, '.vitest/timings/coverage.json'); + try { + mkdirSync(path.dirname(timingFile), { recursive: true }); + writeFileSync( + timingFile, + JSON.stringify({ + 'serial:a.test.ts': 10000, + 'serial:c.test.ts': 9000, + 'parallel:b.test.ts': 10, + 'parallel:d.test.ts': 10, + }), + ); + writeFileSync( + path.join(root, 'vitest.config.ts'), + ` + import { defineConfig } from ${JSON.stringify(import.meta.resolve('vitest/config'))}; + import PerfSequencer from ${JSON.stringify(sequencer.replaceAll('\\', '/'))}; + export default defineConfig({ test: { + sequence: { sequencer: PerfSequencer }, + projects: [ + { extends: true, test: { name: 'serial', include: ['a.test.ts', 'c.test.ts'], fileParallelism: false, sequence: { groupOrder: 1 } } }, + { extends: true, test: { name: 'parallel', include: ['b.test.ts', 'd.test.ts'], sequence: { groupOrder: 2 } } } + ] + } }); + `, + ); + for (const name of ['a', 'b', 'c', 'd']) { + writeFileSync( + path.join(root, `${name}.test.ts`), + `import { test, expect } from ${JSON.stringify(import.meta.resolve('vitest'))}; test('${name}', () => expect(1 + 1).toBe(2));`, + ); + } + const run = (args: string[]) => + spawnSync( + process.execPath, + [vitest, 'run', '--root', root, '--config', path.join(root, 'vitest.config.ts'), ...args], + { + cwd: root, + encoding: 'utf8', + timeout: 30000, + env: { ...process.env, GITNEXUS_SHARD_PROFILE: 'coverage', NO_COLOR: '1' }, + }, + ); + const files: string[][] = []; + for (const index of [1, 2]) { + const result = run([ + `--shard=${index}/2`, + '--reporter=blob', + `--reporter=${execution}`, + `--outputFile.blob=blobs/${index}.json`, + `--outputFile.json=receipt-${index}.json`, + '--maxWorkers=1', + ]); + expect(result.status, result.stdout + result.stderr).toBe(0); + const receipt = JSON.parse(readFileSync(path.join(root, `receipt-${index}.json`), 'utf8')); + expect(receipt.executionReason).toBe('passed'); + files.push(receipt.testResults.map((entry: { name: string }) => path.basename(entry.name))); + } + expect(files.flat().sort()).toEqual(['a.test.ts', 'b.test.ts', 'c.test.ts', 'd.test.ts']); + expect(files.some((bin) => bin.includes('a.test.ts') && bin.includes('c.test.ts'))).toBe(false); + const result = run(['--mergeReports=blobs', `--reporter=${reporter}`]); + expect(result.status, result.stdout + result.stderr).toBe(0); + const learned = readFileSync(timingFile, 'utf8'); + const timings = JSON.parse(learned); + expect(Object.keys(timings).sort()).toEqual([ + 'parallel:b.test.ts', + 'parallel:d.test.ts', + 'serial:a.test.ts', + 'serial:c.test.ts', + ]); + expect(timings['serial:a.test.ts']).toBeGreaterThanOrEqual(5000); + expect(timings['serial:a.test.ts']).toBeLessThan(10000); + rmSync(path.join(root, 'blobs/2.json')); + const incomplete = run(['--mergeReports=blobs', `--reporter=${reporter}`]); + expect(incomplete.status).not.toBe(0); + expect(incomplete.stdout + incomplete.stderr).toContain('Incomplete shard reports'); + expect(readFileSync(timingFile, 'utf8')).toBe(learned); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}, 120000); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index d26305145..71945fd5b 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -51,10 +51,8 @@ export default defineConfig({ }, }, - // Balance shards by estimated work rather than file count, so the - // spawn-heavy sequential suites spread evenly across shard runners instead - // of clustering onto one (see test/helpers/perf-sequencer.ts). Only shard() - // is overridden — groupOrder and sort order are left to the base sequencer. + // Balance shards using vitest-shard-balancer and the frozen CI timing history. + // groupOrder and sort order are left to the base sequencer. sequence: { sequencer: PerfSequencer, }, @@ -165,7 +163,7 @@ export default defineConfig({ // file-lock exposure this project exists to serialize (TESTING.md // § Vitest projects). Registering it here does NOT narrow where it // runs: vitest applies `--shard` once to the combined cross-project - // spec list (PerfSequencer/assignShards is a complete, disjoint + // spec list (PerfSequencer is a complete, disjoint // partition), and run-cross-platform.ts hands vitest explicit file // paths, which resolve against every project's include list. Its // `incremental-vector-extension-ordering` /