mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-11-11 08:37:35 +00:00
feat(ci): rebalance test shards from recorded timings (#3748)
* feat(ci): rebalance test shards from recorded timings * fix(ci): preserve artifact retries for shard timings Find shard-plan generation by its stable step ID and clarify the native runner cohort count (#3748). Note: pre-existing reference-oracle parser type failures in two test_oracle_assets cases remain outside this change. * fix(ci): keep retry timing snapshots immutable Require the existing snapshot on reruns so a failed download cannot refresh history and change shard assignments. Cover the workflow contract and document recovery when the artifact is unavailable. * docs(ci): document shard timing profile selection * fix(ci): balance cold shard estimates consistently --------- Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com>
This commit is contained in:
Gergő Magyar
GitHub
parent
51ce19d77a
commit
13d06d41c4
17 changed files with 629 additions and 538 deletions
142
.github/workflows/ci-tests.yml
vendored
142
.github/workflows/ci-tests.yml
vendored
|
|
@ -139,6 +139,11 @@ jobs:
|
|||
- name: Ensure FTS + VECTOR extensions installed
|
||||
run: npx tsx scripts/ensure-fts.ts
|
||||
working-directory: gitnexus
|
||||
- name: Download frozen timing snapshot
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: shard-timing-input
|
||||
path: gitnexus/.vitest/timings
|
||||
- name: Run sharded tests with coverage (blob)
|
||||
# Shard via env var (not `${{ }}` inlined into the shell) so it isn't a
|
||||
# template-injection sink; shell: bash makes "$SHARD" expand uniformly.
|
||||
|
|
@ -195,11 +200,17 @@ jobs:
|
|||
pattern: coverage-blob-*
|
||||
path: gitnexus/.vitest/blob
|
||||
merge-multiple: true
|
||||
- name: Download frozen timing snapshot
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: shard-timing-input
|
||||
path: gitnexus/.vitest/timings
|
||||
- name: Merge coverage + enforce thresholds
|
||||
run: >-
|
||||
npx vitest --mergeReports=.vitest/blob
|
||||
--reporter=default
|
||||
--reporter=./scripts/execution-reporter.ts
|
||||
--reporter=./scripts/shard-timings-reporter.ts
|
||||
--outputFile=test-results.json
|
||||
--coverage
|
||||
--coverage.reporter=json-summary
|
||||
|
|
@ -207,6 +218,12 @@ jobs:
|
|||
--coverage.reporter=text
|
||||
--coverage.thresholds.autoUpdate=false
|
||||
working-directory: gitnexus
|
||||
- name: Upload learned coverage timings
|
||||
uses: ./.github/actions/upload-test-artifact
|
||||
with:
|
||||
name: timings-coverage
|
||||
path: gitnexus/.vitest/timings/coverage.json
|
||||
include-hidden-files: true
|
||||
# gitnexus-shared already built by setup-gitnexus above
|
||||
- name: Install gitnexus-web dependencies
|
||||
run: npm ci
|
||||
|
|
@ -236,7 +253,8 @@ jobs:
|
|||
# Single source of truth for the platform-sensitive shard count. TOTAL below
|
||||
# generates both the shard index list (the matrix) and the /N denominator (job
|
||||
# name + --shard arg), so they can't drift — bump the shard count by editing
|
||||
# TOTAL alone. Checkout-free (ubuntu ships jq), so no credential surface.
|
||||
# TOTAL alone. Restore timing history once: independent restores in each shard
|
||||
# could select different histories while another run publishes an update.
|
||||
shard-plan:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
|
|
@ -245,6 +263,41 @@ jobs:
|
|||
cov_shards: ${{ steps.gen.outputs.cov_shards }}
|
||||
cov_total: ${{ steps.gen.outputs.cov_total }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Reuse this run's timing input on a retry
|
||||
id: previous-input
|
||||
if: github.run_attempt > 1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: shard-timing-input
|
||||
path: .vitest/timings
|
||||
- name: Restore last trusted timing history
|
||||
if: github.run_attempt == 1
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5
|
||||
with:
|
||||
path: .vitest/timings
|
||||
key: test-timings-v1-node22-vitest5-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
restore-keys: test-timings-v1-node22-vitest5-
|
||||
- name: Prepare timing snapshot
|
||||
if: github.run_attempt == 1
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p .vitest/timings
|
||||
for profile in coverage windows-latest macos-latest; do
|
||||
file=".vitest/timings/$profile.json"
|
||||
if [ ! -f "$file" ]; then printf '{}\n' > "$file"; fi
|
||||
done
|
||||
sha256sum .vitest/timings/*.json >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Freeze timing input for every shard and retry
|
||||
if: github.run_attempt == 1
|
||||
uses: ./.github/actions/upload-test-artifact
|
||||
with:
|
||||
name: shard-timing-input
|
||||
path: .vitest/timings/*.json
|
||||
include-hidden-files: true
|
||||
retention-days: 30
|
||||
- id: gen
|
||||
run: |
|
||||
# Keep the recorded native workload below 17 minutes, leaving
|
||||
|
|
@ -275,17 +328,8 @@ jobs:
|
|||
matrix:
|
||||
# Ubuntu already covered by the coverage job above
|
||||
os: [windows-latest, macos-latest]
|
||||
# Shard the fixed file list across N runners per OS (N = TOTAL in the
|
||||
# shard-plan job). The suite is dominated by ~50 CLI/worker process
|
||||
# spawns and Windows is ~5x slower than macOS at those, so the unsharded
|
||||
# run crept past the 15-min watchdog in run-cross-platform.ts. vitest
|
||||
# shards by file COUNT, not runtime, so the heaviest spawn suites can
|
||||
# cluster on one shard. The busiest Windows shard has grown to the old
|
||||
# 15-minute watchdog (14m57s on the v1.6.10-rc.19 green run, one
|
||||
# observed timeout since — #2449), so the job env below raises the
|
||||
# per-shard watchdog to 20 minutes, still bounded by timeout-minutes.
|
||||
# Shard indices come from the shard-plan job (single source of truth):
|
||||
# its TOTAL drives this list and the /N in the job name + --shard arg.
|
||||
# Timing-based assignment uses the same frozen input for all N runners.
|
||||
# Keep the existing per-shard watchdog and matrix count.
|
||||
shard: ${{ fromJSON(needs.shard-plan.outputs.shards) }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 25
|
||||
|
|
@ -306,6 +350,7 @@ jobs:
|
|||
# failure, never a silent skip (same contract as GITNEXUS_REQUIRE_FTS).
|
||||
GITNEXUS_REQUIRE_VECTOR: '1'
|
||||
GITNEXUS_E2E_CLI: dist
|
||||
GITNEXUS_SHARD_PROFILE: ${{ matrix.os }}
|
||||
# #2449: hosted Windows runners intermittently push the busiest shard past
|
||||
# the default 15-minute watchdog. 20 minutes restores real headroom while
|
||||
# the 25-minute job timeout above still bounds a genuine hang.
|
||||
|
|
@ -336,6 +381,11 @@ jobs:
|
|||
- name: Ensure FTS + VECTOR extensions installed
|
||||
run: npx tsx scripts/ensure-fts.ts
|
||||
working-directory: gitnexus
|
||||
- name: Download frozen timing snapshot
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: shard-timing-input
|
||||
path: gitnexus/.vitest/timings
|
||||
- name: Run platform-sensitive tests
|
||||
# Pass the shard through an env var (not `${{ }}` inlined into the shell)
|
||||
# so it isn't a template-injection sink (zizmor). shell: bash makes the
|
||||
|
|
@ -354,6 +404,74 @@ jobs:
|
|||
name: execution-${{ matrix.os }}-${{ matrix.shard }}
|
||||
path: gitnexus/${{ matrix.os }}-${{ matrix.shard }}.json
|
||||
retention-days: 5
|
||||
- name: Upload platform timing blob
|
||||
if: always()
|
||||
uses: ./.github/actions/upload-test-artifact
|
||||
with:
|
||||
name: platform-blob-${{ matrix.os }}-${{ matrix.shard }}
|
||||
path: gitnexus/.vitest/blob/
|
||||
include-hidden-files: true
|
||||
|
||||
# Merge on the originating OS so absolute paths in Vitest blobs keep their
|
||||
# meaning. No tests are re-executed; the reporter validates the native subset.
|
||||
native-timings:
|
||||
name: ${{ matrix.os }} / shard timings
|
||||
needs: cross-platform
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [windows-latest, macos-latest]
|
||||
env:
|
||||
GITNEXUS_SHARD_PROFILE: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/setup-gitnexus
|
||||
with:
|
||||
lifecycle-scripts: 'false'
|
||||
- name: Download frozen timing snapshot
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: shard-timing-input
|
||||
path: gitnexus/.vitest/timings
|
||||
- name: Download native blobs
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
pattern: platform-blob-${{ matrix.os }}-*
|
||||
path: gitnexus/.vitest/blob
|
||||
merge-multiple: true
|
||||
- name: Check native partition and learn timings
|
||||
run: npx vitest --mergeReports=.vitest/blob --reporter=default --reporter=./scripts/shard-timings-reporter.ts
|
||||
working-directory: gitnexus
|
||||
- name: Upload learned native timings
|
||||
uses: ./.github/actions/upload-test-artifact
|
||||
with:
|
||||
name: timings-${{ matrix.os }}
|
||||
path: gitnexus/.vitest/timings/${{ matrix.os }}.json
|
||||
include-hidden-files: true
|
||||
|
||||
# Only successful trusted main runs publish history. PRs consume it through
|
||||
# the planner; they never replace the baseline used by other PRs.
|
||||
save-shard-timings:
|
||||
needs: [coverage-merge, native-timings, test-completeness]
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Download learned timings
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
pattern: timings-*
|
||||
path: .vitest/timings
|
||||
merge-multiple: true
|
||||
- name: Save timing history for later runs
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5
|
||||
with:
|
||||
path: .vitest/timings
|
||||
key: test-timings-v1-node22-vitest5-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
|
||||
# Branch protection still requires these six names from the three-shard
|
||||
# matrix. Keep them as aggregate gates as the real matrix grows: every native
|
||||
|
|
|
|||
30
TESTING.md
30
TESTING.md
|
|
@ -104,6 +104,36 @@ The subset is defined in `gitnexus/scripts/cross-platform-tests.ts` and includes
|
|||
|
||||
When adding a platform-sensitive test, add it to the appropriate section in `scripts/cross-platform-tests.ts`.
|
||||
|
||||
### Automatic shard balancing
|
||||
|
||||
CI uses `vitest-shard-balancer` to distribute test files by measured duration.
|
||||
Timing history lives in GitHub Actions storage; adding a test does not require
|
||||
editing a weight table. Windows, macOS, and Ubuntu coverage each have their own
|
||||
profile because native startup and coverage costs differ.
|
||||
|
||||
The `shard-plan` job restores history once and uploads `shard-timing-input`.
|
||||
Every shard and report merge downloads that same snapshot. Retries reuse it,
|
||||
so a newly published history cannot change the partition halfway through a run.
|
||||
If the snapshot cannot be restored, the retry fails. If it was never uploaded,
|
||||
deleted, or expired, trigger a fresh workflow run with a new commit; rerunning
|
||||
the old run must not replace its timing input.
|
||||
With no usable history, the sequencer spreads sequential/native work using
|
||||
generic estimates; new files in an established profile use the package's median.
|
||||
|
||||
The merge reporter includes setup, collection, and test execution costs and
|
||||
smooths new measurements with previous ones. It rejects missing, duplicate, or
|
||||
unexpected modules and unsuccessful runs, and retains prior costs for skipped
|
||||
modules. Native blobs are merged on their originating OS. Only main-branch push
|
||||
runs that pass the execution audit publish the updated history to the Actions
|
||||
cache. Cache expiry affects balance, not test discovery or completeness.
|
||||
|
||||
For a local reproduction, download the run's `shard-timing-input` artifact into
|
||||
`gitnexus/.vitest/timings/` and set `GITNEXUS_SHARD_PROFILE` to `windows-latest`,
|
||||
`macos-latest`, or `coverage` (the default). Then use the same `--shard=i/n`
|
||||
argument with `npm run test:cross-platform -- --shard=i/n` or `npm test -- --shard=i/n`.
|
||||
Keep `.vitest/` untracked. Bump the workflow's timing-cache schema key when a
|
||||
runner/measurement change makes the stored profiles incompatible.
|
||||
|
||||
### Confirming no tests are orphaned
|
||||
|
||||
Every test file matches one of the three vitest projects. To verify:
|
||||
|
|
|
|||
|
|
@ -223,6 +223,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max
|
|||
| Variable | Default | Effect | Tune when… |
|
||||
| ----------------------------------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `GITNEXUS_TEST_REPORT` | unset | Write a JSON execution receipt from `scripts/run-cross-platform.ts`; accepts a basename ending in `.json`. | CI platform shards use distinct filenames so the completeness check can require every report. |
|
||||
| `GITNEXUS_SHARD_PROFILE` | `coverage` | Selects the timing profile for Vitest shard balancing and merged timing reports: `coverage`, `windows-latest`, or `macos-latest`. | CI selects its native OS; set the matching profile when reproducing a shard locally. See [Automatic shard balancing](../TESTING.md#automatic-shard-balancing). |
|
||||
| `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers <n>`. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. |
|
||||
| `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. |
|
||||
| `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. |
|
||||
|
|
|
|||
|
|
@ -321,7 +321,7 @@ def test_platform_ci_preserves_required_check_names_with_complete_execution_gate
|
|||
for platform in ("windows-latest", "macos-latest")
|
||||
for check in (1, 2, 3)
|
||||
}
|
||||
plan = jobs["shard-plan"]["steps"][0]["run"]
|
||||
plan = next(step["run"] for step in jobs["shard-plan"]["steps"] if step.get("id") == "gen")
|
||||
total = int(re.search(r"^TOTAL=(\d+)\b", plan, re.MULTILINE)[1])
|
||||
native_names = {
|
||||
"tests / "
|
||||
|
|
@ -408,6 +408,24 @@ def test_test_artifact_retries_cannot_hide_missing_evidence(ci_test_jobs):
|
|||
)
|
||||
|
||||
|
||||
def test_shard_plan_retry_cannot_replace_frozen_timing_input(ci_test_jobs):
|
||||
steps = ci_test_jobs["shard-plan"]["steps"]
|
||||
restore = next(step for step in steps if step.get("id") == "previous-input")
|
||||
assert restore["if"] == "github.run_attempt > 1"
|
||||
assert not restore.get("continue-on-error", False)
|
||||
assert restore["with"]["name"] == "shard-timing-input"
|
||||
|
||||
# A failed restore must never fall through to newer history, including
|
||||
# upload retries that may overwrite an artifact left by a previous attempt.
|
||||
for name in (
|
||||
"Restore last trusted timing history",
|
||||
"Prepare timing snapshot",
|
||||
"Freeze timing input for every shard and retry",
|
||||
):
|
||||
step = next(step for step in steps if step.get("name") == name)
|
||||
assert step["if"] == "github.run_attempt == 1", name
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native_result", ["success", "failure", "cancelled", "skipped", ""])
|
||||
@pytest.mark.parametrize("execution_result", ["success", "failure", "cancelled", "skipped", ""])
|
||||
@pytest.mark.parametrize("scope_bench_result", ["success", "failure", "cancelled", "skipped", ""])
|
||||
|
|
|
|||
16
gitnexus/package-lock.json
generated
16
gitnexus/package-lock.json
generated
|
|
@ -86,7 +86,8 @@
|
|||
"jsdom": "30.1.1",
|
||||
"tsx": "^4.0.0",
|
||||
"typescript": "^7.0.2",
|
||||
"vitest": "^5.0.0"
|
||||
"vitest": "^5.0.0",
|
||||
"vitest-shard-balancer": "0.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^22.18.0 || >=24.11.0"
|
||||
|
|
@ -7349,6 +7350,19 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/vitest-shard-balancer": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/vitest-shard-balancer/-/vitest-shard-balancer-0.1.0.tgz",
|
||||
"integrity": "sha512-pUMR4vaEKKMnfwvCfLKSbYhbb3mPxKzii69toipqmupvO48T4jl2jx57qK/nnGknImkjO9i+fTfWQv/X4096RA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^22.12.0 || >=24.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"vitest": ">=3.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/w3c-xmlserializer": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-6.0.0.tgz",
|
||||
|
|
|
|||
|
|
@ -150,7 +150,8 @@
|
|||
"jsdom": "30.1.1",
|
||||
"tsx": "^4.0.0",
|
||||
"typescript": "^7.0.2",
|
||||
"vitest": "^5.0.0"
|
||||
"vitest": "^5.0.0",
|
||||
"vitest-shard-balancer": "0.1.0"
|
||||
},
|
||||
"overrides": {
|
||||
"adm-zip": ">=0.6.0",
|
||||
|
|
|
|||
|
|
@ -1,211 +0,0 @@
|
|||
/**
|
||||
* Weight-aware partitioning for the cross-platform test matrix.
|
||||
*
|
||||
* WHY THIS EXISTS. `run-cross-platform.ts` used to hand vitest the whole file
|
||||
* list plus `--shard=i/n`, and vitest partitions by file COUNT. Runtime on this
|
||||
* suite is wildly uneven — measured on the Windows runner, `cli-e2e` is 621 s
|
||||
* and `worker-pool` 221 s, while most files are under a second — so a
|
||||
* count-split routinely put several of the heaviest suites on one shard. That
|
||||
* is #2449, and this file's sibling header has documented the symptom ("the
|
||||
* heaviest spawn suites can cluster on one shard") since the watchdog was first
|
||||
* raised from 15 to 20 minutes.
|
||||
*
|
||||
* It went from a latent hazard to a red matrix when three CHEAP files (the
|
||||
* `dist/` module-load closure guards: 448 ms, 53 ms, sub-second) were added to
|
||||
* `SPAWN_CLI`. They cost nothing to run, but a count-split re-partitions on
|
||||
* every insertion, and the reshuffle happened to land `cli-e2e` + `cli-limit-e2e`
|
||||
* + `analyze-heap-oom-e2e` together on shard 1/3 — 32 files against 26 and 29 —
|
||||
* which blew the 20-minute budget with four files still queued. Nothing about
|
||||
* the added files caused it; they were simply the perturbation.
|
||||
*
|
||||
* So the split is done HERE, by weight, and only the chosen shard's files are
|
||||
* handed to vitest. Two properties follow, and both are pinned in
|
||||
* `test/unit/cross-platform-shard.test.ts`:
|
||||
*
|
||||
* - the heaviest suites are spread across shards by construction, so the
|
||||
* busiest shard tracks the ideal rather than the luck of the sort order;
|
||||
* - adding or removing a CHEAP file cannot move a heavy one, so registering a
|
||||
* new platform-sensitive test is no longer a CI-stability gamble. That is the
|
||||
* property whose absence caused this.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Measured wall-clock on the WINDOWS runner (the slowest platform, so it is the
|
||||
* one that decides the budget), in seconds, from the last fully-green matrix run
|
||||
* plus the timed files of the run that failed.
|
||||
*
|
||||
* Only files heavy enough to matter are listed; everything else is carried by
|
||||
* {@link PER_FILE_OVERHEAD_SEC} alone. These are load-balancing hints, NOT
|
||||
* assertions — no test measures elapsed time against this table. Missing or
|
||||
* stale heavy entries can still overload a shard; refresh them from failed
|
||||
* CI logs and replay that profile in the partition tests. Deliberately not
|
||||
* auto-generated: a committed table is reviewable and works offline, and the
|
||||
* alternative (timing files at CI runtime to decide the split) would make the
|
||||
* partition depend on the very machine load it is trying to protect against.
|
||||
*/
|
||||
export const WINDOWS_WEIGHTS_SEC: Readonly<Record<string, number>> = {
|
||||
// Provisional Windows estimates, not Windows measurements: these native
|
||||
// inline-handler proofs took 7.25/6.86/6.52s on Node24 Linux. Apply the
|
||||
// runner's documented ~5x slowdown and round up; keep the watchdog unchanged.
|
||||
'test/integration/cfg/interproc-taint.test.ts': 40,
|
||||
'test/integration/function-local-identity.test.ts': 35,
|
||||
'test/integration/taint-explain.test.ts': 35,
|
||||
// Conservative estimates from Windows run 38144297115: persisted
|
||||
// rename cases include native indexing and take up to ~25s each. Keep
|
||||
// each language together while charging its complete fixture inventory.
|
||||
// Refresh underestimates from run 38697469283; Zig's higher measurement
|
||||
// comes from 38693066214. Stale weights packed too much native work ahead
|
||||
// of cli-e2e and exhausted its shard's 20-minute watchdog.
|
||||
'test/integration/rename-provider-c.test.ts': 202,
|
||||
'test/integration/rename-provider-cobol.test.ts': 75,
|
||||
'test/integration/rename-provider-cpp.test.ts': 250,
|
||||
'test/integration/rename-provider-csharp.test.ts': 266,
|
||||
'test/integration/rename-provider-dart.test.ts': 425,
|
||||
'test/integration/rename-provider-go.test.ts': 221,
|
||||
'test/integration/rename-provider-java.test.ts': 300,
|
||||
'test/integration/rename-provider-javascript.test.ts': 225,
|
||||
'test/integration/rename-provider-kotlin.test.ts': 300,
|
||||
'test/integration/rename-provider-objective-c.test.ts': 386,
|
||||
'test/integration/rename-provider-php.test.ts': 100,
|
||||
'test/integration/rename-provider-python.test.ts': 375,
|
||||
'test/integration/rename-provider-ruby.test.ts': 100,
|
||||
'test/integration/rename-provider-rust.test.ts': 75,
|
||||
'test/integration/rename-provider-swift.test.ts': 300,
|
||||
'test/integration/rename-provider-typescript.test.ts': 625,
|
||||
'test/integration/rename-provider-vue.test.ts': 200,
|
||||
'test/integration/rename-provider-zig.test.ts': 183,
|
||||
// Provisional: seven real-indexing child cases, using the existing ~25s
|
||||
// Windows persisted-fixture profile above. Replace with observed CI timing.
|
||||
'test/integration/rename-preview-plan.test.ts': 175,
|
||||
'test/integration/mcp-index-generation.test.ts': 201,
|
||||
// Windows run 39934695864 timed out after undercharged native suites;
|
||||
// 40172490240 supplies the query-symbol and dirty-recovery measurements.
|
||||
'test/integration/incremental-method-integrity.test.ts': 145,
|
||||
'test/integration/fts-repair-warm-session.test.ts': 62,
|
||||
'test/integration/query-symbol-integrity.test.ts': 34,
|
||||
'test/unit/incremental-dirty-recovery.test.ts': 128,
|
||||
// Re-measured after the analyze --watch e2e landed in #3072. The previous
|
||||
// 361 s entry undercharged this suite and left shard 1 close to the watchdog.
|
||||
'test/integration/cli-e2e.test.ts': 621,
|
||||
'test/integration/worker-pool.test.ts': 254,
|
||||
'test/unit/incremental-vector-extension-ordering.test.ts': 87,
|
||||
// Measured on Windows in run 34014266125 (#3190, 2026-09-06). These DB
|
||||
// suites landed together on shard 3: the old 180s estimate and missing
|
||||
// entries made a ~27-minute recorded load look like an ~12-minute shard.
|
||||
// Upstream speedups may reduce these figures; retaining conservative weights
|
||||
// keeps the expensive suites distributed without changing the watchdog.
|
||||
'test/unit/incremental-index-extension-dml-gate.test.ts': 414,
|
||||
// Re-measured on windows-latest run 34815870795 after vendored-first FTS
|
||||
// rewrote the HOME-layout e2e (373s) and skills-e2e grew to 542s. The old
|
||||
// 146s/444s entries packed both onto shard 2/3 and blew the 20-minute
|
||||
// watchdog with one file still queued.
|
||||
'test/integration/skills-e2e.test.ts': 550,
|
||||
'test/integration/fts-extension-e2e.test.ts': 380,
|
||||
'test/integration/skip-fts.test.ts': 110,
|
||||
// Provisional: twice the local native-suite duration, pending Windows CI.
|
||||
// These scheduling hints do not change test timeouts or pass/fail gates.
|
||||
'test/integration/embedding-identity-integrity.test.ts': 180,
|
||||
'test/integration/shared-store-embedding-reuse.test.ts': 140,
|
||||
// Provisional: twice the 88.42s local native run, pending Windows CI timing.
|
||||
'test/integration/embedding-input-isolation.test.ts': 180,
|
||||
'test/integration/analyze-wal-checkpoint-failure.test.ts': 86,
|
||||
// Windows run 39046807237 measured 308s before two cases reached recovery.
|
||||
// Allow the measured ~19s recovery cost for each corrected in-place case.
|
||||
'test/unit/incremental-write-integrity.test.ts': 360,
|
||||
'test/integration/cli-limit-e2e.test.ts': 75,
|
||||
'test/unit/hooks.test.ts': 26,
|
||||
'test/integration/analyze-heap-oom-e2e.test.ts': 23,
|
||||
'test/unit/git-utils.test.ts': 18,
|
||||
'test/integration/hooks-e2e.test.ts': 15,
|
||||
'test/integration/tree-sitter-languages.test.ts': 9,
|
||||
'test/unit/repo-manager.test.ts': 9,
|
||||
'test/unit/detect-changes-worktree.test.ts': 9,
|
||||
'test/integration/antigravity-hook-e2e.test.ts': 7,
|
||||
'test/unit/index-lock.test.ts': 5,
|
||||
'test/unit/setup.test.ts': 5,
|
||||
// ESTIMATE, not a measurement. This file asserts almost nothing; it READS —
|
||||
// one 4893-file pass over every tracked text file, plus an 830-file pass over
|
||||
// `src/`. Measured at 2.3 s and 0.3 s per pass on a virtualised and a local
|
||||
// Linux filesystem respectively, so the cost is entirely per-file open
|
||||
// latency, which is the term Windows inflates most (NTFS plus Defender on
|
||||
// every read). Scaled from the slower Linux figure to keep the split
|
||||
// conservative rather than let the 8 s PER_FILE_OVERHEAD floor under-charge
|
||||
// a file that touches more paths than anything else here. Replace with a real
|
||||
// figure after the first green Windows matrix run.
|
||||
'test/unit/source-control-bytes.test.ts': 15,
|
||||
};
|
||||
|
||||
/**
|
||||
* Fixed cost every file pays regardless of what it asserts: a pool worker start,
|
||||
* module graph evaluation, and (for most of this list) a native addon load.
|
||||
*
|
||||
* Added to EVERY file's weight, not just unmeasured ones, and that is the point.
|
||||
* Calibrated against the last green Windows matrix: its busiest shard ran 736 s
|
||||
* of wall clock over ~511 s of measured file time, so roughly 8 s per file is
|
||||
* unattributed setup. Without this term the balancer treats a light file as
|
||||
* nearly free and, having isolated the two monsters, piles every remaining file
|
||||
* onto the other shards — trading a runtime imbalance for a file-count one that
|
||||
* costs just as much. With it, the split balances runtime AND count together.
|
||||
*/
|
||||
const PER_FILE_OVERHEAD_SEC = 8;
|
||||
|
||||
/**
|
||||
* Scheduling weight for `file`: its measured runtime (0 if it was fast enough
|
||||
* that vitest printed no duration) plus the per-file floor above.
|
||||
*/
|
||||
export function weightOf(file: string): number {
|
||||
return (WINDOWS_WEIGHTS_SEC[file] ?? 0) + PER_FILE_OVERHEAD_SEC;
|
||||
}
|
||||
|
||||
/**
|
||||
* Partition `files` into `total` shards and return the 1-based `index` one.
|
||||
*
|
||||
* Longest-processing-time first: sort by weight descending, then repeatedly give
|
||||
* the next file to the lightest shard so far. LPT is the standard greedy for
|
||||
* multiprocessor scheduling and is guaranteed within 4/3 of optimal — far more
|
||||
* than enough here, where the goal is only "no shard gets two monsters".
|
||||
*
|
||||
* Ties break on the file path so the partition is DETERMINISTIC: every shard
|
||||
* computes the same split independently, on a different machine, with no
|
||||
* coordination — which is what lets each runner select its own slice.
|
||||
*
|
||||
* Returns files in the input list's original order, not weight order, so failure
|
||||
* output and reruns stay readable.
|
||||
*/
|
||||
export function shardFiles(
|
||||
files: readonly string[],
|
||||
index: number,
|
||||
total: number,
|
||||
): readonly string[] {
|
||||
if (!Number.isInteger(total) || total < 1) {
|
||||
throw new Error(`shard total must be a positive integer, got ${total}`);
|
||||
}
|
||||
if (!Number.isInteger(index) || index < 1 || index > total) {
|
||||
throw new Error(`shard index must be in 1..${total}, got ${index}`);
|
||||
}
|
||||
if (total === 1) return [...files];
|
||||
|
||||
const byWeightDesc = [...files].sort((a, b) => {
|
||||
const diff = weightOf(b) - weightOf(a);
|
||||
return diff !== 0 ? diff : a.localeCompare(b);
|
||||
});
|
||||
|
||||
const loads = Array.from({ length: total }, () => 0);
|
||||
const assigned = Array.from({ length: total }, () => new Set<string>());
|
||||
for (const file of byWeightDesc) {
|
||||
let lightest = 0;
|
||||
for (let i = 1; i < total; i++) {
|
||||
if (loads[i]! < loads[lightest]!) lightest = i;
|
||||
}
|
||||
assigned[lightest]!.add(file);
|
||||
loads[lightest]! += weightOf(file);
|
||||
}
|
||||
|
||||
const mine = assigned[index - 1]!;
|
||||
return files.filter((f) => mine.has(f));
|
||||
}
|
||||
|
||||
/** Total weight of a file set — the shard cost this balancer is minimising. */
|
||||
export function shardWeight(files: readonly string[]): number {
|
||||
return files.reduce((sum, f) => sum + weightOf(f), 0);
|
||||
}
|
||||
|
|
@ -267,6 +267,8 @@ const LBUG_NATIVE = [
|
|||
// process spawning, which behaves differently across platforms (shell
|
||||
// quoting, path resolution, signal handling)
|
||||
const SPAWN_CLI = [
|
||||
// Real shard/blob/JSON roundtrip: path identities and child invocation vary by OS.
|
||||
'test/unit/shard-timings-roundtrip.test.ts',
|
||||
'test/integration/cli-e2e.test.ts',
|
||||
'test/integration/cli-limit-e2e.test.ts',
|
||||
'test/integration/hooks-e2e.test.ts',
|
||||
|
|
@ -321,9 +323,8 @@ const SPAWN_CLI = [
|
|||
// earlier attempt to register them still turned the matrix red — not from
|
||||
// their own cost, but because vitest sharded by file COUNT, so inserting any
|
||||
// file re-partitioned the list and happened to cluster `cli-e2e` (621 s) with
|
||||
// `cli-limit-e2e` (75 s) on one shard. The split is weight-aware now
|
||||
// (`scripts/cross-platform-shard.ts`), so a cheap file can no longer move a
|
||||
// heavy one.
|
||||
// `cli-limit-e2e` (75 s) on one shard. PerfSequencer now uses recorded timings
|
||||
// through vitest-shard-balancer, shared by all shards in each workflow run.
|
||||
//
|
||||
// #2802: MCP startup must not eagerly load the analyze-only language
|
||||
// provider registry or the group contract extractors.
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ import path from 'path';
|
|||
import { fileURLToPath } from 'url';
|
||||
import { ALL_CROSS_PLATFORM } from './cross-platform-tests.js';
|
||||
import { parseShardArg } from './shard-arg.js';
|
||||
import { shardFiles, shardWeight } from './cross-platform-shard.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
|
|
@ -29,14 +28,8 @@ if (missing.length > 0) {
|
|||
process.exit(1);
|
||||
}
|
||||
|
||||
// Optional sharding (CI): `--shard=<i>/<n>` splits the fixed file list across
|
||||
// parallel matrix shards. The split is computed HERE, by measured weight, and
|
||||
// only this shard's files are handed to vitest — it is NOT passed through,
|
||||
// because vitest partitions by file COUNT and this suite's runtimes span three
|
||||
// orders of magnitude (see cross-platform-shard.ts). The
|
||||
// Windows runner is ~5x slower than macOS/Linux on this spawn-heavy suite (~50
|
||||
// CLI/worker process spawns), so a single shard was creeping past the watchdog
|
||||
// below; sharding keeps each runner well under it (see ci-tests.yml matrix).
|
||||
// Pass the complete subset and --shard to the configured timing sequencer.
|
||||
// Every runner reads the same snapshot, selected once by CI's shard-plan job.
|
||||
// Fail loud on a malformed --shard arg (mirrors the missing-files check above):
|
||||
// a silently-dropped shard flag would run the full unsharded suite and re-trip
|
||||
// the watchdog. Kept outside the execFileSync try/catch below so the message
|
||||
|
|
@ -49,13 +42,7 @@ try {
|
|||
process.exit(1);
|
||||
}
|
||||
|
||||
// Per-shard watchdog, default 15 min. Sharding splits the file list by COUNT, not
|
||||
// runtime, so the heaviest spawn suites can cluster on one shard — what this
|
||||
// bounds is the *busiest* shard, not an even 1/n of wall-clock. The busiest
|
||||
// Windows shard has grown to the default (14m57s on the v1.6.10-rc.19 green
|
||||
// run, one observed timeout since — #2449), so CI raises the budget to 20
|
||||
// minutes via GITNEXUS_CROSS_PLATFORM_TIMEOUT_MINUTES; the default stays 15
|
||||
// for local runs.
|
||||
// Preserve the 15-minute local watchdog and the 20-minute CI override.
|
||||
const DEFAULT_TIMEOUT_MIN = 15;
|
||||
const timeoutMinutes = Number.parseInt(
|
||||
process.env.GITNEXUS_CROSS_PLATFORM_TIMEOUT_MINUTES ?? String(DEFAULT_TIMEOUT_MIN),
|
||||
|
|
@ -66,17 +53,9 @@ const timeoutMs =
|
|||
? timeoutMinutes * 60 * 1000
|
||||
: DEFAULT_TIMEOUT_MIN * 60 * 1000;
|
||||
|
||||
// Resolve the shard to an explicit file list. `--shard=i/n` is consumed here,
|
||||
// never forwarded: forwarding it as well would re-partition this slice a second
|
||||
// time and silently drop most of it.
|
||||
const shardParts = shardArg?.replace('--shard=', '').split('/');
|
||||
const shardIndex = shardParts ? Number(shardParts[0]) : 1;
|
||||
const shardTotal = shardParts ? Number(shardParts[1]) : 1;
|
||||
const files = shardFiles(ALL_CROSS_PLATFORM, shardIndex, shardTotal);
|
||||
|
||||
console.log(
|
||||
`Running ${files.length} of ${ALL_CROSS_PLATFORM.length} platform-sensitive tests` +
|
||||
`${shardArg ? ` (shard ${shardIndex}/${shardTotal}, ~${shardWeight(files)}s measured weight)` : ''}...\n`,
|
||||
`Running platform-sensitive test suite (${ALL_CROSS_PLATFORM.length} files total)` +
|
||||
`${shardArg ? ` (${shardArg}, balanced by timing history)` : ''}...\n`,
|
||||
);
|
||||
|
||||
const startedAt = Date.now();
|
||||
|
|
@ -88,16 +67,22 @@ const reportArgs = reportName
|
|||
? [
|
||||
'--reporter=default',
|
||||
'--reporter=./scripts/execution-reporter.ts',
|
||||
`--outputFile=${reportName}`,
|
||||
'--reporter=blob',
|
||||
`--outputFile.json=${reportName}`,
|
||||
`--outputFile.blob=.vitest/blob/${reportName}`,
|
||||
]
|
||||
: [];
|
||||
try {
|
||||
execFileSync('npx', ['vitest', 'run', ...files, ...reportArgs], {
|
||||
cwd: ROOT,
|
||||
stdio: 'inherit',
|
||||
timeout: timeoutMs,
|
||||
shell: true,
|
||||
});
|
||||
execFileSync(
|
||||
'npx',
|
||||
['vitest', 'run', ...ALL_CROSS_PLATFORM, ...(shardArg ? [shardArg] : []), ...reportArgs],
|
||||
{
|
||||
cwd: ROOT,
|
||||
stdio: 'inherit',
|
||||
timeout: timeoutMs,
|
||||
shell: true,
|
||||
},
|
||||
);
|
||||
} catch (err) {
|
||||
// execFileSync sets `killed`/`signal` when the watchdog above kills vitest.
|
||||
const e = err as {
|
||||
|
|
|
|||
57
gitnexus/scripts/shard-timings-reporter.ts
Normal file
57
gitnexus/scripts/shard-timings-reporter.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import path from 'node:path';
|
||||
import type { TestModule, TestRunEndReason, Vitest } from 'vitest/node';
|
||||
import { TimingsReporter } from 'vitest-shard-balancer';
|
||||
import { ALL_CROSS_PLATFORM } from './cross-platform-tests.js';
|
||||
import { shardTimingKey, shardTimingsFile } from './shard-timings.js';
|
||||
|
||||
export default class ShardTimingsReporter extends TimingsReporter {
|
||||
private context!: Vitest;
|
||||
|
||||
constructor() {
|
||||
// The package's whole-config check cannot describe our native subset.
|
||||
// Check the exact cohort below, before excluding skipped modules from learning.
|
||||
super({ timingsFile: shardTimingsFile(), requireAllFiles: false });
|
||||
}
|
||||
|
||||
override onInit(context: Vitest): void {
|
||||
this.context = context;
|
||||
super.onInit(context);
|
||||
}
|
||||
|
||||
override async onTestRunEnd(
|
||||
modules: readonly TestModule[],
|
||||
errors: readonly unknown[] = [],
|
||||
reason: TestRunEndReason = 'interrupted',
|
||||
): Promise<void> {
|
||||
if (!this.context.config.mergeReports) return;
|
||||
if (errors.length || reason !== 'passed') {
|
||||
throw new Error('Refusing to learn shard timings from an unsuccessful test run');
|
||||
}
|
||||
const root = this.context.config.root;
|
||||
const native = (process.env.GITNEXUS_SHARD_PROFILE ?? 'coverage') !== 'coverage';
|
||||
const nativeFiles = new Set<string>(ALL_CROSS_PLATFORM);
|
||||
const specs = (await this.context.globTestSpecifications()).filter(
|
||||
(spec) =>
|
||||
!native || nativeFiles.has(path.relative(root, spec.moduleId).split(path.sep).join('/')),
|
||||
);
|
||||
const expected = new Set(
|
||||
specs.map((spec) => shardTimingKey(root, spec.moduleId, spec.project.name)),
|
||||
);
|
||||
const actual = new Set<string>();
|
||||
for (const module of modules) {
|
||||
const key = shardTimingKey(root, module.moduleId, module.project.name);
|
||||
if (actual.has(key) || !expected.has(key)) {
|
||||
throw new Error(`Duplicate or unexpected test module in shard reports: ${key}`);
|
||||
}
|
||||
actual.add(key);
|
||||
}
|
||||
const missing = [...expected].filter((key) => !actual.has(key));
|
||||
if (!actual.size || missing.length) {
|
||||
throw new Error(
|
||||
`Incomplete shard reports; missing ${missing.length} test modules: ${missing.join(', ')}`,
|
||||
);
|
||||
}
|
||||
// A platform-specific skip must not turn a previously expensive file into zero work.
|
||||
await super.onTestRunEnd(modules.filter((module) => module.state() === 'passed'));
|
||||
}
|
||||
}
|
||||
16
gitnexus/scripts/shard-timings.ts
Normal file
16
gitnexus/scripts/shard-timings.ts
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
import path from 'node:path';
|
||||
|
||||
/** Separate native costs from coverage and from the other operating system. */
|
||||
export function shardTimingsFile(): string {
|
||||
const profile = process.env.GITNEXUS_SHARD_PROFILE ?? 'coverage';
|
||||
if (!['coverage', 'windows-latest', 'macos-latest'].includes(profile)) {
|
||||
throw new Error(`Unknown shard timing profile: ${profile}`);
|
||||
}
|
||||
return `.vitest/timings/${profile}.json`;
|
||||
}
|
||||
|
||||
/** Matches vitest-shard-balancer's project-qualified timing identity. */
|
||||
export function shardTimingKey(root: string, moduleId: string, projectName: string): string {
|
||||
const relative = path.relative(root, moduleId).split(path.sep).join('/');
|
||||
return projectName ? `${projectName}:${relative}` : relative;
|
||||
}
|
||||
|
|
@ -1,35 +1,52 @@
|
|||
import { statSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { BaseSequencer, type TestSpecification } from 'vitest/node';
|
||||
import { assignShards, specWeight } from './shard-balance.js';
|
||||
import { partition, readTimings } from 'vitest-shard-balancer';
|
||||
import { shardTimingKey, shardTimingsFile } from '../../scripts/shard-timings.js';
|
||||
|
||||
/**
|
||||
* Cost-balanced shard sequencer (wired via `sequence.sequencer` in
|
||||
* vitest.config.ts).
|
||||
*
|
||||
* Overrides only `shard()` — `sort()` keeps the base behaviour so the projects'
|
||||
* `groupOrder` and duration-cache ordering are untouched. Instead of vitest's
|
||||
* default hash split (balanced by file COUNT, which piled the spawn-heavy suites
|
||||
* onto one runner), it balances by estimated WORK (see `specWeight`): the
|
||||
* fileParallelism:false spawn-heavy files are spread evenly across shards, so the
|
||||
* slowest shard's wall-clock drops and no single runner carries all the
|
||||
* contention. The partition stays complete and disjoint (see `assignShards`).
|
||||
* Uses the package's duration partitioner with the frozen CI timing snapshot.
|
||||
* Only shard() changes; BaseSequencer keeps groupOrder and local sorting.
|
||||
* On a cold start, spread sequential/native work using a generic size estimate
|
||||
* instead of the count split that previously clustered the slowest suites.
|
||||
*/
|
||||
export default class PerfSequencer extends BaseSequencer {
|
||||
override async shard(specs: TestSpecification[]): Promise<TestSpecification[]> {
|
||||
const shard = this.ctx.config.shard;
|
||||
if (!shard) return specs;
|
||||
const groups = assignShards(
|
||||
specs,
|
||||
shard.count,
|
||||
(spec) => {
|
||||
const config = spec.project.config;
|
||||
const fileParallelism =
|
||||
'fileParallelism' in config && typeof config.fileParallelism === 'boolean'
|
||||
? config.fileParallelism
|
||||
: undefined;
|
||||
return specWeight({ moduleId: spec.moduleId, project: { config: { fileParallelism } } });
|
||||
},
|
||||
(spec) => spec.moduleId,
|
||||
const root = this.ctx.config.root;
|
||||
const keyed = new Map(
|
||||
specs.map((spec) => [shardTimingKey(root, spec.moduleId, spec.project.name), spec]),
|
||||
);
|
||||
if (keyed.size !== specs.length) throw new Error('Duplicate test specification in shard input');
|
||||
const keys = [...keyed.keys()];
|
||||
const timings = readTimings(path.resolve(root, shardTimingsFile()));
|
||||
let groups = partition(keys, timings, shard.count);
|
||||
if (!groups) {
|
||||
const estimates = Object.fromEntries(
|
||||
[...keyed].map(([key, spec]) => {
|
||||
let sizeKb = 0;
|
||||
try {
|
||||
sizeKb = statSync(spec.moduleId).size / 1024;
|
||||
} catch {
|
||||
/* Virtual test module. */
|
||||
}
|
||||
const config = spec.project.config;
|
||||
const sequential = 'fileParallelism' in config && config.fileParallelism === false;
|
||||
// Keep all cold weights on the same scale: 1,000 points represents
|
||||
// 20 seconds of sequential work. A millisecond value here would
|
||||
// dwarf file size and pile the parallel cohort onto fewer shards.
|
||||
return [key, (sequential ? 1_000 : 1) + sizeKb];
|
||||
}),
|
||||
);
|
||||
groups = partition(keys, estimates, shard.count);
|
||||
}
|
||||
const selected = new Set(groups?.[shard.index - 1] ?? []);
|
||||
return specs.filter((spec) =>
|
||||
selected.has(shardTimingKey(root, spec.moduleId, spec.project.name)),
|
||||
);
|
||||
return groups[shard.index - 1] ?? [];
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,250 +0,0 @@
|
|||
/**
|
||||
* Pins the weight-aware split behind the cross-platform matrix (#2449).
|
||||
*
|
||||
* The regression this guards is specific and was expensive: three CHEAP files
|
||||
* were registered in `SPAWN_CLI`, vitest re-partitioned the list by file COUNT,
|
||||
* and the reshuffle clustered `cli-e2e` (now 621 s on Windows) with `cli-limit-e2e`
|
||||
* (75 s) and `analyze-heap-oom-e2e` (23 s) on one shard, which then blew the
|
||||
* 20-minute watchdog. The added files cost nothing; the COUNT-split did it.
|
||||
*
|
||||
* So the load-bearing case here is not "the split is even" — it is
|
||||
* "adding a cheap file does not move a heavy one". A partition that merely
|
||||
* balanced totals could still reshuffle everything on every insertion and would
|
||||
* reproduce the outage exactly.
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
shardFiles,
|
||||
shardWeight,
|
||||
weightOf,
|
||||
WINDOWS_WEIGHTS_SEC,
|
||||
} from '../../scripts/cross-platform-shard.js';
|
||||
import {
|
||||
ALL_CROSS_PLATFORM,
|
||||
LBUG_NATIVE,
|
||||
WORKER_THREADS,
|
||||
} from '../../scripts/cross-platform-tests.js';
|
||||
|
||||
// Replay the actual CI partition, including shard-count changes as the suite grows.
|
||||
const workflow = readFileSync(
|
||||
new URL('../../../.github/workflows/ci-tests.yml', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
const SHARD_TOTAL = Number(workflow.match(/^\s+TOTAL=(\d+)\b/m)?.[1]);
|
||||
|
||||
/** Every shard of a split, as file lists. */
|
||||
const allShards = (files: readonly string[], total: number): readonly (readonly string[])[] =>
|
||||
Array.from({ length: total }, (_unused, i) => shardFiles(files, i + 1, total));
|
||||
|
||||
describe('cross-platform shard partition', () => {
|
||||
it('selects the native inline-handler proofs exactly once across the CI shards', () => {
|
||||
const proofs = [
|
||||
[LBUG_NATIVE, 'test/integration/taint-explain.test.ts'],
|
||||
[WORKER_THREADS, 'test/integration/cfg/interproc-taint.test.ts'],
|
||||
[WORKER_THREADS, 'test/integration/function-local-identity.test.ts'],
|
||||
] as const;
|
||||
const selected = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).flat();
|
||||
for (const [category, file] of proofs) {
|
||||
expect(category.filter((candidate) => candidate === file)).toEqual([file]);
|
||||
expect(selected.filter((candidate) => candidate === file)).toEqual([file]);
|
||||
}
|
||||
});
|
||||
|
||||
it('selects both real-worker local-import proofs exactly once across the CI shards', () => {
|
||||
const proofs = [
|
||||
'test/integration/local-import-worker-parity.test.ts',
|
||||
'test/integration/resolvers/local-import-persistence.test.ts',
|
||||
];
|
||||
const selected = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).flat();
|
||||
for (const file of proofs) {
|
||||
expect(WORKER_THREADS.filter((candidate) => candidate === file)).toEqual([file]);
|
||||
expect(ALL_CROSS_PLATFORM.filter((candidate) => candidate === file)).toEqual([file]);
|
||||
expect(selected.filter((candidate) => candidate === file)).toEqual([file]);
|
||||
}
|
||||
});
|
||||
|
||||
it('does not recreate the overloaded Windows shard from the #3190 CI run', () => {
|
||||
// Run 34014266125: these serialized DB suites were missing or undercharged
|
||||
// in the scheduling table. Keep the observed profile independent of the
|
||||
// table so deleting a weight cannot make this regression pass again.
|
||||
const observed: Readonly<Record<string, number>> = {
|
||||
'test/integration/skills-e2e.test.ts': 550,
|
||||
'test/unit/incremental-index-extension-dml-gate.test.ts': 414,
|
||||
'test/integration/fts-extension-e2e.test.ts': 380,
|
||||
'test/integration/analyze-wal-checkpoint-failure.test.ts': 86,
|
||||
'test/integration/skip-fts.test.ts': 110,
|
||||
};
|
||||
const floor = weightOf('test/unmeasured.test.ts');
|
||||
const loads = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).map((files) =>
|
||||
files.reduce((sum, file) => sum + Math.max(weightOf(file), (observed[file] ?? 0) + floor), 0),
|
||||
);
|
||||
// This is a deterministic replay of recorded weights, not a timing test.
|
||||
// The runner's existing watchdog remains 20 minutes.
|
||||
expect(Math.max(...loads)).toBeLessThan(20 * 60);
|
||||
const shards = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL);
|
||||
const heavyweightLocations = [
|
||||
'test/integration/cli-e2e.test.ts',
|
||||
'test/integration/skills-e2e.test.ts',
|
||||
'test/unit/incremental-index-extension-dml-gate.test.ts',
|
||||
].map((file) => shards.findIndex((files) => files.includes(file)));
|
||||
expect(heavyweightLocations).not.toContain(-1);
|
||||
expect(new Set(heavyweightLocations).size).toBe(heavyweightLocations.length);
|
||||
expect(
|
||||
shards.filter(
|
||||
(s) =>
|
||||
s.includes('test/integration/skills-e2e.test.ts') &&
|
||||
s.includes('test/integration/fts-extension-e2e.test.ts'),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('covers every file exactly once, with no overlap between shards', () => {
|
||||
const shards = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL);
|
||||
const seen = shards.flatMap((s) => [...s]);
|
||||
|
||||
expect(seen.slice().sort()).toEqual([...ALL_CROSS_PLATFORM].sort());
|
||||
expect(new Set(seen).size).toBe(ALL_CROSS_PLATFORM.length);
|
||||
});
|
||||
|
||||
it('leaves watchdog headroom for the recorded October Windows workload', () => {
|
||||
// Run 38697469283 exhausted the 20-minute watchdog before cli-e2e
|
||||
// finished. Its preceding serialized suites were undercharged or absent
|
||||
// from the weight table. The CLI and Zig timings below come from the
|
||||
// preceding successful run 38693066214; the other timings are from the
|
||||
// failing run. Keep this replay independent of the scheduling table.
|
||||
const observed: Readonly<Record<string, number>> = {
|
||||
'test/integration/cli-e2e.test.ts': 563,
|
||||
'test/integration/mcp-index-generation.test.ts': 201,
|
||||
'test/integration/rename-provider-objective-c.test.ts': 285,
|
||||
'test/integration/rename-provider-zig.test.ts': 183,
|
||||
'test/integration/rename-provider-go.test.ts': 221,
|
||||
'test/integration/worker-pool.test.ts': 254,
|
||||
'test/integration/rename-provider-swift.test.ts': 300,
|
||||
'test/integration/rename-provider-c.test.ts': 202,
|
||||
'test/unit/incremental-dirty-recovery.test.ts': 76,
|
||||
};
|
||||
const floor = weightOf('test/unmeasured.test.ts');
|
||||
const loads = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).map((files) =>
|
||||
files.reduce((sum, file) => sum + Math.max(weightOf(file), (observed[file] ?? 0) + floor), 0),
|
||||
);
|
||||
// Leave at least three minutes for hosted-runner variation. Merely
|
||||
// fitting under 20 minutes left the previous partition prone to timeout.
|
||||
expect(Math.max(...loads)).toBeLessThanOrEqual(17 * 60);
|
||||
});
|
||||
|
||||
it('leaves watchdog headroom for the October 23 native-suite timeout', () => {
|
||||
// Windows run 39934695864: shard 10 timed out with Kotlin still pending.
|
||||
// Use its completed suite times, plus the October 25 green run 40172490240
|
||||
// for query-symbol-integrity and incremental-dirty-recovery. These measured
|
||||
// costs must remain independent of the scheduling hints being tested.
|
||||
const observed: Readonly<Record<string, number>> = {
|
||||
'test/integration/rename-provider-objective-c.test.ts': 385.174,
|
||||
'test/integration/rename-provider-csharp.test.ts': 265.99,
|
||||
'test/integration/incremental-method-integrity.test.ts': 144.974,
|
||||
'test/integration/fts-repair-warm-session.test.ts': 61.015,
|
||||
'test/integration/query-symbol-integrity.test.ts': 33.892,
|
||||
'test/unit/incremental-dirty-recovery.test.ts': 127.8,
|
||||
};
|
||||
const floor = weightOf('test/unmeasured.test.ts');
|
||||
const loads = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).map((files) =>
|
||||
files.reduce((sum, file) => sum + Math.max(weightOf(file), (observed[file] ?? 0) + floor), 0),
|
||||
);
|
||||
expect(Math.max(...loads)).toBeLessThanOrEqual(17 * 60);
|
||||
});
|
||||
|
||||
it('keeps each shard within a shard of the ideal weight', () => {
|
||||
const shards = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL);
|
||||
const weights = shards.map(shardWeight);
|
||||
const ideal = shardWeight(ALL_CROSS_PLATFORM) / SHARD_TOTAL;
|
||||
|
||||
// LPT's guarantee is 4/3 of optimal, and optimal is at least the ideal
|
||||
// average. A hard 1.34x ceiling on the busiest shard is what keeps the
|
||||
// matrix inside its watchdog no matter how the list is edited.
|
||||
expect(Math.max(...weights)).toBeLessThanOrEqual(ideal * 1.34);
|
||||
});
|
||||
|
||||
it('keeps the CLI and worker-pool suites on different shards', () => {
|
||||
// The exact shape of the outage: cli-e2e and worker-pool are 621 s and
|
||||
// 222 s, so together they are most of a shard's budget before anything else
|
||||
// is scheduled.
|
||||
const shards = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL);
|
||||
const withBoth = shards.filter(
|
||||
(s) =>
|
||||
s.includes('test/integration/cli-e2e.test.ts') &&
|
||||
s.includes('test/integration/worker-pool.test.ts'),
|
||||
);
|
||||
|
||||
expect(withBoth).toEqual([]);
|
||||
});
|
||||
|
||||
it('does not move a heavy file when a cheap file is added — the #2449 regression', () => {
|
||||
const heavy = Object.keys(WINDOWS_WEIGHTS_SEC);
|
||||
const placementOf = (files: readonly string[]): ReadonlyMap<string, number> => {
|
||||
const shards = allShards(files, SHARD_TOTAL);
|
||||
return new Map(
|
||||
heavy
|
||||
.map((f) => [f, shards.findIndex((s) => s.includes(f))] as const)
|
||||
.filter(([, i]) => i >= 0),
|
||||
);
|
||||
};
|
||||
|
||||
const before = placementOf(ALL_CROSS_PLATFORM);
|
||||
|
||||
// The inserted names sort EARLY, and there is a case that is NOT a multiple
|
||||
// of the shard count. Both details are load-bearing, and getting them wrong
|
||||
// made earlier versions of this test vacuous:
|
||||
// - names that sort last cannot disturb anything under any scheme;
|
||||
// - adding exactly `total` files leaves an equal-weight round-robin in the
|
||||
// same rotation, so a count-split would pass too.
|
||||
// Under the real weighted split, heavy files are scheduled before every
|
||||
// light one, so no number of cheap insertions can move them.
|
||||
const afterOne = placementOf([...ALL_CROSS_PLATFORM, 'test/aaa-new-cheap-a.test.ts']);
|
||||
const afterTwo = placementOf([
|
||||
...ALL_CROSS_PLATFORM,
|
||||
'test/aaa-new-cheap-a.test.ts',
|
||||
'test/aaa-new-cheap-b.test.ts',
|
||||
]);
|
||||
|
||||
expect(Object.fromEntries(afterOne)).toMatchObject(Object.fromEntries(before));
|
||||
expect(Object.fromEntries(afterTwo)).toMatchObject(Object.fromEntries(before));
|
||||
});
|
||||
|
||||
it('is deterministic, so every runner computes the same split independently', () => {
|
||||
// Each matrix job resolves its own slice on its own machine with no shared
|
||||
// state, so an unstable sort would silently drop or duplicate files.
|
||||
const once = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).map((s) => [...s]);
|
||||
const twice = allShards([...ALL_CROSS_PLATFORM].reverse(), SHARD_TOTAL).map((s) =>
|
||||
[...s].sort(),
|
||||
);
|
||||
|
||||
expect(twice).toEqual(once.map((s) => [...s].sort()));
|
||||
});
|
||||
|
||||
it('returns every file for a single-shard run, and rejects an out-of-range shard', () => {
|
||||
expect(shardFiles(ALL_CROSS_PLATFORM, 1, 1)).toEqual([...ALL_CROSS_PLATFORM]);
|
||||
expect(() => shardFiles(ALL_CROSS_PLATFORM, 0, 3)).toThrow(/shard index/);
|
||||
expect(() => shardFiles(ALL_CROSS_PLATFORM, 4, 3)).toThrow(/shard index/);
|
||||
expect(() => shardFiles(ALL_CROSS_PLATFORM, 1, 0)).toThrow(/shard total/);
|
||||
});
|
||||
|
||||
it('charges every file the per-file floor, so light files are never free', () => {
|
||||
// Without this, the balancer isolates the monsters and then piles all the
|
||||
// light files onto the remaining shards — a count imbalance that costs just
|
||||
// as much wall clock as the runtime one it just fixed.
|
||||
expect(weightOf('test/unit/zzz-does-not-exist.test.ts')).toBeGreaterThan(0);
|
||||
expect(weightOf('test/integration/cli-e2e.test.ts')).toBeGreaterThan(
|
||||
WINDOWS_WEIGHTS_SEC['test/integration/cli-e2e.test.ts'] ?? 0,
|
||||
);
|
||||
});
|
||||
|
||||
it('weights only files that are actually registered', () => {
|
||||
// A weight entry for a file no longer in the list is dead config that the
|
||||
// balancer silently ignores; catching it here keeps the table honest.
|
||||
const registered = new Set(ALL_CROSS_PLATFORM);
|
||||
const stale = Object.keys(WINDOWS_WEIGHTS_SEC).filter((f) => !registered.has(f));
|
||||
|
||||
expect(stale).toEqual([]);
|
||||
});
|
||||
});
|
||||
113
gitnexus/test/unit/perf-sequencer.test.ts
Normal file
113
gitnexus/test/unit/perf-sequencer.test.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { TestSpecification, Vitest } from 'vitest/node';
|
||||
import PerfSequencer from '../helpers/perf-sequencer.js';
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function fixture() {
|
||||
const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-shards-'));
|
||||
roots.push(root);
|
||||
mkdirSync(path.join(root, '.vitest/timings'), { recursive: true });
|
||||
const specs = ['a', 'b', 'c', 'd'].map((name) => {
|
||||
const moduleId = path.join(root, `${name}.test.ts`);
|
||||
writeFileSync(moduleId, '// same size\n');
|
||||
return {
|
||||
moduleId,
|
||||
project: { name: 'native', config: { fileParallelism: false } },
|
||||
} as unknown as TestSpecification;
|
||||
});
|
||||
const history = (profile: string, weights: number[]) =>
|
||||
writeFileSync(
|
||||
path.join(root, '.vitest/timings', `${profile}.json`),
|
||||
JSON.stringify(
|
||||
Object.fromEntries(
|
||||
specs.map((spec, i) => [`native:${path.basename(spec.moduleId)}`, weights[i]]),
|
||||
),
|
||||
),
|
||||
);
|
||||
const split = async () =>
|
||||
Promise.all(
|
||||
[1, 2].map(async (index) => {
|
||||
const ctx = { config: { root, shard: { index, count: 2 } } } as Vitest;
|
||||
return new PerfSequencer(ctx).shard(specs);
|
||||
}),
|
||||
);
|
||||
return { root, specs, history, split };
|
||||
}
|
||||
|
||||
describe('timing-based CI sharding', () => {
|
||||
it('learns from measured durations instead of putting both slow files on one shard', async () => {
|
||||
const { specs, history, split } = fixture();
|
||||
history('coverage', [10000, 10, 9000, 10]);
|
||||
const bins = await split();
|
||||
expect(bins.some((bin) => bin.includes(specs[0]) && bin.includes(specs[2]))).toBe(false);
|
||||
expect(
|
||||
bins
|
||||
.flat()
|
||||
.map((spec) => spec.moduleId)
|
||||
.sort(),
|
||||
).toEqual(specs.map((spec) => spec.moduleId).sort());
|
||||
});
|
||||
|
||||
it('keeps platform histories separate and gives independent runners identical partitions', async () => {
|
||||
const { specs, history, split } = fixture();
|
||||
history('windows-latest', [10000, 10, 9000, 10]);
|
||||
history('macos-latest', [9000, 10000, 10, 10]);
|
||||
vi.stubEnv('GITNEXUS_SHARD_PROFILE', 'windows-latest');
|
||||
const windows = await split();
|
||||
expect(await split()).toEqual(windows);
|
||||
expect(windows.some((bin) => bin.includes(specs[0]) && bin.includes(specs[2]))).toBe(false);
|
||||
vi.stubEnv('GITNEXUS_SHARD_PROFILE', 'macos-latest');
|
||||
const mac = await split();
|
||||
expect(mac.some((bin) => bin.includes(specs[0]) && bin.includes(specs[1]))).toBe(false);
|
||||
expect(mac).not.toEqual(windows);
|
||||
});
|
||||
|
||||
it('still schedules new files and cold starts as complete disjoint partitions', async () => {
|
||||
const { specs, history, split } = fixture();
|
||||
const cold = await split();
|
||||
expect(cold.map((bin) => bin.length)).toEqual([2, 2]);
|
||||
history('coverage', [10000]);
|
||||
const warm = await split();
|
||||
expect(new Set(warm.flat()).size).toBe(specs.length);
|
||||
expect(warm.flat().length).toBe(specs.length);
|
||||
});
|
||||
|
||||
it('spreads the parallel cohort across all cold shards when sequential counts differ', async () => {
|
||||
const { root } = fixture();
|
||||
const specs = Array.from({ length: 1414 }, (_, i) => {
|
||||
const moduleId = path.join(root, `${i}.test.ts`);
|
||||
writeFileSync(moduleId, ' '.repeat(10 * 1024));
|
||||
const sequential = i < 94;
|
||||
return {
|
||||
moduleId,
|
||||
project: {
|
||||
name: sequential ? 'native' : 'default',
|
||||
config: { fileParallelism: !sequential },
|
||||
},
|
||||
} as unknown as TestSpecification;
|
||||
});
|
||||
const bins = await Promise.all(
|
||||
Array.from({ length: 8 }, (_, i) => {
|
||||
const ctx = { config: { root, shard: { index: i + 1, count: 8 } } } as Vitest;
|
||||
return new PerfSequencer(ctx).shard(specs);
|
||||
}),
|
||||
);
|
||||
const parallelCounts = bins.map(
|
||||
(bin) => bin.filter((spec) => spec.project.name === 'default').length,
|
||||
);
|
||||
// A one-file difference in sequential work must not concentrate the entire
|
||||
// parallel cohort on two runners (660 files each in the failing CI run).
|
||||
expect(Math.min(...parallelCounts)).toBeGreaterThan(100);
|
||||
expect(Math.max(...parallelCounts)).toBeLessThan(250);
|
||||
expect(new Set(bins.flat()).size).toBe(specs.length);
|
||||
expect(bins.flat()).toHaveLength(specs.length);
|
||||
});
|
||||
});
|
||||
87
gitnexus/test/unit/shard-timings-reporter.test.ts
Normal file
87
gitnexus/test/unit/shard-timings-reporter.test.ts
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { TestModule, Vitest } from 'vitest/node';
|
||||
import ShardTimingsReporter from '../../scripts/shard-timings-reporter.js';
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function fixture() {
|
||||
const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-timings-'));
|
||||
roots.push(root);
|
||||
const file = path.join(root, '.vitest/timings/coverage.json');
|
||||
mkdirSync(path.dirname(file), { recursive: true });
|
||||
writeFileSync(file, JSON.stringify({ 'native:a.test.ts': 1000, 'native:b.test.ts': 500 }));
|
||||
const modules = ['a', 'b'].map((name) => ({
|
||||
moduleId: path.join(root, `${name}.test.ts`),
|
||||
project: { name: 'native' },
|
||||
state: () => 'passed',
|
||||
diagnostic: () => ({
|
||||
environmentSetupDuration: 10,
|
||||
prepareDuration: 20,
|
||||
setupDuration: 30,
|
||||
collectDuration: 40,
|
||||
duration: 100,
|
||||
}),
|
||||
})) as unknown as TestModule[];
|
||||
const reporter = new ShardTimingsReporter();
|
||||
reporter.onInit({
|
||||
config: { root, mergeReports: 'blobs' },
|
||||
globTestSpecifications: async () => modules,
|
||||
} as unknown as Vitest);
|
||||
return { file, modules, reporter };
|
||||
}
|
||||
|
||||
describe('learning shard timings', () => {
|
||||
it('includes setup and collection, smooths measurements, and retains skipped-file history', async () => {
|
||||
const { file, modules, reporter } = fixture();
|
||||
modules[1].state = () => 'skipped';
|
||||
await reporter.onTestRunEnd(modules, [], 'passed');
|
||||
expect(JSON.parse(readFileSync(file, 'utf8'))).toEqual({
|
||||
'native:a.test.ts': 600,
|
||||
'native:b.test.ts': 500,
|
||||
});
|
||||
});
|
||||
|
||||
it.each(['missing', 'duplicate', 'unexpected'] as const)(
|
||||
'rejects %s modules without replacing history',
|
||||
async (kind) => {
|
||||
const { file, modules, reporter } = fixture();
|
||||
const before = readFileSync(file, 'utf8');
|
||||
const inputs =
|
||||
kind === 'missing'
|
||||
? modules.slice(0, 1)
|
||||
: [
|
||||
...modules,
|
||||
kind === 'duplicate'
|
||||
? modules[0]
|
||||
: ({ ...modules[0], moduleId: '/unexpected.test.ts' } as unknown as TestModule),
|
||||
];
|
||||
await expect(reporter.onTestRunEnd(inputs, [], 'passed')).rejects.toThrow(
|
||||
/Incomplete|Duplicate or unexpected/,
|
||||
);
|
||||
expect(readFileSync(file, 'utf8')).toBe(before);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['failed', 'interrupted'] as const)('does not learn from a %s run', async (reason) => {
|
||||
const { file, modules, reporter } = fixture();
|
||||
const before = readFileSync(file, 'utf8');
|
||||
await expect(reporter.onTestRunEnd(modules, [], reason)).rejects.toThrow(/unsuccessful/);
|
||||
expect(readFileSync(file, 'utf8')).toBe(before);
|
||||
});
|
||||
|
||||
it('does not learn when the runner reports an unhandled error', async () => {
|
||||
const { file, modules, reporter } = fixture();
|
||||
const before = readFileSync(file, 'utf8');
|
||||
await expect(
|
||||
reporter.onTestRunEnd(modules, [new Error('worker crashed')], 'passed'),
|
||||
).rejects.toThrow(/unsuccessful/);
|
||||
expect(readFileSync(file, 'utf8')).toBe(before);
|
||||
});
|
||||
});
|
||||
96
gitnexus/test/unit/shard-timings-roundtrip.test.ts
Normal file
96
gitnexus/test/unit/shard-timings-roundtrip.test.ts
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { expect, it } from 'vitest';
|
||||
|
||||
it('runs disjoint Vitest shards, preserves JSON receipts, learns from blobs, and rejects a missing shard', () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), 'gitnexus-shard-roundtrip-'));
|
||||
const vitest = fileURLToPath(new URL('../../node_modules/vitest/vitest.mjs', import.meta.url));
|
||||
const sequencer = fileURLToPath(new URL('../helpers/perf-sequencer.ts', import.meta.url));
|
||||
const reporter = fileURLToPath(
|
||||
new URL('../../scripts/shard-timings-reporter.ts', import.meta.url),
|
||||
);
|
||||
const execution = fileURLToPath(new URL('../../scripts/execution-reporter.ts', import.meta.url));
|
||||
const timingFile = path.join(root, '.vitest/timings/coverage.json');
|
||||
try {
|
||||
mkdirSync(path.dirname(timingFile), { recursive: true });
|
||||
writeFileSync(
|
||||
timingFile,
|
||||
JSON.stringify({
|
||||
'serial:a.test.ts': 10000,
|
||||
'serial:c.test.ts': 9000,
|
||||
'parallel:b.test.ts': 10,
|
||||
'parallel:d.test.ts': 10,
|
||||
}),
|
||||
);
|
||||
writeFileSync(
|
||||
path.join(root, 'vitest.config.ts'),
|
||||
`
|
||||
import { defineConfig } from ${JSON.stringify(import.meta.resolve('vitest/config'))};
|
||||
import PerfSequencer from ${JSON.stringify(sequencer.replaceAll('\\', '/'))};
|
||||
export default defineConfig({ test: {
|
||||
sequence: { sequencer: PerfSequencer },
|
||||
projects: [
|
||||
{ extends: true, test: { name: 'serial', include: ['a.test.ts', 'c.test.ts'], fileParallelism: false, sequence: { groupOrder: 1 } } },
|
||||
{ extends: true, test: { name: 'parallel', include: ['b.test.ts', 'd.test.ts'], sequence: { groupOrder: 2 } } }
|
||||
]
|
||||
} });
|
||||
`,
|
||||
);
|
||||
for (const name of ['a', 'b', 'c', 'd']) {
|
||||
writeFileSync(
|
||||
path.join(root, `${name}.test.ts`),
|
||||
`import { test, expect } from ${JSON.stringify(import.meta.resolve('vitest'))}; test('${name}', () => expect(1 + 1).toBe(2));`,
|
||||
);
|
||||
}
|
||||
const run = (args: string[]) =>
|
||||
spawnSync(
|
||||
process.execPath,
|
||||
[vitest, 'run', '--root', root, '--config', path.join(root, 'vitest.config.ts'), ...args],
|
||||
{
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
timeout: 30000,
|
||||
env: { ...process.env, GITNEXUS_SHARD_PROFILE: 'coverage', NO_COLOR: '1' },
|
||||
},
|
||||
);
|
||||
const files: string[][] = [];
|
||||
for (const index of [1, 2]) {
|
||||
const result = run([
|
||||
`--shard=${index}/2`,
|
||||
'--reporter=blob',
|
||||
`--reporter=${execution}`,
|
||||
`--outputFile.blob=blobs/${index}.json`,
|
||||
`--outputFile.json=receipt-${index}.json`,
|
||||
'--maxWorkers=1',
|
||||
]);
|
||||
expect(result.status, result.stdout + result.stderr).toBe(0);
|
||||
const receipt = JSON.parse(readFileSync(path.join(root, `receipt-${index}.json`), 'utf8'));
|
||||
expect(receipt.executionReason).toBe('passed');
|
||||
files.push(receipt.testResults.map((entry: { name: string }) => path.basename(entry.name)));
|
||||
}
|
||||
expect(files.flat().sort()).toEqual(['a.test.ts', 'b.test.ts', 'c.test.ts', 'd.test.ts']);
|
||||
expect(files.some((bin) => bin.includes('a.test.ts') && bin.includes('c.test.ts'))).toBe(false);
|
||||
const result = run(['--mergeReports=blobs', `--reporter=${reporter}`]);
|
||||
expect(result.status, result.stdout + result.stderr).toBe(0);
|
||||
const learned = readFileSync(timingFile, 'utf8');
|
||||
const timings = JSON.parse(learned);
|
||||
expect(Object.keys(timings).sort()).toEqual([
|
||||
'parallel:b.test.ts',
|
||||
'parallel:d.test.ts',
|
||||
'serial:a.test.ts',
|
||||
'serial:c.test.ts',
|
||||
]);
|
||||
expect(timings['serial:a.test.ts']).toBeGreaterThanOrEqual(5000);
|
||||
expect(timings['serial:a.test.ts']).toBeLessThan(10000);
|
||||
rmSync(path.join(root, 'blobs/2.json'));
|
||||
const incomplete = run(['--mergeReports=blobs', `--reporter=${reporter}`]);
|
||||
expect(incomplete.status).not.toBe(0);
|
||||
expect(incomplete.stdout + incomplete.stderr).toContain('Incomplete shard reports');
|
||||
expect(readFileSync(timingFile, 'utf8')).toBe(learned);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}, 120000);
|
||||
|
|
@ -51,10 +51,8 @@ export default defineConfig({
|
|||
},
|
||||
},
|
||||
|
||||
// Balance shards by estimated work rather than file count, so the
|
||||
// spawn-heavy sequential suites spread evenly across shard runners instead
|
||||
// of clustering onto one (see test/helpers/perf-sequencer.ts). Only shard()
|
||||
// is overridden — groupOrder and sort order are left to the base sequencer.
|
||||
// Balance shards using vitest-shard-balancer and the frozen CI timing history.
|
||||
// groupOrder and sort order are left to the base sequencer.
|
||||
sequence: {
|
||||
sequencer: PerfSequencer,
|
||||
},
|
||||
|
|
@ -165,7 +163,7 @@ export default defineConfig({
|
|||
// file-lock exposure this project exists to serialize (TESTING.md
|
||||
// § Vitest projects). Registering it here does NOT narrow where it
|
||||
// runs: vitest applies `--shard` once to the combined cross-project
|
||||
// spec list (PerfSequencer/assignShards is a complete, disjoint
|
||||
// spec list (PerfSequencer is a complete, disjoint
|
||||
// partition), and run-cross-platform.ts hands vitest explicit file
|
||||
// paths, which resolve against every project's include list. Its
|
||||
// `incremental-vector-extension-ordering` /
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue