diff --git a/eval/tests/test_runner_hardening.py b/eval/tests/test_runner_hardening.py index 7ccfc3b26..c3c1375af 100644 --- a/eval/tests/test_runner_hardening.py +++ b/eval/tests/test_runner_hardening.py @@ -727,17 +727,18 @@ def test_sweep_of_one_worker_never_leaves_the_calling_thread(): assert seen == [caller] * len(CELLS) -def test_sweep_keeps_the_rows_of_cells_that_finished_beside_a_failing_one(): +@pytest.mark.parametrize("failure", [KeyError("harness bug"), SystemExit(97)]) +def test_sweep_keeps_the_rows_of_cells_that_finished_beside_a_failing_one(failure): def run(run_idx, arm): if (run_idx, arm) == (0, "candidate_workflow"): - raise KeyError("harness bug") + raise failure return _row() progress = _progress() # The failing cell's two siblings completed and spent their budget before # the harness bug surfaced. Reading the futures in order and raising on the # first failure would drop their rows: money spent, no evidence written. - with pytest.raises(KeyError): + with pytest.raises(type(failure)): _sweep(CELLS, workers=3, run=run, into=progress) assert progress.kept == [(0, "workflow"), (1, "workflow")] diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index 019e9f5d3..410157739 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -687,10 +687,10 @@ def _run_wave( def _settle(future: Future[dict[str, Any]]) -> CellOutcome: """A completed future as (record, error) — exactly one of them is set.""" - try: - return future.result(), None - except BaseException as error: # noqa: BLE001 - re-raised by the caller, in order + error = future.exception() + if error is not None: return None, error + return future.result(), None def sweep_task_cells(