diff --git a/benchmark/mobilemem/run.py b/benchmark/mobilemem/run.py index c3302f27..28962ab4 100644 --- a/benchmark/mobilemem/run.py +++ b/benchmark/mobilemem/run.py @@ -369,11 +369,24 @@ async def close_application(app, directory, *, user_id=None): save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id)) -async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) -> list[dict]: +async def gather_tasks(coroutines) -> None: + """Wait for owned tasks, cancelling and joining siblings on failure or cancellation.""" + tasks = [asyncio.create_task(coroutine) for coroutine in coroutines] + try: + await asyncio.gather(*tasks) + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) -> None: """Ingest the complete chronological history into one fresh user workspace.""" app, response, session_id = None, None, None stage = "startup" - results, memory_usage, aliases = [], [], {} + completed = 0 + memory_usage, aliases = [], {} try: app = Application(**candidate_config(directory / "workspace", model)) await app.start() @@ -421,6 +434,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) save(directory / "memory_usage.json", aggregate_usage(memory_usage)) async def answer_question(ordinal, question): + nonlocal completed question_dir = directory / "questions" / f"{ordinal:04d}" answer_stage, answer_response = "answer", None try: @@ -469,7 +483,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) "judge_parse_default_zero": answer_response.metadata["parse_default_zero"], } save(question_dir / "result.json", result) - results.append(result) + completed += 1 print(f"{scene['id']}: {question['question_id']} completed", flush=True) except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught save( @@ -487,14 +501,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) print(f"{scene['id']}: {question['question_id']} failed ({type(error).__name__})", flush=True) stage, response = "answers", None - tasks = [asyncio.create_task(answer_question(i, question)) for i, question in enumerate(scene["questions"])] - try: - await asyncio.gather(*tasks) - finally: - for task in tasks: - if not task.done(): - task.cancel() - await asyncio.gather(*tasks, return_exceptions=True) + await gather_tasks(answer_question(i, question) for i, question in enumerate(scene["questions"])) except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught save( directory / "error.json", @@ -507,8 +514,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) try: await close_application(app, directory, user_id=scene["id"]) finally: - save(directory / "summary.json", {"completed": len(results), "expected": len(scene["questions"])}) - return results + save(directory / "summary.json", {"completed": completed, "expected": len(scene["questions"])}) def summarize_run(scenes, output: Path, model: str) -> dict: @@ -568,14 +574,7 @@ async def run(scenes, output: Path, model: str, user_concurrency: int, query_con async with user_slots: await evaluate_user(scene, output / "users" / scene["id"], model, judge, query_slots) - tasks = [asyncio.create_task(user_job(scene)) for scene in scenes] - try: - await asyncio.gather(*tasks) - finally: - for task in tasks: - if not task.done(): - task.cancel() - await asyncio.gather(*tasks, return_exceptions=True) + await gather_tasks(user_job(scene) for scene in scenes) except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught save(output / "error.json", error_record(error, stage="run")) raise diff --git a/tests/unit/test_mobilemem_audit.py b/tests/unit/test_mobilemem_audit.py index a29f6c51..e9a78def 100644 --- a/tests/unit/test_mobilemem_audit.py +++ b/tests/unit/test_mobilemem_audit.py @@ -291,6 +291,30 @@ async def test_cancelled_request_finalizes_once_and_closes_owned_apps(flow): assert read(flow.directory.parents[1] / "summary.json")["memory_usage"]["total_tokens"] == 120 +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel", [False, True]) +async def test_failed_task_joins_sibling_cleanup_before_propagating(cancel): + started, closed = asyncio.Event(), asyncio.Event() + + async def waiting(): + try: + started.set() + await asyncio.Event().wait() + finally: + await asyncio.sleep(0) + closed.set() + + async def interrupted(): + await started.wait() + if cancel: + raise asyncio.CancelledError() + raise ValueError("offline task failed") + + with pytest.raises(asyncio.CancelledError if cancel else ValueError): + await run.gather_tasks([waiting(), interrupted()]) + assert closed.is_set() + + def test_error_artifacts_remove_secrets_payloads_and_locals(monkeypatch): secret = "offline-sensitive-value" monkeypatch.setenv("LLM_API_KEY", secret) diff --git a/tests/unit/test_mobilemem_benchmark.py b/tests/unit/test_mobilemem_benchmark.py index 3c7da30f..54fc256f 100644 --- a/tests/unit/test_mobilemem_benchmark.py +++ b/tests/unit/test_mobilemem_benchmark.py @@ -364,7 +364,7 @@ async def test_image_only_pipeline_keeps_all_sessions_and_gold_is_judge_only(mob monkeypatch.setattr(mobilemem, "candidate_config", configured) monkeypatch.setattr(mobilemem, "observe_usage", lambda _: None) monkeypatch.setattr(mobilemem, "observe_completion", lambda _: None) - rows = await mobilemem.evaluate_user( + await mobilemem.evaluate_user( scene, tmp_path / "new-user", "candidate", @@ -387,8 +387,8 @@ async def test_image_only_pipeline_keeps_all_sessions_and_gold_is_judge_only(mob judge_inputs = judge.run_job.call_args.kwargs assert judge_inputs["reference"] == "HIDDEN_GOLD" assert judge_inputs["evidence"] == ["HIDDEN_EVIDENCE"] - assert rows[0]["metrics"]["llm_judge"] == 1 and apps[0].closed - assert (tmp_path / "new-user/questions/0000/result.json").is_file() + result = json.loads((tmp_path / "new-user/questions/0000/result.json").read_text(encoding="utf-8")) + assert result["metrics"]["llm_judge"] == 1 and apps[0].closed assert normalized_usage({"prompt_tokens": 100, "completion_tokens": 20})["cached_input_tokens"] is None assert ( normalized_usage({"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 30}})[