refactor(mobilemem): share task cleanup and use persisted results

This commit is contained in:
jinli.yl 2026-10-26 15:15:51 +08:00
parent 2467c7248e
commit bfbd6de6aa
3 changed files with 47 additions and 24 deletions

View file

@ -369,11 +369,24 @@ async def close_application(app, directory, *, user_id=None):
save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id))
async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) -> list[dict]:
async def gather_tasks(coroutines) -> None:
"""Wait for owned tasks, cancelling and joining siblings on failure or cancellation."""
tasks = [asyncio.create_task(coroutine) for coroutine in coroutines]
try:
await asyncio.gather(*tasks)
finally:
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) -> None:
"""Ingest the complete chronological history into one fresh user workspace."""
app, response, session_id = None, None, None
stage = "startup"
results, memory_usage, aliases = [], [], {}
completed = 0
memory_usage, aliases = [], {}
try:
app = Application(**candidate_config(directory / "workspace", model))
await app.start()
@ -421,6 +434,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots)
save(directory / "memory_usage.json", aggregate_usage(memory_usage))
async def answer_question(ordinal, question):
nonlocal completed
question_dir = directory / "questions" / f"{ordinal:04d}"
answer_stage, answer_response = "answer", None
try:
@ -469,7 +483,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots)
"judge_parse_default_zero": answer_response.metadata["parse_default_zero"],
}
save(question_dir / "result.json", result)
results.append(result)
completed += 1
print(f"{scene['id']}: {question['question_id']} completed", flush=True)
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
save(
@ -487,14 +501,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots)
print(f"{scene['id']}: {question['question_id']} failed ({type(error).__name__})", flush=True)
stage, response = "answers", None
tasks = [asyncio.create_task(answer_question(i, question)) for i, question in enumerate(scene["questions"])]
try:
await asyncio.gather(*tasks)
finally:
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
await gather_tasks(answer_question(i, question) for i, question in enumerate(scene["questions"]))
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
save(
directory / "error.json",
@ -507,8 +514,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots)
try:
await close_application(app, directory, user_id=scene["id"])
finally:
save(directory / "summary.json", {"completed": len(results), "expected": len(scene["questions"])})
return results
save(directory / "summary.json", {"completed": completed, "expected": len(scene["questions"])})
def summarize_run(scenes, output: Path, model: str) -> dict:
@ -568,14 +574,7 @@ async def run(scenes, output: Path, model: str, user_concurrency: int, query_con
async with user_slots:
await evaluate_user(scene, output / "users" / scene["id"], model, judge, query_slots)
tasks = [asyncio.create_task(user_job(scene)) for scene in scenes]
try:
await asyncio.gather(*tasks)
finally:
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
await gather_tasks(user_job(scene) for scene in scenes)
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
save(output / "error.json", error_record(error, stage="run"))
raise

View file

@ -291,6 +291,30 @@ async def test_cancelled_request_finalizes_once_and_closes_owned_apps(flow):
assert read(flow.directory.parents[1] / "summary.json")["memory_usage"]["total_tokens"] == 120
@pytest.mark.asyncio
@pytest.mark.parametrize("cancel", [False, True])
async def test_failed_task_joins_sibling_cleanup_before_propagating(cancel):
started, closed = asyncio.Event(), asyncio.Event()
async def waiting():
try:
started.set()
await asyncio.Event().wait()
finally:
await asyncio.sleep(0)
closed.set()
async def interrupted():
await started.wait()
if cancel:
raise asyncio.CancelledError()
raise ValueError("offline task failed")
with pytest.raises(asyncio.CancelledError if cancel else ValueError):
await run.gather_tasks([waiting(), interrupted()])
assert closed.is_set()
def test_error_artifacts_remove_secrets_payloads_and_locals(monkeypatch):
secret = "offline-sensitive-value"
monkeypatch.setenv("LLM_API_KEY", secret)

View file

@ -364,7 +364,7 @@ async def test_image_only_pipeline_keeps_all_sessions_and_gold_is_judge_only(mob
monkeypatch.setattr(mobilemem, "candidate_config", configured)
monkeypatch.setattr(mobilemem, "observe_usage", lambda _: None)
monkeypatch.setattr(mobilemem, "observe_completion", lambda _: None)
rows = await mobilemem.evaluate_user(
await mobilemem.evaluate_user(
scene,
tmp_path / "new-user",
"candidate",
@ -387,8 +387,8 @@ async def test_image_only_pipeline_keeps_all_sessions_and_gold_is_judge_only(mob
judge_inputs = judge.run_job.call_args.kwargs
assert judge_inputs["reference"] == "HIDDEN_GOLD"
assert judge_inputs["evidence"] == ["HIDDEN_EVIDENCE"]
assert rows[0]["metrics"]["llm_judge"] == 1 and apps[0].closed
assert (tmp_path / "new-user/questions/0000/result.json").is_file()
result = json.loads((tmp_path / "new-user/questions/0000/result.json").read_text(encoding="utf-8"))
assert result["metrics"]["llm_judge"] == 1 and apps[0].closed
assert normalized_usage({"prompt_tokens": 100, "completion_tokens": 20})["cached_input_tokens"] is None
assert (
normalized_usage({"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 30}})[