mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-18 09:42:31 +00:00
refactor(mobilemem): share task cleanup and use persisted results
This commit is contained in:
parent
2467c7248e
commit
bfbd6de6aa
3 changed files with 47 additions and 24 deletions
|
|
@ -369,11 +369,24 @@ async def close_application(app, directory, *, user_id=None):
|
|||
save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id))
|
||||
|
||||
|
||||
async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) -> list[dict]:
|
||||
async def gather_tasks(coroutines) -> None:
|
||||
"""Wait for owned tasks, cancelling and joining siblings on failure or cancellation."""
|
||||
tasks = [asyncio.create_task(coroutine) for coroutine in coroutines]
|
||||
try:
|
||||
await asyncio.gather(*tasks)
|
||||
finally:
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) -> None:
|
||||
"""Ingest the complete chronological history into one fresh user workspace."""
|
||||
app, response, session_id = None, None, None
|
||||
stage = "startup"
|
||||
results, memory_usage, aliases = [], [], {}
|
||||
completed = 0
|
||||
memory_usage, aliases = [], {}
|
||||
try:
|
||||
app = Application(**candidate_config(directory / "workspace", model))
|
||||
await app.start()
|
||||
|
|
@ -421,6 +434,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots)
|
|||
save(directory / "memory_usage.json", aggregate_usage(memory_usage))
|
||||
|
||||
async def answer_question(ordinal, question):
|
||||
nonlocal completed
|
||||
question_dir = directory / "questions" / f"{ordinal:04d}"
|
||||
answer_stage, answer_response = "answer", None
|
||||
try:
|
||||
|
|
@ -469,7 +483,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots)
|
|||
"judge_parse_default_zero": answer_response.metadata["parse_default_zero"],
|
||||
}
|
||||
save(question_dir / "result.json", result)
|
||||
results.append(result)
|
||||
completed += 1
|
||||
print(f"{scene['id']}: {question['question_id']} completed", flush=True)
|
||||
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
|
||||
save(
|
||||
|
|
@ -487,14 +501,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots)
|
|||
print(f"{scene['id']}: {question['question_id']} failed ({type(error).__name__})", flush=True)
|
||||
|
||||
stage, response = "answers", None
|
||||
tasks = [asyncio.create_task(answer_question(i, question)) for i, question in enumerate(scene["questions"])]
|
||||
try:
|
||||
await asyncio.gather(*tasks)
|
||||
finally:
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
await gather_tasks(answer_question(i, question) for i, question in enumerate(scene["questions"]))
|
||||
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
|
||||
save(
|
||||
directory / "error.json",
|
||||
|
|
@ -507,8 +514,7 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots)
|
|||
try:
|
||||
await close_application(app, directory, user_id=scene["id"])
|
||||
finally:
|
||||
save(directory / "summary.json", {"completed": len(results), "expected": len(scene["questions"])})
|
||||
return results
|
||||
save(directory / "summary.json", {"completed": completed, "expected": len(scene["questions"])})
|
||||
|
||||
|
||||
def summarize_run(scenes, output: Path, model: str) -> dict:
|
||||
|
|
@ -568,14 +574,7 @@ async def run(scenes, output: Path, model: str, user_concurrency: int, query_con
|
|||
async with user_slots:
|
||||
await evaluate_user(scene, output / "users" / scene["id"], model, judge, query_slots)
|
||||
|
||||
tasks = [asyncio.create_task(user_job(scene)) for scene in scenes]
|
||||
try:
|
||||
await asyncio.gather(*tasks)
|
||||
finally:
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
await gather_tasks(user_job(scene) for scene in scenes)
|
||||
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
|
||||
save(output / "error.json", error_record(error, stage="run"))
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -291,6 +291,30 @@ async def test_cancelled_request_finalizes_once_and_closes_owned_apps(flow):
|
|||
assert read(flow.directory.parents[1] / "summary.json")["memory_usage"]["total_tokens"] == 120
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("cancel", [False, True])
|
||||
async def test_failed_task_joins_sibling_cleanup_before_propagating(cancel):
|
||||
started, closed = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def waiting():
|
||||
try:
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
await asyncio.sleep(0)
|
||||
closed.set()
|
||||
|
||||
async def interrupted():
|
||||
await started.wait()
|
||||
if cancel:
|
||||
raise asyncio.CancelledError()
|
||||
raise ValueError("offline task failed")
|
||||
|
||||
with pytest.raises(asyncio.CancelledError if cancel else ValueError):
|
||||
await run.gather_tasks([waiting(), interrupted()])
|
||||
assert closed.is_set()
|
||||
|
||||
|
||||
def test_error_artifacts_remove_secrets_payloads_and_locals(monkeypatch):
|
||||
secret = "offline-sensitive-value"
|
||||
monkeypatch.setenv("LLM_API_KEY", secret)
|
||||
|
|
|
|||
|
|
@ -364,7 +364,7 @@ async def test_image_only_pipeline_keeps_all_sessions_and_gold_is_judge_only(mob
|
|||
monkeypatch.setattr(mobilemem, "candidate_config", configured)
|
||||
monkeypatch.setattr(mobilemem, "observe_usage", lambda _: None)
|
||||
monkeypatch.setattr(mobilemem, "observe_completion", lambda _: None)
|
||||
rows = await mobilemem.evaluate_user(
|
||||
await mobilemem.evaluate_user(
|
||||
scene,
|
||||
tmp_path / "new-user",
|
||||
"candidate",
|
||||
|
|
@ -387,8 +387,8 @@ async def test_image_only_pipeline_keeps_all_sessions_and_gold_is_judge_only(mob
|
|||
judge_inputs = judge.run_job.call_args.kwargs
|
||||
assert judge_inputs["reference"] == "HIDDEN_GOLD"
|
||||
assert judge_inputs["evidence"] == ["HIDDEN_EVIDENCE"]
|
||||
assert rows[0]["metrics"]["llm_judge"] == 1 and apps[0].closed
|
||||
assert (tmp_path / "new-user/questions/0000/result.json").is_file()
|
||||
result = json.loads((tmp_path / "new-user/questions/0000/result.json").read_text(encoding="utf-8"))
|
||||
assert result["metrics"]["llm_judge"] == 1 and apps[0].closed
|
||||
assert normalized_usage({"prompt_tokens": 100, "completion_tokens": 20})["cached_input_tokens"] is None
|
||||
assert (
|
||||
normalized_usage({"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 30}})[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue