From 117ca19d3cf69f43ccf4c836bebe53f3435d593f Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Mon, 26 Oct 2026 14:46:33 +0800 Subject: [PATCH] refactor(memlens): simplify diagnostics and summary aggregation --- benchmark/memlens/run.py | 36 ++++++---------------- tests/unit/test_memlens_benchmark.py | 2 +- tests/unit/test_memlens_runner_failures.py | 10 +++--- 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/benchmark/memlens/run.py b/benchmark/memlens/run.py index 15833aa9..433270fc 100644 --- a/benchmark/memlens/run.py +++ b/benchmark/memlens/run.py @@ -330,14 +330,15 @@ def app_config(workspace: Path, model: str, *, judge: bool = False) -> dict: return values -def check_response(response): - """Do not reinterpret execution failures as incorrect predictions.""" +def check_response(response, diagnostic_path: Path) -> None: + """Save failed responses before validation; keep execution failures unscored.""" + if not response.success: + save(diagnostic_path, {"response": response_record(response)}) if asyncio.current_task().cancelling(): raise asyncio.CancelledError if not response.success: detail = response.answer or response.metadata.get("error") or "No failure detail provided." raise RuntimeError(f"ReMe Job failed; response detail: {safe_value(detail)}") - return response async def close_application(app) -> None: @@ -390,12 +391,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots): include_images=True, include_source_sessions=True, ) - if not response.success: - save( - directory / "diagnostics" / f"{ordinal:05d}-response.json", - {"response": response_record(response)}, - ) - check_response(response) + check_response(response, directory / "diagnostics" / f"{ordinal:05d}-response.json") save( directory / "sessions" / f"{ordinal:05d}.json", { @@ -407,12 +403,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots): status.pop("session_id", None) status["stage"] = "index_update" response = await app.run_job("index_update") - if not response.success: - save( - case / "ingestion/attempt-001/diagnostics/index-response.json", - {"response": response_record(response)}, - ) - check_response(response) + check_response(response, case / "ingestion/attempt-001/diagnostics/index-response.json") status["memory_complete"] = True for index, question in enumerate(scene["questions"]): directory = case / "questions" / f"{index:04d}" @@ -429,12 +420,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots): include_source_sessions=True, max_iters=30, ) - if not response.success: - save( - directory / "diagnostics/answer-response.json", - {"response": response_record(response)}, - ) - check_response(response) + check_response(response, directory / "diagnostics/answer-response.json") save( directory / "answer.json", { @@ -455,9 +441,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots): subtype=question.get("subtype", ""), old_answer=question.get("old_answer", ""), ) - if not judgment.success: - save(directory / "diagnostics/judge-response.json", {"response": response_record(judgment)}) - check_response(judgment) + check_response(judgment, directory / "diagnostics/judge-response.json") status["stage"] = "score" metrics = score_answer(question, response.answer) metrics["llm_judge"] = judgment.metadata["llm_judge_score"] @@ -494,7 +478,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots): if "error" in status: status["error_type"] = status["error"]["type"] save(case / "status.json", status_record(status)) - return results, memory, answers + return results, memory, answers, status async def run(settings: dict, model: str, length: str, limit: int | None = None) -> dict: @@ -557,7 +541,7 @@ async def run(settings: dict, model: str, length: str, limit: int | None = None) memory = [row for output in outputs for row in output[1]] answers = [row for output in outputs for row in output[2]] expected = sum(len(scene["questions"]) for scene in scenes) - statuses = [json.loads(path.read_text()) for path in (group / "reme_image/scenes").glob("*/status.json")] + statuses = [output[3] for output in outputs] cleanup_errors = sum("cleanup_error" in row or row["stage"] == "cleanup" for row in statuses) summary = { "method": "reme_image", diff --git a/tests/unit/test_memlens_benchmark.py b/tests/unit/test_memlens_benchmark.py index 5f39fc2a..1a461ca5 100644 --- a/tests/unit/test_memlens_benchmark.py +++ b/tests/unit/test_memlens_benchmark.py @@ -295,7 +295,7 @@ async def test_image_only_pipeline_preserves_sessions_and_official_query(candida monkeypatch.setattr(runner, "app_config", lambda *_, **__: {}) monkeypatch.setattr(runner, "session_messages", lambda session: [{"text": session["id"]}]) monkeypatch.setattr(runner, "observe_usage", lambda _: None) - rows, _, _ = await runner.evaluate_scene(scene, 0, "candidate", workspace, judge, asyncio.Semaphore(2)) + rows, *_ = await runner.evaluate_scene(scene, 0, "candidate", workspace, judge, asyncio.Semaphore(2)) assert [name for name, _ in calls] == ["auto_memory", "auto_memory", "index_update", "agentic_answer"] assert [kw["session_id"] for name, kw in calls if name == "auto_memory"] == ["first", "second"] for name, kwargs in calls: diff --git a/tests/unit/test_memlens_runner_failures.py b/tests/unit/test_memlens_runner_failures.py index ba878e31..095278ff 100644 --- a/tests/unit/test_memlens_runner_failures.py +++ b/tests/unit/test_memlens_runner_failures.py @@ -65,10 +65,10 @@ async def test_index_failure_retains_completed_session_usage(scene_run): return Response(success=False, answer="Index unavailable") app.run_job.side_effect = job - results, memory, answers = await evaluate() + results, memory, answers, status = await evaluate() assert results == answers == [] assert runner.total_usage(memory)["total_tokens"] == 120 - status = json.loads((case / "status.json").read_text()) + assert json.loads((case / "status.json").read_text()) == status assert status["stage"] == "index_update" and status["scene_id"] == "scene-1" assert "response detail: Index unavailable" in status["error"]["message"] assert status["error_type"] == "RuntimeError" @@ -96,7 +96,7 @@ async def test_failed_memory_keeps_its_successful_requests_and_diagnostics(scene return Response(success=False, metadata={"error": "Memory failed"}) app.run_job.side_effect = job - _, memory, _ = await evaluate() + _, memory, _, _ = await evaluate() assert runner.total_usage(memory)["total_tokens"] == 240 status = json.loads((case / "status.json").read_text()) assert status["stage"] == "auto_memory" and status["session_id"] == "session-2" @@ -126,7 +126,7 @@ async def test_failed_answer_or_judge_keeps_answer_usage_once(scene_run, failed_ app.run_job.side_effect = job if failed_job == "judge": judge.run_job.return_value = Response(success=False, metadata={"error": "Judge failed"}) - results, memory, answers = await evaluate() + results, memory, answers, _ = await evaluate() assert results == [] and len(memory) == len(answers) == 1 assert runner.total_usage(answers)["total_tokens"] == 120 status = json.loads((case / "status.json").read_text()) @@ -253,7 +253,7 @@ async def test_successful_predictions_and_references_remain_verbatim(scene_run): answer = "token=ordinary-text https://example.test/path?a=b data:image/png;base64," + "QmFzZTY0" * 20 scene["questions"][0]["answer"] = answer app.run_job.return_value = Response(answer=answer) - results, _, _ = await evaluate() + results, *_ = await evaluate() assert results[0]["prediction"] == results[0]["reference"] == answer assert judge.run_job.call_args.kwargs["prediction"] == answer for name in ("answer.json", "result.json"):