refactor(memlens): simplify diagnostics and summary aggregation

This commit is contained in:
jinli.yl 2026-10-26 14:46:33 +08:00
parent 5bbf2d3ed0
commit 117ca19d3c
3 changed files with 16 additions and 32 deletions

View file

@ -330,14 +330,15 @@ def app_config(workspace: Path, model: str, *, judge: bool = False) -> dict:
return values
def check_response(response):
"""Do not reinterpret execution failures as incorrect predictions."""
def check_response(response, diagnostic_path: Path) -> None:
"""Save failed responses before validation; keep execution failures unscored."""
if not response.success:
save(diagnostic_path, {"response": response_record(response)})
if asyncio.current_task().cancelling():
raise asyncio.CancelledError
if not response.success:
detail = response.answer or response.metadata.get("error") or "No failure detail provided."
raise RuntimeError(f"ReMe Job failed; response detail: {safe_value(detail)}")
return response
async def close_application(app) -> None:
@ -390,12 +391,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
include_images=True,
include_source_sessions=True,
)
if not response.success:
save(
directory / "diagnostics" / f"{ordinal:05d}-response.json",
{"response": response_record(response)},
)
check_response(response)
check_response(response, directory / "diagnostics" / f"{ordinal:05d}-response.json")
save(
directory / "sessions" / f"{ordinal:05d}.json",
{
@ -407,12 +403,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
status.pop("session_id", None)
status["stage"] = "index_update"
response = await app.run_job("index_update")
if not response.success:
save(
case / "ingestion/attempt-001/diagnostics/index-response.json",
{"response": response_record(response)},
)
check_response(response)
check_response(response, case / "ingestion/attempt-001/diagnostics/index-response.json")
status["memory_complete"] = True
for index, question in enumerate(scene["questions"]):
directory = case / "questions" / f"{index:04d}"
@ -429,12 +420,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
include_source_sessions=True,
max_iters=30,
)
if not response.success:
save(
directory / "diagnostics/answer-response.json",
{"response": response_record(response)},
)
check_response(response)
check_response(response, directory / "diagnostics/answer-response.json")
save(
directory / "answer.json",
{
@ -455,9 +441,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
subtype=question.get("subtype", ""),
old_answer=question.get("old_answer", ""),
)
if not judgment.success:
save(directory / "diagnostics/judge-response.json", {"response": response_record(judgment)})
check_response(judgment)
check_response(judgment, directory / "diagnostics/judge-response.json")
status["stage"] = "score"
metrics = score_answer(question, response.answer)
metrics["llm_judge"] = judgment.metadata["llm_judge_score"]
@ -494,7 +478,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
if "error" in status:
status["error_type"] = status["error"]["type"]
save(case / "status.json", status_record(status))
return results, memory, answers
return results, memory, answers, status
async def run(settings: dict, model: str, length: str, limit: int | None = None) -> dict:
@ -557,7 +541,7 @@ async def run(settings: dict, model: str, length: str, limit: int | None = None)
memory = [row for output in outputs for row in output[1]]
answers = [row for output in outputs for row in output[2]]
expected = sum(len(scene["questions"]) for scene in scenes)
statuses = [json.loads(path.read_text()) for path in (group / "reme_image/scenes").glob("*/status.json")]
statuses = [output[3] for output in outputs]
cleanup_errors = sum("cleanup_error" in row or row["stage"] == "cleanup" for row in statuses)
summary = {
"method": "reme_image",

View file

@ -295,7 +295,7 @@ async def test_image_only_pipeline_preserves_sessions_and_official_query(candida
monkeypatch.setattr(runner, "app_config", lambda *_, **__: {})
monkeypatch.setattr(runner, "session_messages", lambda session: [{"text": session["id"]}])
monkeypatch.setattr(runner, "observe_usage", lambda _: None)
rows, _, _ = await runner.evaluate_scene(scene, 0, "candidate", workspace, judge, asyncio.Semaphore(2))
rows, *_ = await runner.evaluate_scene(scene, 0, "candidate", workspace, judge, asyncio.Semaphore(2))
assert [name for name, _ in calls] == ["auto_memory", "auto_memory", "index_update", "agentic_answer"]
assert [kw["session_id"] for name, kw in calls if name == "auto_memory"] == ["first", "second"]
for name, kwargs in calls:

View file

@ -65,10 +65,10 @@ async def test_index_failure_retains_completed_session_usage(scene_run):
return Response(success=False, answer="Index unavailable")
app.run_job.side_effect = job
results, memory, answers = await evaluate()
results, memory, answers, status = await evaluate()
assert results == answers == []
assert runner.total_usage(memory)["total_tokens"] == 120
status = json.loads((case / "status.json").read_text())
assert json.loads((case / "status.json").read_text()) == status
assert status["stage"] == "index_update" and status["scene_id"] == "scene-1"
assert "response detail: Index unavailable" in status["error"]["message"]
assert status["error_type"] == "RuntimeError"
@ -96,7 +96,7 @@ async def test_failed_memory_keeps_its_successful_requests_and_diagnostics(scene
return Response(success=False, metadata={"error": "Memory failed"})
app.run_job.side_effect = job
_, memory, _ = await evaluate()
_, memory, _, _ = await evaluate()
assert runner.total_usage(memory)["total_tokens"] == 240
status = json.loads((case / "status.json").read_text())
assert status["stage"] == "auto_memory" and status["session_id"] == "session-2"
@ -126,7 +126,7 @@ async def test_failed_answer_or_judge_keeps_answer_usage_once(scene_run, failed_
app.run_job.side_effect = job
if failed_job == "judge":
judge.run_job.return_value = Response(success=False, metadata={"error": "Judge failed"})
results, memory, answers = await evaluate()
results, memory, answers, _ = await evaluate()
assert results == [] and len(memory) == len(answers) == 1
assert runner.total_usage(answers)["total_tokens"] == 120
status = json.loads((case / "status.json").read_text())
@ -253,7 +253,7 @@ async def test_successful_predictions_and_references_remain_verbatim(scene_run):
answer = "token=ordinary-text https://example.test/path?a=b data:image/png;base64," + "QmFzZTY0" * 20
scene["questions"][0]["answer"] = answer
app.run_job.return_value = Response(answer=answer)
results, _, _ = await evaluate()
results, *_ = await evaluate()
assert results[0]["prediction"] == results[0]["reference"] == answer
assert judge.run_job.call_args.kwargs["prediction"] == answer
for name in ("answer.json", "result.json"):