mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-17 09:32:01 +00:00
refactor(memlens): simplify diagnostics and summary aggregation
This commit is contained in:
parent
5bbf2d3ed0
commit
117ca19d3c
3 changed files with 16 additions and 32 deletions
|
|
@ -330,14 +330,15 @@ def app_config(workspace: Path, model: str, *, judge: bool = False) -> dict:
|
|||
return values
|
||||
|
||||
|
||||
def check_response(response):
|
||||
"""Do not reinterpret execution failures as incorrect predictions."""
|
||||
def check_response(response, diagnostic_path: Path) -> None:
|
||||
"""Save failed responses before validation; keep execution failures unscored."""
|
||||
if not response.success:
|
||||
save(diagnostic_path, {"response": response_record(response)})
|
||||
if asyncio.current_task().cancelling():
|
||||
raise asyncio.CancelledError
|
||||
if not response.success:
|
||||
detail = response.answer or response.metadata.get("error") or "No failure detail provided."
|
||||
raise RuntimeError(f"ReMe Job failed; response detail: {safe_value(detail)}")
|
||||
return response
|
||||
|
||||
|
||||
async def close_application(app) -> None:
|
||||
|
|
@ -390,12 +391,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
|
|||
include_images=True,
|
||||
include_source_sessions=True,
|
||||
)
|
||||
if not response.success:
|
||||
save(
|
||||
directory / "diagnostics" / f"{ordinal:05d}-response.json",
|
||||
{"response": response_record(response)},
|
||||
)
|
||||
check_response(response)
|
||||
check_response(response, directory / "diagnostics" / f"{ordinal:05d}-response.json")
|
||||
save(
|
||||
directory / "sessions" / f"{ordinal:05d}.json",
|
||||
{
|
||||
|
|
@ -407,12 +403,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
|
|||
status.pop("session_id", None)
|
||||
status["stage"] = "index_update"
|
||||
response = await app.run_job("index_update")
|
||||
if not response.success:
|
||||
save(
|
||||
case / "ingestion/attempt-001/diagnostics/index-response.json",
|
||||
{"response": response_record(response)},
|
||||
)
|
||||
check_response(response)
|
||||
check_response(response, case / "ingestion/attempt-001/diagnostics/index-response.json")
|
||||
status["memory_complete"] = True
|
||||
for index, question in enumerate(scene["questions"]):
|
||||
directory = case / "questions" / f"{index:04d}"
|
||||
|
|
@ -429,12 +420,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
|
|||
include_source_sessions=True,
|
||||
max_iters=30,
|
||||
)
|
||||
if not response.success:
|
||||
save(
|
||||
directory / "diagnostics/answer-response.json",
|
||||
{"response": response_record(response)},
|
||||
)
|
||||
check_response(response)
|
||||
check_response(response, directory / "diagnostics/answer-response.json")
|
||||
save(
|
||||
directory / "answer.json",
|
||||
{
|
||||
|
|
@ -455,9 +441,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
|
|||
subtype=question.get("subtype", ""),
|
||||
old_answer=question.get("old_answer", ""),
|
||||
)
|
||||
if not judgment.success:
|
||||
save(directory / "diagnostics/judge-response.json", {"response": response_record(judgment)})
|
||||
check_response(judgment)
|
||||
check_response(judgment, directory / "diagnostics/judge-response.json")
|
||||
status["stage"] = "score"
|
||||
metrics = score_answer(question, response.answer)
|
||||
metrics["llm_judge"] = judgment.metadata["llm_judge_score"]
|
||||
|
|
@ -494,7 +478,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
|
|||
if "error" in status:
|
||||
status["error_type"] = status["error"]["type"]
|
||||
save(case / "status.json", status_record(status))
|
||||
return results, memory, answers
|
||||
return results, memory, answers, status
|
||||
|
||||
|
||||
async def run(settings: dict, model: str, length: str, limit: int | None = None) -> dict:
|
||||
|
|
@ -557,7 +541,7 @@ async def run(settings: dict, model: str, length: str, limit: int | None = None)
|
|||
memory = [row for output in outputs for row in output[1]]
|
||||
answers = [row for output in outputs for row in output[2]]
|
||||
expected = sum(len(scene["questions"]) for scene in scenes)
|
||||
statuses = [json.loads(path.read_text()) for path in (group / "reme_image/scenes").glob("*/status.json")]
|
||||
statuses = [output[3] for output in outputs]
|
||||
cleanup_errors = sum("cleanup_error" in row or row["stage"] == "cleanup" for row in statuses)
|
||||
summary = {
|
||||
"method": "reme_image",
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ async def test_image_only_pipeline_preserves_sessions_and_official_query(candida
|
|||
monkeypatch.setattr(runner, "app_config", lambda *_, **__: {})
|
||||
monkeypatch.setattr(runner, "session_messages", lambda session: [{"text": session["id"]}])
|
||||
monkeypatch.setattr(runner, "observe_usage", lambda _: None)
|
||||
rows, _, _ = await runner.evaluate_scene(scene, 0, "candidate", workspace, judge, asyncio.Semaphore(2))
|
||||
rows, *_ = await runner.evaluate_scene(scene, 0, "candidate", workspace, judge, asyncio.Semaphore(2))
|
||||
assert [name for name, _ in calls] == ["auto_memory", "auto_memory", "index_update", "agentic_answer"]
|
||||
assert [kw["session_id"] for name, kw in calls if name == "auto_memory"] == ["first", "second"]
|
||||
for name, kwargs in calls:
|
||||
|
|
|
|||
|
|
@ -65,10 +65,10 @@ async def test_index_failure_retains_completed_session_usage(scene_run):
|
|||
return Response(success=False, answer="Index unavailable")
|
||||
|
||||
app.run_job.side_effect = job
|
||||
results, memory, answers = await evaluate()
|
||||
results, memory, answers, status = await evaluate()
|
||||
assert results == answers == []
|
||||
assert runner.total_usage(memory)["total_tokens"] == 120
|
||||
status = json.loads((case / "status.json").read_text())
|
||||
assert json.loads((case / "status.json").read_text()) == status
|
||||
assert status["stage"] == "index_update" and status["scene_id"] == "scene-1"
|
||||
assert "response detail: Index unavailable" in status["error"]["message"]
|
||||
assert status["error_type"] == "RuntimeError"
|
||||
|
|
@ -96,7 +96,7 @@ async def test_failed_memory_keeps_its_successful_requests_and_diagnostics(scene
|
|||
return Response(success=False, metadata={"error": "Memory failed"})
|
||||
|
||||
app.run_job.side_effect = job
|
||||
_, memory, _ = await evaluate()
|
||||
_, memory, _, _ = await evaluate()
|
||||
assert runner.total_usage(memory)["total_tokens"] == 240
|
||||
status = json.loads((case / "status.json").read_text())
|
||||
assert status["stage"] == "auto_memory" and status["session_id"] == "session-2"
|
||||
|
|
@ -126,7 +126,7 @@ async def test_failed_answer_or_judge_keeps_answer_usage_once(scene_run, failed_
|
|||
app.run_job.side_effect = job
|
||||
if failed_job == "judge":
|
||||
judge.run_job.return_value = Response(success=False, metadata={"error": "Judge failed"})
|
||||
results, memory, answers = await evaluate()
|
||||
results, memory, answers, _ = await evaluate()
|
||||
assert results == [] and len(memory) == len(answers) == 1
|
||||
assert runner.total_usage(answers)["total_tokens"] == 120
|
||||
status = json.loads((case / "status.json").read_text())
|
||||
|
|
@ -253,7 +253,7 @@ async def test_successful_predictions_and_references_remain_verbatim(scene_run):
|
|||
answer = "token=ordinary-text https://example.test/path?a=b data:image/png;base64," + "QmFzZTY0" * 20
|
||||
scene["questions"][0]["answer"] = answer
|
||||
app.run_job.return_value = Response(answer=answer)
|
||||
results, _, _ = await evaluate()
|
||||
results, *_ = await evaluate()
|
||||
assert results[0]["prediction"] == results[0]["reference"] == answer
|
||||
assert judge.run_job.call_args.kwargs["prediction"] == answer
|
||||
for name in ("answer.json", "result.json"):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue