From 3c6222ff1cb5740cc98d681d725a38cbfed7e269 Mon Sep 17 00:00:00 2001 From: wang-qisen Date: Mon, 19 Oct 2026 22:50:46 +0800 Subject: [PATCH] fix(benchmark): preserve MobileMem usage and failure diagnostics --- benchmark/mobilemem/README.md | 22 +- benchmark/mobilemem/README_ZH.md | 18 +- benchmark/mobilemem/_audit.py | 163 ++++++++++ benchmark/mobilemem/run.py | 290 +++++++++++------ plugins/mobilemem-judge/README.md | 6 +- tests/unit/test_mobilemem_audit.py | 423 +++++++++++++++++++++++++ tests/unit/test_mobilemem_benchmark.py | 6 + 7 files changed, 829 insertions(+), 99 deletions(-) create mode 100644 benchmark/mobilemem/_audit.py create mode 100644 tests/unit/test_mobilemem_audit.py diff --git a/benchmark/mobilemem/README.md b/benchmark/mobilemem/README.md index 12bcbd23..3bccec4e 100644 --- a/benchmark/mobilemem/README.md +++ b/benchmark/mobilemem/README.md @@ -43,9 +43,25 @@ MemBase executable-default `mobilemem-omni-judge` rubric, `qwen3.8-max`, tempera Each user's `workspace/` retains notes, source JSONL, images and native AgentScope trajectories. `sessions/*.json` and `questions/*/` retain responses, judgments, metrics and technical errors. -`summary.json` reports completed/expected questions, scores, memory/answer token totals and averages. -Provider usage includes compression; absent cache fields stay unknown. Judge usage is excluded from method cost. -Native interrupted replies and failed Jobs remain unscored technical gaps. The runner adds no fallback or retry queue. +`summary.json` reports completed/expected questions and scores over successfully scored results only. +Native interrupted replies and failed Jobs remain unscored technical gaps, not zero-score answers. +The runner adds no fallback or retry queue. + +Usage policy version 2 counts observed successful HTTP requests, including compression and requests made before +a candidate history or answer later fails. Memory cost comes from `sessions/*.json` and `index-response.json`; +answer cost comes only from `answer-response.json`, including failed responses or `response: null` when a call raises. +Copies in `answer.json`, result files and response metadata are not counted again. `memory_usage.json` remains a +completion marker: it is written only after the full history and index update succeed. +Judge usage is excluded; absent counters stay unknown and each counter includes its observation coverage. +The `mean_tokens_per_user` / `mean_tokens_per_question` fields amortize this run's observed cost over scheduled users / +expected questions, with explicit denominators and attempt/completion counts. They are not successful-sample averages. +Earlier selected-success-only reports retain their original accounting policy. + +Errors retain the failing stage, user/session/question identifiers, safe image paths, sanitized reasons and stack +locations; credentials, request headers, image payloads and frame locals are omitted. Cleanup errors are separate; +`cleanup_error_count` makes the run exit with status 2 without discarding already scored results. +`judge-settings.json` records a credential-free whitelist of the validated Judge model, parameters and protocol, +including an optional `JUDGE_MODEL_NAME` override, so scoring can be traced to the effective configuration. Sources: [MobileMem evaluation code](https://github.com/zjunlp/MobileMem/tree/fa5cd13c9c659699df9caf650987ba2bea8ca085/omni/eval/eval) and [MemBase judge](https://github.com/zjunlp/MemBase/blob/a9ba9495d48541d3e68588ff097294cc7a1201bc/membase/inference_utils/prompts.py). diff --git a/benchmark/mobilemem/README_ZH.md b/benchmark/mobilemem/README_ZH.md index c05ced8c..4987bb24 100644 --- a/benchmark/mobilemem/README_ZH.md +++ b/benchmark/mobilemem/README_ZH.md @@ -41,8 +41,20 @@ temperature 为 0,开启 thinking,输出预算 65,536,上下文为 1,000,0 每个用户的 `workspace/` 保存笔记、原始 JSONL、图像及原生 AgentScope 轨迹。 `sessions/*.json` 与 `questions/*/` 保存响应、评分、指标和技术错误。 -`summary.json` 汇总完成/应测题数、指标、记忆与回答 token 总量及平均值。 -用量覆盖 provider 返回的压缩调用;缺失的 cache 字段保持未知。Judge 用量不计入方法成本。 -原生中断与失败 Job 作为未评分的技术缺口保存;runner 不额外加入 fallback 或重试队列。 +`summary.json` 汇总完成/应测题数,指标只对成功评分的结果求均值。 +原生中断与失败 Job 保留为未评分的技术缺口,不作为零分答案;runner 不额外加入 fallback 或重试队列。 + +usage policy 第 2 版统计已观测的成功 HTTP 请求,包括压缩调用及摄入/回答最终失败前已经产生的请求。 +记忆成本只汇总 `sessions/*.json` 和 `index-response.json`;回答成本只汇总 `answer-response.json`, +包括失败响应及直接异常时的 `response: null`。`answer.json`、result 和 response metadata 中的副本不重复计费。 +`memory_usage.json` 仍仅在完整历史摄入及索引更新成功后生成,不能将它误作部分摄入标记。 +Judge 用量不计入方法成本;缺失字段保持未知,并记录每个字段的观测覆盖数。 +`mean_tokens_per_user` / `mean_tokens_per_question` 是本次已观测成本按计划用户数/应测题数均摊, +summary 明确记录分母和尝试/完成数,不代表成功样本平均成本。旧的 selected-success-only 报告不改写口径。 + +错误记录包含失败阶段、用户/会话/题号、安全的图片相对路径、脱敏原因和调用栈位置,不保存凭据、请求 headers、 +图片载荷或 frame locals;清理错误单独保留,`cleanup_error_count` 使本轮以状态码 2 退出,但不丢弃已评分结果。 +`judge-settings.json` 从实际校验后的配置中保存 Judge 模型、参数和协议白名单, +包括可选的 `JUDGE_MODEL_NAME` 覆盖值,不导出凭据。 协议来源、数据版本和改编代码的完整 MIT 许可证见 [英文说明](README.md)。 diff --git a/benchmark/mobilemem/_audit.py b/benchmark/mobilemem/_audit.py new file mode 100644 index 00000000..68e9509d --- /dev/null +++ b/benchmark/mobilemem/_audit.py @@ -0,0 +1,163 @@ +"""Small, credential-safe diagnostics and accounting helpers for this runner.""" + +import os +from pathlib import Path, PurePosixPath +import re +import traceback + +USAGE_POLICY = { + "version": 2, + "scope": "observed successful HTTP requests, including failed candidate flows and compression; judge excluded", + "memory_sources": ["users/*/sessions/*.json", "users/*/index-response.json"], + "answer_source": "users/*/questions/*/answer-response.json", + "means": "observed run cost amortized over scheduled users / expected questions, not successful samples", +} +_SENSITIVE = re.compile( + r"authorization|credential|secret|password|api.?key|cookie|headers|" + r"(?:^|[_-])(?:access[_-]?token|refresh[_-]?token|id[_-]?token|token)$", + re.I, +) + + +def safe_text(value, limit: int = 1200) -> str: + """Retain short diagnostic text, without credentials, URLs or binary payloads.""" + text = str(value) + for key, secret in os.environ.items(): + if secret and (_SENSITIVE.search(key) or re.search(r"(?:^|_)TOKEN(?:$|_)", key.upper())): + if len(secret) < 12: + text = re.sub(r"(?]+", "[url redacted]", text, flags=re.I) + text = re.sub(r"(? 6: + return "[nested data omitted]" + if isinstance(value, dict): + return { + safe_text(key, 100): ( + "[redacted]" + if _SENSITIVE.search(str(key)) or str(key).lower() in {"base64", "data", "body", "request", "traceback"} + else safe_value(item, depth + 1) + ) + for key, item in list(value.items())[:50] + } + if isinstance(value, (list, tuple)): + return [safe_value(item, depth + 1) for item in value[:20]] + if value is None or isinstance(value, (bool, int, float)): + return value + return safe_text(value) if isinstance(value, str) else f"[{type(value).__name__} omitted]" + + +def response_record(response): + """Keep successful response artifacts compatible; sanitize failed envelopes.""" + if response is None: + return None + value = response.model_dump() + return value if response.success else safe_value(value) + + +def safe_image_path(source) -> str: + """Only retain bounded dataset-relative image identifiers in diagnostics.""" + source = str(source) + path = PurePosixPath(source) + if not source or len(source) > 300 or path.is_absolute() or ".." in path.parts: + return "[invalid image path]" + if any(character in source for character in ("\\", ":", "\n", "\r")): + return "[invalid image path]" + return safe_text(source, 300) + + +class ImageInputError(ValueError): + """Attach only the dataset-relative image identifier to a preprocessing error.""" + + def __init__(self, source, error): + super().__init__(safe_text(error)) + self.image_path = safe_image_path(source) + + +def error_record(error, *, stage: str, user_id=None, session_id=None, question_id=None, response=None) -> dict: + """Preserve stage, a bounded cause chain and stack locations, never frame locals.""" + chain, seen = [], set() + current = error + image_path = None + while current is not None and id(current) not in seen and len(chain) < 5: + seen.add(id(current)) + image_path = image_path or getattr(current, "image_path", None) + chain.append( + { + "type": type(current).__name__, + "reason": safe_text(current), + "traceback": [ + {"file": Path(frame.filename).name, "line": frame.lineno, "function": frame.name} + for frame in traceback.extract_tb(current.__traceback__)[-12:] + ], + }, + ) + current = current.__cause__ or (None if current.__suppress_context__ else current.__context__) + reason = response.answer if response is not None and not response.success else str(error) + return { + "stage": stage, + "user_id": safe_text(user_id, 120) if user_id is not None else None, + "session_id": safe_text(session_id, 120) if session_id is not None else None, + "question_id": safe_text(question_id, 120) if question_id is not None else None, + "image_path": image_path, + "error_type": type(error).__name__, + "reason": safe_value(reason), + "chain": chain, + } + + +def mean_tokens(total: dict, denominator: int) -> dict: + """Amortize observed counters over the explicit scheduled population.""" + return { + key: total[key] / denominator if denominator and total[key] is not None else None + for key in ("input_tokens", "output_tokens", "total_tokens", "cached_input_tokens", "noncached_input_tokens") + } + + +def judge_settings(app) -> dict: + """Export a whitelist from the validated application, not unresolved YAML/env text.""" + from judge_mobilemem.protocol import MEMBASE_REVISION, SOURCE_REVISION, protocol_id + + model = app.config.components["as_llm"]["judge"].model_dump() + mode = app.config.jobs["answer_judge"].model_dump().get("judge_protocol", "membase_default") + fields = ("backend", "model", "stream", "context_size", "max_retries", "retry_delay") + return { + **{key: safe_value(model[key]) for key in fields if key in model}, + "parameters": safe_value( + { + key: value + for key, value in (model.get("parameters") or {}).items() + if key in {"temperature", "max_tokens", "thinking_enable", "top_p"} + } + ), + "extra_body": {"enable_thinking": (model.get("extra_body") or {}).get("enable_thinking")}, + "client_kwargs": { + key: value for key, value in (model.get("client_kwargs") or {}).items() if key in {"timeout", "max_retries"} + }, + "protocol": protocol_id(mode), + "protocol_mode": mode, + "mobilemem_revision": SOURCE_REVISION, + "membase_revision": MEMBASE_REVISION, + } diff --git a/benchmark/mobilemem/run.py b/benchmark/mobilemem/run.py index 4e1be782..2ac98787 100644 --- a/benchmark/mobilemem/run.py +++ b/benchmark/mobilemem/run.py @@ -24,6 +24,15 @@ import yaml from reme import Application from reme.config import resolve_app_config +from benchmark.mobilemem._audit import ( + ImageInputError, + USAGE_POLICY, + error_record, + judge_settings, + mean_tokens, + response_record, +) + def timestamp(value: str) -> str: """Preserve published start-time precision in AgentScope ISO timestamps.""" @@ -128,10 +137,14 @@ def session_messages(session: dict, image_root: Path, aliases=None) -> list[dict if turn["content_type"] == "image" and not refs: raise ValueError("MobileMem image turn has no image reference") for source in refs: + try: + image = image_block(resolve_image(image_root, source, aliases)) + except Exception as error: # pylint: disable=broad-exception-caught + raise ImageInputError(source, error) from error content.extend( [ {"type": "text", "text": f"[Image ID: {source}]"}, - image_block(resolve_image(image_root, source, aliases)), + image, ], ) messages.append( @@ -329,76 +342,124 @@ def judge_config(workspace: Path) -> dict: def check_response(response): """Keep failed Jobs as technical errors rather than incorrect predictions.""" + if response.metadata.get("cancelled"): + raise asyncio.CancelledError("ReMe Job cancelled") if not response.success: raise RuntimeError("ReMe Job failed; see the saved response artifact.") return response +async def close_application(app, directory, *, user_id=None): + """Finish owned cleanup; keep its diagnostics separate from the primary failure.""" + if app is None: + return + closing = asyncio.create_task(app.close()) + try: + await asyncio.shield(closing) + except asyncio.CancelledError as cancellation: + if closing.cancelled(): + save(directory / "cleanup-error.json", error_record(cancellation, stage="cleanup", user_id=user_id)) + return + try: + await closing + except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught + save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id)) + raise + except Exception as error: # pylint: disable=broad-exception-caught + save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id)) + + async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) -> list[dict]: """Ingest the complete chronological history into one fresh user workspace.""" - app = Application(**candidate_config(directory / "workspace", model)) + app, response, session_id = None, None, None + stage = "startup" results, memory_usage, aliases = [], [], {} try: + app = Application(**candidate_config(directory / "workspace", model)) await app.start() observe_usage(app) observe_completion(app) for ordinal, session in enumerate(scene["sessions"]): + session_id, response = session["session_id"], None + stage = "preprocessing" with usage_scope() as measured: - response = await app.run_job( - "auto_memory", - messages=await asyncio.to_thread(session_messages, session, scene["image_root"], aliases), - session_id=session["session_id"], - date=timestamp(session["event_start_time"])[:10], - include_images=True, - include_source_sessions=True, - ) - save( - directory / "sessions" / f"{ordinal:05d}.json", - { - "session_id": session["session_id"], - "response": response.model_dump(), - "usage": aggregate_usage(measured), - }, - ) - check_response(response) + try: + messages = await asyncio.to_thread(session_messages, session, scene["image_root"], aliases) + stage = "memory" + response = await app.run_job( + "auto_memory", + messages=messages, + session_id=session_id, + date=timestamp(session["event_start_time"])[:10], + include_images=True, + include_source_sessions=True, + ) + check_response(response) + finally: + save( + directory / "sessions" / f"{ordinal:05d}.json", + { + "session_id": session_id, + "response": response_record(response), + "usage": aggregate_usage(measured), + }, + ) memory_usage.extend(measured) print(f"{scene['id']}: session {ordinal + 1}/{len(scene['sessions'])}", flush=True) - check_response(await app.run_job("index_update")) + stage, session_id, response = "index", None, None + with usage_scope() as measured: + try: + response = await app.run_job("index_update") + check_response(response) + finally: + save( + directory / "index-response.json", + {"response": response_record(response), "usage": aggregate_usage(measured)}, + ) + memory_usage.extend(measured) + # Compatibility marker: never write this for a partial history or failed index. save(directory / "memory_usage.json", aggregate_usage(memory_usage)) async def answer_question(ordinal, question): question_dir = directory / "questions" / f"{ordinal:04d}" + answer_stage, answer_response = "answer", None try: async with query_slots: with usage_scope() as measured: - response = await app.run_job( - "agentic_answer", - query=query_prompt(question["question"], "Use the available evidence tools."), - scene_id=scene["id"], - include_images=True, - include_source_sessions=True, - max_iters=30, - ) - answer_record = { - "question_id": question["question_id"], - "response": response.model_dump(), - "usage": aggregate_usage(measured), - } - save(question_dir / "answer-response.json", answer_record) - check_response(response) + try: + answer_response = await app.run_job( + "agentic_answer", + query=query_prompt(question["question"], "Use the available evidence tools."), + scene_id=scene["id"], + include_images=True, + include_source_sessions=True, + max_iters=30, + ) + check_response(answer_response) + finally: + answer_record = { + "question_id": question["question_id"], + "response": response_record(answer_response), + "usage": aggregate_usage(measured), + } + save(question_dir / "answer-response.json", answer_record) save(question_dir / "answer.json", answer_record) - prediction = response.answer - judgment = await judge.run_job( - "answer_judge", - question=question["question"], - reference=question["answer"], - prediction=prediction, - evidence=evidence_values(question), - ) - save(question_dir / "judge.json", judgment.model_dump()) - check_response(judgment) + prediction = answer_response.answer + answer_stage, answer_response = "judge", None + try: + answer_response = await judge.run_job( + "answer_judge", + question=question["question"], + reference=question["answer"], + prediction=prediction, + evidence=evidence_values(question), + ) + check_response(answer_response) + finally: + save(question_dir / "judge.json", response_record(answer_response)) + answer_stage = "metrics" metrics = score_answer(question["answer"], prediction) - metrics["llm_judge"] = judgment.metadata["llm_judge_score"] + metrics["llm_judge"] = answer_response.metadata["llm_judge_score"] result = { "question_id": question["question_id"], "category": question["question_type"], @@ -407,43 +468,61 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) "prediction": prediction, "metrics": metrics, "usage": aggregate_usage(measured), - "judge_parse_default_zero": judgment.metadata["parse_default_zero"], + "judge_parse_default_zero": answer_response.metadata["parse_default_zero"], } save(question_dir / "result.json", result) results.append(result) print(f"{scene['id']}: {question['question_id']} completed", flush=True) - except Exception as error: # pylint: disable=broad-exception-caught - save(question_dir / "error.json", {"error_type": type(error).__name__}) + except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught + save( + question_dir / "error.json", + error_record( + error, + stage=answer_stage, + user_id=scene["id"], + question_id=question["question_id"], + response=answer_response, + ), + ) + if isinstance(error, asyncio.CancelledError): + raise print(f"{scene['id']}: {question['question_id']} failed ({type(error).__name__})", flush=True) - await asyncio.gather(*(answer_question(i, question) for i, question in enumerate(scene["questions"]))) - except Exception as error: # pylint: disable=broad-exception-caught - save(directory / "error.json", {"error_type": type(error).__name__}) - print(f"{scene['id']}: ingestion failed ({type(error).__name__})", flush=True) + stage, response = "answers", None + tasks = [asyncio.create_task(answer_question(i, question)) for i, question in enumerate(scene["questions"])] + try: + await asyncio.gather(*tasks) + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught + save( + directory / "error.json", + error_record(error, stage=stage, user_id=scene["id"], session_id=session_id, response=response), + ) + if isinstance(error, asyncio.CancelledError): + raise + print(f"{scene['id']}: {stage} failed ({type(error).__name__})", flush=True) finally: - await app.close() - save(directory / "summary.json", {"completed": len(results), "expected": len(scene["questions"])}) + try: + await close_application(app, directory, user_id=scene["id"]) + finally: + save(directory / "summary.json", {"completed": len(results), "expected": len(scene["questions"])}) return results -async def run(scenes, output: Path, model: str, user_concurrency: int, query_concurrency: int) -> bool: - """Run independent users concurrently and save official metrics plus method cost.""" - judge = Application(**judge_config(output / "judge-workspace")) - rows = [] - user_slots, query_slots = asyncio.Semaphore(user_concurrency), asyncio.Semaphore(query_concurrency) - try: - await judge.start() - - async def user_job(scene): - async with user_slots: - rows.extend(await evaluate_user(scene, output / "users" / scene["id"], model, judge, query_slots)) - - await asyncio.gather(*(user_job(scene) for scene in scenes)) - finally: - await judge.close() +def summarize_run(scenes, output: Path, model: str) -> dict: + """Read each canonical receipt/result once, including a run interrupted after partial progress.""" expected = sum(len(scene["questions"]) for scene in scenes) - memory = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/memory_usage.json")] - answers = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/answer.json")] + rows = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/result.json")] + sessions = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/sessions/*.json")] + indexes = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/index-response.json")] + memory = [row["usage"] for row in sessions + indexes] + answers = [ + json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/answer-response.json") + ] answer_usage = [row["usage"] for row in answers] total = aggregate_usage(memory + answer_usage) summary = { @@ -457,31 +536,58 @@ async def run(scenes, output: Path, model: str, user_concurrency: int, query_con "memory_usage": aggregate_usage(memory), "answer_usage": aggregate_usage(answer_usage), "token_usage": total, - "mean_tokens_per_user": { - key: total[key] / len(memory) if memory and total[key] is not None else None - for key in ( - "input_tokens", - "output_tokens", - "total_tokens", - "cached_input_tokens", - "noncached_input_tokens", - ) - }, - "mean_tokens_per_question": { - key: total[key] / len(answers) if answers and total[key] is not None else None - for key in ( - "input_tokens", - "output_tokens", - "total_tokens", - "cached_input_tokens", - "noncached_input_tokens", - ) + "usage_policy": USAGE_POLICY, + "cleanup_error_count": len(list(output.glob("users/*/cleanup-error.json"))) + + int((output / "cleanup-error.json").exists()), + "usage_denominators": {"scheduled_users": len(scenes), "expected_questions": expected}, + "usage_counts": { + "attempted_users": len(list(output.glob("users/*/summary.json"))), + "attempted_sessions": len(sessions), + "successful_sessions": sum(bool((row.get("response") or {}).get("success")) for row in sessions), + "completed_ingestion_users": len(list(output.glob("users/*/memory_usage.json"))), + "attempted_answers": len(answers), + "successful_answers": sum(bool((row.get("response") or {}).get("success")) for row in answers), + "scored_questions": len(rows), }, + "mean_tokens_per_user": mean_tokens(total, len(scenes)), + "mean_tokens_per_question": mean_tokens(total, expected), "judge_parse_default_zero": sum(row["judge_parse_default_zero"] for row in rows), } save(output / "summary.json", summary) + return summary + + +async def run(scenes, output: Path, model: str, user_concurrency: int, query_concurrency: int) -> bool: + """Run independent users concurrently and save official metrics plus method cost.""" + judge = None + user_slots, query_slots = asyncio.Semaphore(user_concurrency), asyncio.Semaphore(query_concurrency) + try: + judge = Application(**judge_config(output / "judge-workspace")) + save(output / "judge-settings.json", judge_settings(judge)) + await judge.start() + + async def user_job(scene): + async with user_slots: + await evaluate_user(scene, output / "users" / scene["id"], model, judge, query_slots) + + tasks = [asyncio.create_task(user_job(scene)) for scene in scenes] + try: + await asyncio.gather(*tasks) + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught + save(output / "error.json", error_record(error, stage="run")) + raise + finally: + try: + await close_application(judge, output) + finally: + summary = summarize_run(scenes, output, model) print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True) - return len(rows) == expected + return summary["completed"] == summary["expected"] and summary["cleanup_error_count"] == 0 def main() -> int: diff --git a/plugins/mobilemem-judge/README.md b/plugins/mobilemem-judge/README.md index 001a9e8b..65faee79 100644 --- a/plugins/mobilemem-judge/README.md +++ b/plugins/mobilemem-judge/README.md @@ -16,9 +16,13 @@ and default-zero handling of successful responses without a valid label. remain failed Jobs. Judge usage is evaluation cost, not method cost. Defaults are `qwen3.8-max`, temperature 0, thinking disabled and 4,096 output tokens. +`JUDGE_MODEL_NAME` can override the Judge model independently of the candidate. Candidate methods and their workspaces are not dependencies of this plugin. The benchmark runner invokes this Job in a separate application and saves its -prompt, provider output and verdict alongside the candidate prediction. +prompt, provider output and verdict alongside the candidate prediction. Its +`judge-settings.json` records the validated model, generation/retry settings and +protocol identifier through a credential-free whitelist, rather than recording +only the default model name. Sources: [MemBase executable judge](https://github.com/zjunlp/MemBase/blob/a9ba9495d48541d3e68588ff097294cc7a1201bc/membase/inference_utils/prompts.py) and [MobileMem published strict rubric](https://github.com/zjunlp/MobileMem/blob/fa5cd13c9c659699df9caf650987ba2bea8ca085/omni/eval/eval/question_answering_and_judge_prompts.txt). diff --git a/tests/unit/test_mobilemem_audit.py b/tests/unit/test_mobilemem_audit.py new file mode 100644 index 00000000..c4295eb3 --- /dev/null +++ b/tests/unit/test_mobilemem_audit.py @@ -0,0 +1,423 @@ +"""Failure-path receipts and cost accounting, without provider or server calls.""" + +# pylint: disable=missing-function-docstring,protected-access +import asyncio +import json +from types import SimpleNamespace + +import pytest + +from benchmark.mobilemem import run +from benchmark.mobilemem._audit import ImageInputError, error_record, mean_tokens, response_record +from reme.schema import Response + + +def read(path): + return json.loads(path.read_text(encoding="utf-8")) + + +@pytest.fixture(name="flow") +def fixture_flow(tmp_path, monkeypatch): + scenes = [ + { + "id": "uuid_0", + "image_root": tmp_path, + "sessions": [{"session_id": "s0", "event_start_time": "2025-01-01T09:00:00"}], + "questions": [ + {"question_id": "q0", "question": "What?", "answer": "answer", "question_type": "visual_reasoning"}, + ], + }, + ] + state = SimpleNamespace( + scenes=scenes, + apps=[], + fail=None, + direct=False, + cancel=False, + cleanup_fail=False, + startup_fail=False, + constructor_fail=False, + calls=[], + ) + + class FakeApp: + """Mock one owned candidate or Judge application.""" + + def __init__(self, *, judge): + if state.constructor_fail and not judge: + raise ValueError("candidate construction failed") + self.judge, self.closed = judge, False + state.apps.append(self) + + async def start(self): + if state.startup_fail and not self.judge: + raise ValueError("candidate startup failed") + + async def close(self): + self.closed = True + if state.cleanup_fail and not self.judge: + raise ValueError("candidate cleanup failed") + + async def run_job(self, name, **kwargs): + identity = kwargs.get("session_id", kwargs.get("query", "")) + state.calls.append((name, kwargs)) + if name == "answer_judge": + assert run._CURRENT.get() is None # Judge traffic cannot join candidate cost. + elif name != "index_update": + run._CURRENT.get().append(run.normalized_usage({"prompt_tokens": 100, "completion_tokens": 20})) + await asyncio.sleep(0) # Force interleaving of independently scoped requests. + if state.fail in (name, identity): + if state.cancel: + raise asyncio.CancelledError("cancelled offline call") + if state.direct: + raise ValueError("direct offline failure") + return Response(success=False, answer="HTTP 400 input length exceeds limit") + if name == "answer_judge": + return Response(answer="1", metadata={"llm_judge_score": 1.0, "parse_default_zero": False}) + # This portable counter is deliberately different: never sum it with HTTP receipts. + return Response(answer="answer", metadata={"usage": {"total_tokens": 999999}}) + + monkeypatch.setattr(run, "Application", FakeApp) + monkeypatch.setattr(run, "candidate_config", lambda *_: {"judge": False}) + monkeypatch.setattr(run, "judge_config", lambda *_: {"judge": True}) + monkeypatch.setattr(run, "judge_settings", lambda _: {"model": "offline", "protocol": "offline"}) + monkeypatch.setattr(run, "session_messages", lambda *_: [{"original": "unchanged"}]) + monkeypatch.setattr(run, "observe_usage", lambda _: None) + monkeypatch.setattr(run, "observe_completion", lambda _: None) + + async def execute(): + complete = await run.run(scenes, tmp_path / "run", "candidate", 4, 16) + return complete, read(tmp_path / "run/summary.json") + + state.execute = execute + state.directory = tmp_path / "run/users/uuid_0" + return state + + +@pytest.mark.asyncio +async def test_failed_second_memory_keeps_both_requests(flow): + flow.scenes[0]["sessions"].append({"session_id": "s1", "event_start_time": "2025-01-02T09:00:00"}) + flow.fail = "s1" + complete, summary = await flow.execute() + assert not complete and summary["completed"] == 0 + assert summary["memory_usage"]["total_tokens"] == summary["token_usage"]["total_tokens"] == 240 + assert summary["token_usage"]["records"] == 2 + assert not (flow.directory / "memory_usage.json").exists() + assert read(flow.directory / "sessions/00001.json")["response"]["success"] is False + error = read(flow.directory / "error.json") + assert (error["stage"], error["user_id"], error["session_id"]) == ("memory", "uuid_0", "s1") + assert "input length" in error["reason"] and all(app.closed for app in flow.apps) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("direct", [False, True]) +async def test_failed_answer_keeps_memory_and_answer_cost_once(flow, direct): + flow.fail, flow.direct = "agentic_answer", direct + complete, summary = await flow.execute() + assert not complete and summary["completed"] == 0 and summary["metrics"]["llm_judge"] is None + assert summary["token_usage"]["total_tokens"] == 240 + assert summary["memory_usage"]["total_tokens"] == summary["answer_usage"]["total_tokens"] == 120 + assert summary["usage_counts"]["attempted_answers"] == 1 + assert summary["usage_counts"]["successful_answers"] == 0 + assert summary["mean_tokens_per_question"]["total_tokens"] == 240 + assert summary["usage_denominators"] == {"scheduled_users": 1, "expected_questions": 1} + assert summary["usage_policy"]["version"] == 2 + assert (flow.directory / "memory_usage.json").exists() + question = flow.directory / "questions/0000" + receipt = read(question / "answer-response.json") + assert receipt["usage"]["total_tokens"] == 120 + assert (receipt["response"] is None) is direct + assert not (question / "answer.json").exists() and not (question / "result.json").exists() + assert read(question / "error.json")["stage"] == "answer" + + +@pytest.mark.asyncio +async def test_direct_memory_exception_keeps_null_response_receipt(flow): + flow.fail, flow.direct = "auto_memory", True + _, summary = await flow.execute() + assert summary["memory_usage"]["total_tokens"] == 120 + receipt = read(flow.directory / "sessions/00000.json") + assert receipt["response"] is None and receipt["usage"]["records"] == 1 + assert not (flow.directory / "memory_usage.json").exists() + + +@pytest.mark.asyncio +async def test_preprocessing_failure_keeps_earlier_memory_and_image_identity(flow, monkeypatch): + flow.scenes[0]["sessions"].append({"session_id": "s1", "event_start_time": "2025-01-02T09:00:00"}) + + def messages(session, *_): + if session["session_id"] == "s1": + try: + raise FileNotFoundError("missing original image") + except FileNotFoundError as error: + raise ImageInputError("image/uid0/photo.png", error) from error + return [] + + monkeypatch.setattr(run, "session_messages", messages) + _, summary = await flow.execute() + assert summary["memory_usage"]["total_tokens"] == 120 + receipt = read(flow.directory / "sessions/00001.json") + assert receipt["response"] is None and receipt["usage"]["records"] == 0 + error = read(flow.directory / "error.json") + assert error["stage"] == "preprocessing" and error["image_path"] == "image/uid0/photo.png" + assert [cause["type"] for cause in error["chain"]] == ["ImageInputError", "FileNotFoundError"] + assert not (flow.directory / "memory_usage.json").exists() + + +@pytest.mark.asyncio +async def test_index_failure_is_saved_without_complete_memory_marker(flow): + flow.fail = "index_update" + _, summary = await flow.execute() + assert summary["memory_usage"]["total_tokens"] == 120 + assert read(flow.directory / "index-response.json")["response"]["success"] is False + assert read(flow.directory / "error.json")["stage"] == "index" + assert not (flow.directory / "memory_usage.json").exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("direct", [False, True]) +async def test_judge_failure_preserves_answer_without_double_counting(flow, direct): + flow.fail, flow.direct = "answer_judge", direct + complete, summary = await flow.execute() + assert not complete and summary["token_usage"]["total_tokens"] == 240 + assert summary["answer_usage"]["records"] == 1 + question = flow.directory / "questions/0000" + assert read(question / "answer.json") == read(question / "answer-response.json") + assert not (question / "result.json").exists() + assert read(question / "error.json")["stage"] == "judge" + assert (read(question / "judge.json") is None) is direct + + +@pytest.mark.asyncio +async def test_all_success_old_payloads_metrics_and_usage_stay_compatible(flow): + flow.scenes[0]["sessions"].append({"session_id": "s1", "event_start_time": "2025-01-02T09:00:00"}) + complete, summary = await flow.execute() + assert complete and summary["completed"] == summary["expected"] == 1 + assert summary["metrics"] == {"f1": 1.0, "bleu_1": 1.0, "llm_judge": 1.0} + assert summary["token_usage"]["total_tokens"] == 360 + assert summary["memory_usage"]["total_tokens"] == 240 + assert summary["answer_usage"]["total_tokens"] == 120 + assert summary["mean_tokens_per_user"]["total_tokens"] == summary["mean_tokens_per_question"]["total_tokens"] == 360 + assert summary["token_usage"]["cached_input_tokens"] is None + assert summary["token_usage"]["coverage"]["cached_input_tokens"] == 0 + assert read(flow.directory / "memory_usage.json") == summary["memory_usage"] + assert [name for name, _ in flow.calls] == [ + "auto_memory", + "auto_memory", + "index_update", + "agentic_answer", + "answer_judge", + ] + question = flow.directory / "questions/0000" + assert read(question / "answer.json") == read(question / "answer-response.json") + result = read(question / "result.json") + assert result == { + "question_id": "q0", + "category": "visual_reasoning", + "question": "What?", + "reference": "answer", + "prediction": "answer", + "metrics": summary["metrics"], + "usage": summary["answer_usage"], + "judge_parse_default_zero": False, + } + + +@pytest.mark.asyncio +async def test_concurrent_questions_keep_cost_scopes_and_scheduled_denominator(flow): + flow.scenes[0]["questions"].append({**flow.scenes[0]["questions"][0], "question_id": "q1"}) + flow.scenes.append({**flow.scenes[0], "id": "uuid_1"}) + complete, summary = await flow.execute() + assert complete and summary["completed"] == 4 + assert summary["token_usage"]["records"] == 6 and summary["token_usage"]["total_tokens"] == 720 + assert summary["mean_tokens_per_user"]["total_tokens"] == 360 + assert summary["mean_tokens_per_question"]["total_tokens"] == 180 + + +@pytest.mark.asyncio +async def test_failure_means_use_scheduled_not_successful_denominators(flow): + flow.scenes[0]["questions"].append({**flow.scenes[0]["questions"][0], "question_id": "q1"}) + flow.fail = "agentic_answer" + _, summary = await flow.execute() + assert summary["completed"] == 0 and summary["expected"] == 2 + assert summary["token_usage"]["total_tokens"] == 360 + assert summary["mean_tokens_per_question"]["total_tokens"] == 180 + + +@pytest.mark.asyncio +async def test_empty_scheduled_population_preserves_unknown_means(flow): + flow.scenes.clear() + _, summary = await flow.execute() + assert summary["usage_denominators"] == {"scheduled_users": 0, "expected_questions": 0} + assert all(value is None for value in summary["mean_tokens_per_user"].values()) + assert all(value is None for value in summary["mean_tokens_per_question"].values()) + assert summary["token_usage"]["records"] == 0 + + +@pytest.mark.asyncio +async def test_summary_ignores_legacy_answer_and_result_usage_copies(flow, monkeypatch): + original = run.close_application + + async def close(app, directory, **kwargs): + await original(app, directory, **kwargs) + if app is not None and app.judge: + for name in ("answer.json", "result.json"): + path = flow.directory / "questions/0000" / name + data = read(path) + data["usage"] = {"total_tokens": 999999} + run.save(path, data) + + monkeypatch.setattr(run, "close_application", close) + _, summary = await flow.execute() + assert summary["token_usage"]["total_tokens"] == 240 + assert summary["answer_usage"]["records"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("constructor", [False, True]) +async def test_startup_and_cleanup_failures_are_separate(flow, constructor): + flow.constructor_fail, flow.startup_fail = constructor, not constructor + flow.cleanup_fail = True + complete, summary = await flow.execute() + assert not complete and summary["token_usage"]["total_tokens"] is None + assert read(flow.directory / "error.json")["stage"] == "startup" + assert (flow.directory / "cleanup-error.json").exists() is not constructor + assert all(app.closed for app in flow.apps) + + +@pytest.mark.asyncio +async def test_cancelled_request_finalizes_once_and_closes_owned_apps(flow): + flow.fail, flow.cancel = "auto_memory", True + with pytest.raises(asyncio.CancelledError): + await flow.execute() + receipt = read(flow.directory / "sessions/00000.json") + assert receipt["response"] is None and receipt["usage"]["total_tokens"] == 120 + assert read(flow.directory / "error.json")["error_type"] == "CancelledError" + assert all(app.closed for app in flow.apps) + assert read(flow.directory.parents[1] / "summary.json")["memory_usage"]["total_tokens"] == 120 + + +def test_error_artifacts_remove_secrets_payloads_and_locals(monkeypatch): + secret = "offline-sensitive-value" + monkeypatch.setenv("LLM_API_KEY", secret) + binary = "YWJj" * 100 + response = Response( + success=False, + answer=f"input length error api_key={secret} data:image/png;base64,{binary}", + metadata={ + "headers": {"X-Private": "hidden-header"}, + "request": {"body": "hidden-body"}, + "credential": {"token": "hidden-credential"}, + "traceback": "hidden-local", + "status_code": 400, + }, + ) + try: + raise ValueError(f"https://user:password@example.invalid/?key={secret} Bearer token-value sk-private-key") + except ValueError as error: + record = error_record(error, stage="answer", user_id="uuid_0", question_id="q0", response=response) + serialized = json.dumps([record, response_record(response)]) + for value in ( + secret, + binary, + "hidden-header", + "hidden-body", + "hidden-credential", + "hidden-local", + "token-value", + "sk-private-key", + "user:password", + ): + assert value not in serialized + assert "input length" in record["reason"] + assert set(record["chain"][0]["traceback"][0]) == {"file", "line", "function"} + assert ImageInputError("../../private.png", ValueError("bad")).image_path == "[invalid image path]" + successful = Response(answer="URL https://example.test/ and literal sk-example-text", metadata={"raw": binary}) + assert response_record(successful) == successful.model_dump() + + +@pytest.mark.parametrize("name", ["token", "refresh_token", "id_token", "access_token", "auth_token"]) +def test_failed_response_token_aliases_are_redacted_without_removing_usage_counters(name): + sentinel = "opaque-new-session" + response = Response( + success=False, + answer=f"{name}='{sentinel}'", + metadata={ + name: sentinel, + "input_tokens": 100, + "output_tokens": 20, + "cached_input_tokens": 30, + }, + ) + record = response_record(response) + assert sentinel not in json.dumps(record) + assert record["metadata"]["input_tokens"] == 100 + assert record["metadata"]["output_tokens"] == 20 + assert record["metadata"]["cached_input_tokens"] == 30 + + +@pytest.mark.parametrize("header", ["Cookie", "Set-Cookie"]) +def test_failed_response_redacts_whole_cookie_header_without_changing_ordinary_prose(header): + response = Response( + success=False, + answer=f"{header}: session=opaque; other=secondsecret\nThe cookie parser failed with status 400.", + ) + record = response_record(response) + assert "opaque" not in record["answer"] and "secondsecret" not in record["answer"] + assert "The cookie parser failed with status 400." in record["answer"] + + +@pytest.mark.asyncio +async def test_http_observer_includes_compressor_subsets_and_null_coverage_without_duplicate_wrapping(): + async def send(request): + await asyncio.sleep(0) + return SimpleNamespace(status_code=200, is_stream_consumed=True, json=lambda: {"usage": request}) + + clients = [SimpleNamespace(send=send) for _ in range(3)] + app = SimpleNamespace( + context=SimpleNamespace( + components={ + "as_llm": { + name: SimpleNamespace(model=SimpleNamespace(client=SimpleNamespace(_client=client))) + for name, client in zip(("default", "bench", "compressor"), clients) + } + } + ) + ) + run.observe_usage(app) + run.observe_usage(app) + + async def scoped(amount): + with run.usage_scope() as records: + await clients[0].send( + {"prompt_tokens": amount, "completion_tokens": 20, "prompt_tokens_details": {"cached_tokens": 30}} + ) + await clients[2].send( + {"prompt_tokens": 50, "completion_tokens": 10, "completion_tokens_details": {"reasoning_tokens": 4}} + ) + return run.aggregate_usage(records) + + first, second = await asyncio.gather(scoped(100), scoped(200)) + assert first["records"] == second["records"] == 2 + assert (first["total_tokens"], second["total_tokens"]) == (180, 280) + assert first["cached_input_tokens"] == 30 and first["noncached_input_tokens"] == 70 + assert first["coverage"]["cached_input_tokens"] == 1 and first["reasoning_tokens"] == 4 + assert mean_tokens(first, 0)["total_tokens"] is None + + +@pytest.mark.asyncio +async def test_cleanup_error_does_not_replace_primary_failure(flow): + flow.fail, flow.cleanup_fail = "auto_memory", True + _, summary = await flow.execute() + assert summary["memory_usage"]["total_tokens"] == 120 + assert read(flow.directory / "error.json")["stage"] == "memory" + assert read(flow.directory / "cleanup-error.json")["stage"] == "cleanup" + + +@pytest.mark.asyncio +async def test_cleanup_failure_keeps_valid_scores_but_does_not_report_operational_success(flow): + flow.cleanup_fail = True + complete, summary = await flow.execute() + assert not complete and summary["completed"] == summary["expected"] == 1 + assert summary["cleanup_error_count"] == 1 and summary["metrics"]["llm_judge"] == 1.0 + assert summary["token_usage"]["total_tokens"] == 240 diff --git a/tests/unit/test_mobilemem_benchmark.py b/tests/unit/test_mobilemem_benchmark.py index aa003009..3b1fc357 100644 --- a/tests/unit/test_mobilemem_benchmark.py +++ b/tests/unit/test_mobilemem_benchmark.py @@ -148,6 +148,7 @@ async def test_candidate_and_judge_start_with_native_settings(tmp_path, monkeypa monkeypatch.setattr("reme.config.config_parser.find_entry_points", installed) monkeypatch.setenv("LLM_API_KEY", "offline") monkeypatch.setenv("LLM_BASE_URL", "https://provider.invalid/v1") + monkeypatch.setenv("JUDGE_MODEL_NAME", "offline-judge-override") candidate_app = Application(**mobilemem.candidate_config(tmp_path / "candidate", "candidate-model")) judge_app = Application(**mobilemem.judge_config(tmp_path / "judge")) try: @@ -155,6 +156,11 @@ async def test_candidate_and_judge_start_with_native_settings(tmp_path, monkeypa await judge_app.start() assert set(judge_app.context.jobs) == {"answer_judge"} assert set(judge_app.context.components) == {"as_llm"} + audit = mobilemem.judge_settings(judge_app) + assert audit["model"] == "offline-judge-override" + assert audit["parameters"] == {"max_tokens": 4096, "temperature": 0, "thinking_enable": False} + assert audit["protocol"] == "mobilemem-omni-membase-default-judge-v1" + assert "credential" not in json.dumps(audit) and "provider.invalid" not in json.dumps(audit) models = candidate_app.context.components["as_llm"] for component in models.values(): assert component.kwargs["model"] == "candidate-model"