fix(benchmark): preserve MobileMem usage and failure diagnostics

This commit is contained in:
wang-qisen 2026-10-19 22:50:46 +08:00
parent c249ce5691
commit 3c6222ff1c
7 changed files with 829 additions and 99 deletions

View file

@ -43,9 +43,25 @@ MemBase executable-default `mobilemem-omni-judge` rubric, `qwen3.8-max`, tempera
Each user's `workspace/` retains notes, source JSONL, images and native AgentScope trajectories.
`sessions/*.json` and `questions/*/` retain responses, judgments, metrics and technical errors.
`summary.json` reports completed/expected questions, scores, memory/answer token totals and averages.
Provider usage includes compression; absent cache fields stay unknown. Judge usage is excluded from method cost.
Native interrupted replies and failed Jobs remain unscored technical gaps. The runner adds no fallback or retry queue.
`summary.json` reports completed/expected questions and scores over successfully scored results only.
Native interrupted replies and failed Jobs remain unscored technical gaps, not zero-score answers.
The runner adds no fallback or retry queue.
Usage policy version 2 counts observed successful HTTP requests, including compression and requests made before
a candidate history or answer later fails. Memory cost comes from `sessions/*.json` and `index-response.json`;
answer cost comes only from `answer-response.json`, including failed responses or `response: null` when a call raises.
Copies in `answer.json`, result files and response metadata are not counted again. `memory_usage.json` remains a
completion marker: it is written only after the full history and index update succeed.
Judge usage is excluded; absent counters stay unknown and each counter includes its observation coverage.
The `mean_tokens_per_user` / `mean_tokens_per_question` fields amortize this run's observed cost over scheduled users /
expected questions, with explicit denominators and attempt/completion counts. They are not successful-sample averages.
Earlier selected-success-only reports retain their original accounting policy.
Errors retain the failing stage, user/session/question identifiers, safe image paths, sanitized reasons and stack
locations; credentials, request headers, image payloads and frame locals are omitted. Cleanup errors are separate;
`cleanup_error_count` makes the run exit with status 2 without discarding already scored results.
`judge-settings.json` records a credential-free whitelist of the validated Judge model, parameters and protocol,
including an optional `JUDGE_MODEL_NAME` override, so scoring can be traced to the effective configuration.
Sources: [MobileMem evaluation code](https://github.com/zjunlp/MobileMem/tree/fa5cd13c9c659699df9caf650987ba2bea8ca085/omni/eval/eval)
and [MemBase judge](https://github.com/zjunlp/MemBase/blob/a9ba9495d48541d3e68588ff097294cc7a1201bc/membase/inference_utils/prompts.py).

View file

@ -41,8 +41,20 @@ temperature 为 0,开启 thinking,输出预算 65,536,上下文为 1,000,0
每个用户的 `workspace/` 保存笔记、原始 JSONL、图像及原生 AgentScope 轨迹。
`sessions/*.json` 与 `questions/*/` 保存响应、评分、指标和技术错误。
`summary.json` 汇总完成/应测题数、指标、记忆与回答 token 总量及平均值。
用量覆盖 provider 返回的压缩调用;缺失的 cache 字段保持未知。Judge 用量不计入方法成本。
原生中断与失败 Job 作为未评分的技术缺口保存;runner 不额外加入 fallback 或重试队列。
`summary.json` 汇总完成/应测题数,指标只对成功评分的结果求均值。
原生中断与失败 Job 保留为未评分的技术缺口,不作为零分答案;runner 不额外加入 fallback 或重试队列。
usage policy 第 2 版统计已观测的成功 HTTP 请求,包括压缩调用及摄入/回答最终失败前已经产生的请求。
记忆成本只汇总 `sessions/*.json` 和 `index-response.json`;回答成本只汇总 `answer-response.json`,
包括失败响应及直接异常时的 `response: null`。`answer.json`、result 和 response metadata 中的副本不重复计费。
`memory_usage.json` 仍仅在完整历史摄入及索引更新成功后生成,不能将它误作部分摄入标记。
Judge 用量不计入方法成本;缺失字段保持未知,并记录每个字段的观测覆盖数。
`mean_tokens_per_user` / `mean_tokens_per_question` 是本次已观测成本按计划用户数/应测题数均摊,
summary 明确记录分母和尝试/完成数,不代表成功样本平均成本。旧的 selected-success-only 报告不改写口径。
错误记录包含失败阶段、用户/会话/题号、安全的图片相对路径、脱敏原因和调用栈位置,不保存凭据、请求 headers、
图片载荷或 frame locals;清理错误单独保留,`cleanup_error_count` 使本轮以状态码 2 退出,但不丢弃已评分结果。
`judge-settings.json` 从实际校验后的配置中保存 Judge 模型、参数和协议白名单,
包括可选的 `JUDGE_MODEL_NAME` 覆盖值,不导出凭据。
协议来源、数据版本和改编代码的完整 MIT 许可证见 [英文说明](README.md)。

View file

@ -0,0 +1,163 @@
"""Small, credential-safe diagnostics and accounting helpers for this runner."""
import os
from pathlib import Path, PurePosixPath
import re
import traceback
USAGE_POLICY = {
"version": 2,
"scope": "observed successful HTTP requests, including failed candidate flows and compression; judge excluded",
"memory_sources": ["users/*/sessions/*.json", "users/*/index-response.json"],
"answer_source": "users/*/questions/*/answer-response.json",
"means": "observed run cost amortized over scheduled users / expected questions, not successful samples",
}
_SENSITIVE = re.compile(
r"authorization|credential|secret|password|api.?key|cookie|headers|"
r"(?:^|[_-])(?:access[_-]?token|refresh[_-]?token|id[_-]?token|token)$",
re.I,
)
def safe_text(value, limit: int = 1200) -> str:
"""Retain short diagnostic text, without credentials, URLs or binary payloads."""
text = str(value)
for key, secret in os.environ.items():
if secret and (_SENSITIVE.search(key) or re.search(r"(?:^|_)TOKEN(?:$|_)", key.upper())):
if len(secret) < 12:
text = re.sub(r"(?<![\w-])" + re.escape(secret) + r"(?![\w-])", "[redacted]", text)
else:
text = text.replace(secret, "[redacted]")
text = re.sub(
r"-----BEGIN [^-]*PRIVATE KEY-----.*?-----END [^-]*PRIVATE KEY-----", "[private key redacted]", text, flags=re.S
)
text = re.sub(r"(?i)\b(?:request[_ -]?)?headers\s*[:=].*", "headers=[redacted]", text)
text = re.sub(r"(?i)\b(?:set-cookie|cookie)\s*:[^\r\n]*", "cookie: [redacted]", text)
text = re.sub(r"data:[^\s,;]+(?:;[^\s,]*)?,[^\s\"']+", "[image data redacted]", text, flags=re.I)
text = re.sub(r"https?://[^\s\"'<>]+", "[url redacted]", text, flags=re.I)
text = re.sub(r"(?<![\w])(?:[A-Za-z]:\\|/)[^\s\"']+", "[path redacted]", text)
text = re.sub(r"\b(?:Bearer|Basic)\s+[^\s,;\"'}]+", "[authorization redacted]", text, flags=re.I)
text = re.sub(
r"(?i)([\"']?(?:api[_-]?key|authorization|password|secret|access[_-]?token|"
r"refresh[_-]?token|id[_-]?token|token|cookie)[\"']?\s*[:=]\s*)"
r"(?:[\"'][^\"']*[\"']|[^\s,;}]+)",
r"\1[redacted]",
text,
)
text = re.sub(r"\b(?:sk-|sk_|ghp_|github_pat_)[A-Za-z0-9_-]+", "[key redacted]", text)
text = re.sub(r"[A-Za-z0-9+/=_-]{80,}", "[payload redacted]", text)
return text[:limit]
def safe_value(value, depth: int = 0):
"""Sanitize failed response envelopes, never serialize arbitrary objects/locals."""
if depth > 6:
return "[nested data omitted]"
if isinstance(value, dict):
return {
safe_text(key, 100): (
"[redacted]"
if _SENSITIVE.search(str(key)) or str(key).lower() in {"base64", "data", "body", "request", "traceback"}
else safe_value(item, depth + 1)
)
for key, item in list(value.items())[:50]
}
if isinstance(value, (list, tuple)):
return [safe_value(item, depth + 1) for item in value[:20]]
if value is None or isinstance(value, (bool, int, float)):
return value
return safe_text(value) if isinstance(value, str) else f"[{type(value).__name__} omitted]"
def response_record(response):
"""Keep successful response artifacts compatible; sanitize failed envelopes."""
if response is None:
return None
value = response.model_dump()
return value if response.success else safe_value(value)
def safe_image_path(source) -> str:
"""Only retain bounded dataset-relative image identifiers in diagnostics."""
source = str(source)
path = PurePosixPath(source)
if not source or len(source) > 300 or path.is_absolute() or ".." in path.parts:
return "[invalid image path]"
if any(character in source for character in ("\\", ":", "\n", "\r")):
return "[invalid image path]"
return safe_text(source, 300)
class ImageInputError(ValueError):
"""Attach only the dataset-relative image identifier to a preprocessing error."""
def __init__(self, source, error):
super().__init__(safe_text(error))
self.image_path = safe_image_path(source)
def error_record(error, *, stage: str, user_id=None, session_id=None, question_id=None, response=None) -> dict:
"""Preserve stage, a bounded cause chain and stack locations, never frame locals."""
chain, seen = [], set()
current = error
image_path = None
while current is not None and id(current) not in seen and len(chain) < 5:
seen.add(id(current))
image_path = image_path or getattr(current, "image_path", None)
chain.append(
{
"type": type(current).__name__,
"reason": safe_text(current),
"traceback": [
{"file": Path(frame.filename).name, "line": frame.lineno, "function": frame.name}
for frame in traceback.extract_tb(current.__traceback__)[-12:]
],
},
)
current = current.__cause__ or (None if current.__suppress_context__ else current.__context__)
reason = response.answer if response is not None and not response.success else str(error)
return {
"stage": stage,
"user_id": safe_text(user_id, 120) if user_id is not None else None,
"session_id": safe_text(session_id, 120) if session_id is not None else None,
"question_id": safe_text(question_id, 120) if question_id is not None else None,
"image_path": image_path,
"error_type": type(error).__name__,
"reason": safe_value(reason),
"chain": chain,
}
def mean_tokens(total: dict, denominator: int) -> dict:
"""Amortize observed counters over the explicit scheduled population."""
return {
key: total[key] / denominator if denominator and total[key] is not None else None
for key in ("input_tokens", "output_tokens", "total_tokens", "cached_input_tokens", "noncached_input_tokens")
}
def judge_settings(app) -> dict:
"""Export a whitelist from the validated application, not unresolved YAML/env text."""
from judge_mobilemem.protocol import MEMBASE_REVISION, SOURCE_REVISION, protocol_id
model = app.config.components["as_llm"]["judge"].model_dump()
mode = app.config.jobs["answer_judge"].model_dump().get("judge_protocol", "membase_default")
fields = ("backend", "model", "stream", "context_size", "max_retries", "retry_delay")
return {
**{key: safe_value(model[key]) for key in fields if key in model},
"parameters": safe_value(
{
key: value
for key, value in (model.get("parameters") or {}).items()
if key in {"temperature", "max_tokens", "thinking_enable", "top_p"}
}
),
"extra_body": {"enable_thinking": (model.get("extra_body") or {}).get("enable_thinking")},
"client_kwargs": {
key: value for key, value in (model.get("client_kwargs") or {}).items() if key in {"timeout", "max_retries"}
},
"protocol": protocol_id(mode),
"protocol_mode": mode,
"mobilemem_revision": SOURCE_REVISION,
"membase_revision": MEMBASE_REVISION,
}

View file

@ -24,6 +24,15 @@ import yaml
from reme import Application
from reme.config import resolve_app_config
from benchmark.mobilemem._audit import (
ImageInputError,
USAGE_POLICY,
error_record,
judge_settings,
mean_tokens,
response_record,
)
def timestamp(value: str) -> str:
"""Preserve published start-time precision in AgentScope ISO timestamps."""
@ -128,10 +137,14 @@ def session_messages(session: dict, image_root: Path, aliases=None) -> list[dict
if turn["content_type"] == "image" and not refs:
raise ValueError("MobileMem image turn has no image reference")
for source in refs:
try:
image = image_block(resolve_image(image_root, source, aliases))
except Exception as error: # pylint: disable=broad-exception-caught
raise ImageInputError(source, error) from error
content.extend(
[
{"type": "text", "text": f"[Image ID: {source}]"},
image_block(resolve_image(image_root, source, aliases)),
image,
],
)
messages.append(
@ -329,76 +342,124 @@ def judge_config(workspace: Path) -> dict:
def check_response(response):
"""Keep failed Jobs as technical errors rather than incorrect predictions."""
if response.metadata.get("cancelled"):
raise asyncio.CancelledError("ReMe Job cancelled")
if not response.success:
raise RuntimeError("ReMe Job failed; see the saved response artifact.")
return response
async def close_application(app, directory, *, user_id=None):
"""Finish owned cleanup; keep its diagnostics separate from the primary failure."""
if app is None:
return
closing = asyncio.create_task(app.close())
try:
await asyncio.shield(closing)
except asyncio.CancelledError as cancellation:
if closing.cancelled():
save(directory / "cleanup-error.json", error_record(cancellation, stage="cleanup", user_id=user_id))
return
try:
await closing
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id))
raise
except Exception as error: # pylint: disable=broad-exception-caught
save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id))
async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) -> list[dict]:
"""Ingest the complete chronological history into one fresh user workspace."""
app = Application(**candidate_config(directory / "workspace", model))
app, response, session_id = None, None, None
stage = "startup"
results, memory_usage, aliases = [], [], {}
try:
app = Application(**candidate_config(directory / "workspace", model))
await app.start()
observe_usage(app)
observe_completion(app)
for ordinal, session in enumerate(scene["sessions"]):
session_id, response = session["session_id"], None
stage = "preprocessing"
with usage_scope() as measured:
response = await app.run_job(
"auto_memory",
messages=await asyncio.to_thread(session_messages, session, scene["image_root"], aliases),
session_id=session["session_id"],
date=timestamp(session["event_start_time"])[:10],
include_images=True,
include_source_sessions=True,
)
save(
directory / "sessions" / f"{ordinal:05d}.json",
{
"session_id": session["session_id"],
"response": response.model_dump(),
"usage": aggregate_usage(measured),
},
)
check_response(response)
try:
messages = await asyncio.to_thread(session_messages, session, scene["image_root"], aliases)
stage = "memory"
response = await app.run_job(
"auto_memory",
messages=messages,
session_id=session_id,
date=timestamp(session["event_start_time"])[:10],
include_images=True,
include_source_sessions=True,
)
check_response(response)
finally:
save(
directory / "sessions" / f"{ordinal:05d}.json",
{
"session_id": session_id,
"response": response_record(response),
"usage": aggregate_usage(measured),
},
)
memory_usage.extend(measured)
print(f"{scene['id']}: session {ordinal + 1}/{len(scene['sessions'])}", flush=True)
check_response(await app.run_job("index_update"))
stage, session_id, response = "index", None, None
with usage_scope() as measured:
try:
response = await app.run_job("index_update")
check_response(response)
finally:
save(
directory / "index-response.json",
{"response": response_record(response), "usage": aggregate_usage(measured)},
)
memory_usage.extend(measured)
# Compatibility marker: never write this for a partial history or failed index.
save(directory / "memory_usage.json", aggregate_usage(memory_usage))
async def answer_question(ordinal, question):
question_dir = directory / "questions" / f"{ordinal:04d}"
answer_stage, answer_response = "answer", None
try:
async with query_slots:
with usage_scope() as measured:
response = await app.run_job(
"agentic_answer",
query=query_prompt(question["question"], "Use the available evidence tools."),
scene_id=scene["id"],
include_images=True,
include_source_sessions=True,
max_iters=30,
)
answer_record = {
"question_id": question["question_id"],
"response": response.model_dump(),
"usage": aggregate_usage(measured),
}
save(question_dir / "answer-response.json", answer_record)
check_response(response)
try:
answer_response = await app.run_job(
"agentic_answer",
query=query_prompt(question["question"], "Use the available evidence tools."),
scene_id=scene["id"],
include_images=True,
include_source_sessions=True,
max_iters=30,
)
check_response(answer_response)
finally:
answer_record = {
"question_id": question["question_id"],
"response": response_record(answer_response),
"usage": aggregate_usage(measured),
}
save(question_dir / "answer-response.json", answer_record)
save(question_dir / "answer.json", answer_record)
prediction = response.answer
judgment = await judge.run_job(
"answer_judge",
question=question["question"],
reference=question["answer"],
prediction=prediction,
evidence=evidence_values(question),
)
save(question_dir / "judge.json", judgment.model_dump())
check_response(judgment)
prediction = answer_response.answer
answer_stage, answer_response = "judge", None
try:
answer_response = await judge.run_job(
"answer_judge",
question=question["question"],
reference=question["answer"],
prediction=prediction,
evidence=evidence_values(question),
)
check_response(answer_response)
finally:
save(question_dir / "judge.json", response_record(answer_response))
answer_stage = "metrics"
metrics = score_answer(question["answer"], prediction)
metrics["llm_judge"] = judgment.metadata["llm_judge_score"]
metrics["llm_judge"] = answer_response.metadata["llm_judge_score"]
result = {
"question_id": question["question_id"],
"category": question["question_type"],
@ -407,43 +468,61 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots)
"prediction": prediction,
"metrics": metrics,
"usage": aggregate_usage(measured),
"judge_parse_default_zero": judgment.metadata["parse_default_zero"],
"judge_parse_default_zero": answer_response.metadata["parse_default_zero"],
}
save(question_dir / "result.json", result)
results.append(result)
print(f"{scene['id']}: {question['question_id']} completed", flush=True)
except Exception as error: # pylint: disable=broad-exception-caught
save(question_dir / "error.json", {"error_type": type(error).__name__})
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
save(
question_dir / "error.json",
error_record(
error,
stage=answer_stage,
user_id=scene["id"],
question_id=question["question_id"],
response=answer_response,
),
)
if isinstance(error, asyncio.CancelledError):
raise
print(f"{scene['id']}: {question['question_id']} failed ({type(error).__name__})", flush=True)
await asyncio.gather(*(answer_question(i, question) for i, question in enumerate(scene["questions"])))
except Exception as error: # pylint: disable=broad-exception-caught
save(directory / "error.json", {"error_type": type(error).__name__})
print(f"{scene['id']}: ingestion failed ({type(error).__name__})", flush=True)
stage, response = "answers", None
tasks = [asyncio.create_task(answer_question(i, question)) for i, question in enumerate(scene["questions"])]
try:
await asyncio.gather(*tasks)
finally:
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
save(
directory / "error.json",
error_record(error, stage=stage, user_id=scene["id"], session_id=session_id, response=response),
)
if isinstance(error, asyncio.CancelledError):
raise
print(f"{scene['id']}: {stage} failed ({type(error).__name__})", flush=True)
finally:
await app.close()
save(directory / "summary.json", {"completed": len(results), "expected": len(scene["questions"])})
try:
await close_application(app, directory, user_id=scene["id"])
finally:
save(directory / "summary.json", {"completed": len(results), "expected": len(scene["questions"])})
return results
async def run(scenes, output: Path, model: str, user_concurrency: int, query_concurrency: int) -> bool:
"""Run independent users concurrently and save official metrics plus method cost."""
judge = Application(**judge_config(output / "judge-workspace"))
rows = []
user_slots, query_slots = asyncio.Semaphore(user_concurrency), asyncio.Semaphore(query_concurrency)
try:
await judge.start()
async def user_job(scene):
async with user_slots:
rows.extend(await evaluate_user(scene, output / "users" / scene["id"], model, judge, query_slots))
await asyncio.gather(*(user_job(scene) for scene in scenes))
finally:
await judge.close()
def summarize_run(scenes, output: Path, model: str) -> dict:
"""Read each canonical receipt/result once, including a run interrupted after partial progress."""
expected = sum(len(scene["questions"]) for scene in scenes)
memory = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/memory_usage.json")]
answers = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/answer.json")]
rows = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/result.json")]
sessions = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/sessions/*.json")]
indexes = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/index-response.json")]
memory = [row["usage"] for row in sessions + indexes]
answers = [
json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/answer-response.json")
]
answer_usage = [row["usage"] for row in answers]
total = aggregate_usage(memory + answer_usage)
summary = {
@ -457,31 +536,58 @@ async def run(scenes, output: Path, model: str, user_concurrency: int, query_con
"memory_usage": aggregate_usage(memory),
"answer_usage": aggregate_usage(answer_usage),
"token_usage": total,
"mean_tokens_per_user": {
key: total[key] / len(memory) if memory and total[key] is not None else None
for key in (
"input_tokens",
"output_tokens",
"total_tokens",
"cached_input_tokens",
"noncached_input_tokens",
)
},
"mean_tokens_per_question": {
key: total[key] / len(answers) if answers and total[key] is not None else None
for key in (
"input_tokens",
"output_tokens",
"total_tokens",
"cached_input_tokens",
"noncached_input_tokens",
)
"usage_policy": USAGE_POLICY,
"cleanup_error_count": len(list(output.glob("users/*/cleanup-error.json")))
+ int((output / "cleanup-error.json").exists()),
"usage_denominators": {"scheduled_users": len(scenes), "expected_questions": expected},
"usage_counts": {
"attempted_users": len(list(output.glob("users/*/summary.json"))),
"attempted_sessions": len(sessions),
"successful_sessions": sum(bool((row.get("response") or {}).get("success")) for row in sessions),
"completed_ingestion_users": len(list(output.glob("users/*/memory_usage.json"))),
"attempted_answers": len(answers),
"successful_answers": sum(bool((row.get("response") or {}).get("success")) for row in answers),
"scored_questions": len(rows),
},
"mean_tokens_per_user": mean_tokens(total, len(scenes)),
"mean_tokens_per_question": mean_tokens(total, expected),
"judge_parse_default_zero": sum(row["judge_parse_default_zero"] for row in rows),
}
save(output / "summary.json", summary)
return summary
async def run(scenes, output: Path, model: str, user_concurrency: int, query_concurrency: int) -> bool:
"""Run independent users concurrently and save official metrics plus method cost."""
judge = None
user_slots, query_slots = asyncio.Semaphore(user_concurrency), asyncio.Semaphore(query_concurrency)
try:
judge = Application(**judge_config(output / "judge-workspace"))
save(output / "judge-settings.json", judge_settings(judge))
await judge.start()
async def user_job(scene):
async with user_slots:
await evaluate_user(scene, output / "users" / scene["id"], model, judge, query_slots)
tasks = [asyncio.create_task(user_job(scene)) for scene in scenes]
try:
await asyncio.gather(*tasks)
finally:
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
save(output / "error.json", error_record(error, stage="run"))
raise
finally:
try:
await close_application(judge, output)
finally:
summary = summarize_run(scenes, output, model)
print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True)
return len(rows) == expected
return summary["completed"] == summary["expected"] and summary["cleanup_error_count"] == 0
def main() -> int:

View file

@ -16,9 +16,13 @@ and default-zero handling of successful responses without a valid label.
remain failed Jobs. Judge usage is evaluation cost, not method cost.
Defaults are `qwen3.8-max`, temperature 0, thinking disabled and 4,096 output tokens.
`JUDGE_MODEL_NAME` can override the Judge model independently of the candidate.
Candidate methods and their workspaces are not dependencies of this plugin.
The benchmark runner invokes this Job in a separate application and saves its
prompt, provider output and verdict alongside the candidate prediction.
prompt, provider output and verdict alongside the candidate prediction. Its
`judge-settings.json` records the validated model, generation/retry settings and
protocol identifier through a credential-free whitelist, rather than recording
only the default model name.
Sources: [MemBase executable judge](https://github.com/zjunlp/MemBase/blob/a9ba9495d48541d3e68588ff097294cc7a1201bc/membase/inference_utils/prompts.py)
and [MobileMem published strict rubric](https://github.com/zjunlp/MobileMem/blob/fa5cd13c9c659699df9caf650987ba2bea8ca085/omni/eval/eval/question_answering_and_judge_prompts.txt).

View file

@ -0,0 +1,423 @@
"""Failure-path receipts and cost accounting, without provider or server calls."""
# pylint: disable=missing-function-docstring,protected-access
import asyncio
import json
from types import SimpleNamespace
import pytest
from benchmark.mobilemem import run
from benchmark.mobilemem._audit import ImageInputError, error_record, mean_tokens, response_record
from reme.schema import Response
def read(path):
return json.loads(path.read_text(encoding="utf-8"))
@pytest.fixture(name="flow")
def fixture_flow(tmp_path, monkeypatch):
scenes = [
{
"id": "uuid_0",
"image_root": tmp_path,
"sessions": [{"session_id": "s0", "event_start_time": "2025-01-01T09:00:00"}],
"questions": [
{"question_id": "q0", "question": "What?", "answer": "answer", "question_type": "visual_reasoning"},
],
},
]
state = SimpleNamespace(
scenes=scenes,
apps=[],
fail=None,
direct=False,
cancel=False,
cleanup_fail=False,
startup_fail=False,
constructor_fail=False,
calls=[],
)
class FakeApp:
"""Mock one owned candidate or Judge application."""
def __init__(self, *, judge):
if state.constructor_fail and not judge:
raise ValueError("candidate construction failed")
self.judge, self.closed = judge, False
state.apps.append(self)
async def start(self):
if state.startup_fail and not self.judge:
raise ValueError("candidate startup failed")
async def close(self):
self.closed = True
if state.cleanup_fail and not self.judge:
raise ValueError("candidate cleanup failed")
async def run_job(self, name, **kwargs):
identity = kwargs.get("session_id", kwargs.get("query", ""))
state.calls.append((name, kwargs))
if name == "answer_judge":
assert run._CURRENT.get() is None # Judge traffic cannot join candidate cost.
elif name != "index_update":
run._CURRENT.get().append(run.normalized_usage({"prompt_tokens": 100, "completion_tokens": 20}))
await asyncio.sleep(0) # Force interleaving of independently scoped requests.
if state.fail in (name, identity):
if state.cancel:
raise asyncio.CancelledError("cancelled offline call")
if state.direct:
raise ValueError("direct offline failure")
return Response(success=False, answer="HTTP 400 input length exceeds limit")
if name == "answer_judge":
return Response(answer="1", metadata={"llm_judge_score": 1.0, "parse_default_zero": False})
# This portable counter is deliberately different: never sum it with HTTP receipts.
return Response(answer="answer", metadata={"usage": {"total_tokens": 999999}})
monkeypatch.setattr(run, "Application", FakeApp)
monkeypatch.setattr(run, "candidate_config", lambda *_: {"judge": False})
monkeypatch.setattr(run, "judge_config", lambda *_: {"judge": True})
monkeypatch.setattr(run, "judge_settings", lambda _: {"model": "offline", "protocol": "offline"})
monkeypatch.setattr(run, "session_messages", lambda *_: [{"original": "unchanged"}])
monkeypatch.setattr(run, "observe_usage", lambda _: None)
monkeypatch.setattr(run, "observe_completion", lambda _: None)
async def execute():
complete = await run.run(scenes, tmp_path / "run", "candidate", 4, 16)
return complete, read(tmp_path / "run/summary.json")
state.execute = execute
state.directory = tmp_path / "run/users/uuid_0"
return state
@pytest.mark.asyncio
async def test_failed_second_memory_keeps_both_requests(flow):
flow.scenes[0]["sessions"].append({"session_id": "s1", "event_start_time": "2025-01-02T09:00:00"})
flow.fail = "s1"
complete, summary = await flow.execute()
assert not complete and summary["completed"] == 0
assert summary["memory_usage"]["total_tokens"] == summary["token_usage"]["total_tokens"] == 240
assert summary["token_usage"]["records"] == 2
assert not (flow.directory / "memory_usage.json").exists()
assert read(flow.directory / "sessions/00001.json")["response"]["success"] is False
error = read(flow.directory / "error.json")
assert (error["stage"], error["user_id"], error["session_id"]) == ("memory", "uuid_0", "s1")
assert "input length" in error["reason"] and all(app.closed for app in flow.apps)
@pytest.mark.asyncio
@pytest.mark.parametrize("direct", [False, True])
async def test_failed_answer_keeps_memory_and_answer_cost_once(flow, direct):
flow.fail, flow.direct = "agentic_answer", direct
complete, summary = await flow.execute()
assert not complete and summary["completed"] == 0 and summary["metrics"]["llm_judge"] is None
assert summary["token_usage"]["total_tokens"] == 240
assert summary["memory_usage"]["total_tokens"] == summary["answer_usage"]["total_tokens"] == 120
assert summary["usage_counts"]["attempted_answers"] == 1
assert summary["usage_counts"]["successful_answers"] == 0
assert summary["mean_tokens_per_question"]["total_tokens"] == 240
assert summary["usage_denominators"] == {"scheduled_users": 1, "expected_questions": 1}
assert summary["usage_policy"]["version"] == 2
assert (flow.directory / "memory_usage.json").exists()
question = flow.directory / "questions/0000"
receipt = read(question / "answer-response.json")
assert receipt["usage"]["total_tokens"] == 120
assert (receipt["response"] is None) is direct
assert not (question / "answer.json").exists() and not (question / "result.json").exists()
assert read(question / "error.json")["stage"] == "answer"
@pytest.mark.asyncio
async def test_direct_memory_exception_keeps_null_response_receipt(flow):
flow.fail, flow.direct = "auto_memory", True
_, summary = await flow.execute()
assert summary["memory_usage"]["total_tokens"] == 120
receipt = read(flow.directory / "sessions/00000.json")
assert receipt["response"] is None and receipt["usage"]["records"] == 1
assert not (flow.directory / "memory_usage.json").exists()
@pytest.mark.asyncio
async def test_preprocessing_failure_keeps_earlier_memory_and_image_identity(flow, monkeypatch):
flow.scenes[0]["sessions"].append({"session_id": "s1", "event_start_time": "2025-01-02T09:00:00"})
def messages(session, *_):
if session["session_id"] == "s1":
try:
raise FileNotFoundError("missing original image")
except FileNotFoundError as error:
raise ImageInputError("image/uid0/photo.png", error) from error
return []
monkeypatch.setattr(run, "session_messages", messages)
_, summary = await flow.execute()
assert summary["memory_usage"]["total_tokens"] == 120
receipt = read(flow.directory / "sessions/00001.json")
assert receipt["response"] is None and receipt["usage"]["records"] == 0
error = read(flow.directory / "error.json")
assert error["stage"] == "preprocessing" and error["image_path"] == "image/uid0/photo.png"
assert [cause["type"] for cause in error["chain"]] == ["ImageInputError", "FileNotFoundError"]
assert not (flow.directory / "memory_usage.json").exists()
@pytest.mark.asyncio
async def test_index_failure_is_saved_without_complete_memory_marker(flow):
flow.fail = "index_update"
_, summary = await flow.execute()
assert summary["memory_usage"]["total_tokens"] == 120
assert read(flow.directory / "index-response.json")["response"]["success"] is False
assert read(flow.directory / "error.json")["stage"] == "index"
assert not (flow.directory / "memory_usage.json").exists()
@pytest.mark.asyncio
@pytest.mark.parametrize("direct", [False, True])
async def test_judge_failure_preserves_answer_without_double_counting(flow, direct):
flow.fail, flow.direct = "answer_judge", direct
complete, summary = await flow.execute()
assert not complete and summary["token_usage"]["total_tokens"] == 240
assert summary["answer_usage"]["records"] == 1
question = flow.directory / "questions/0000"
assert read(question / "answer.json") == read(question / "answer-response.json")
assert not (question / "result.json").exists()
assert read(question / "error.json")["stage"] == "judge"
assert (read(question / "judge.json") is None) is direct
@pytest.mark.asyncio
async def test_all_success_old_payloads_metrics_and_usage_stay_compatible(flow):
flow.scenes[0]["sessions"].append({"session_id": "s1", "event_start_time": "2025-01-02T09:00:00"})
complete, summary = await flow.execute()
assert complete and summary["completed"] == summary["expected"] == 1
assert summary["metrics"] == {"f1": 1.0, "bleu_1": 1.0, "llm_judge": 1.0}
assert summary["token_usage"]["total_tokens"] == 360
assert summary["memory_usage"]["total_tokens"] == 240
assert summary["answer_usage"]["total_tokens"] == 120
assert summary["mean_tokens_per_user"]["total_tokens"] == summary["mean_tokens_per_question"]["total_tokens"] == 360
assert summary["token_usage"]["cached_input_tokens"] is None
assert summary["token_usage"]["coverage"]["cached_input_tokens"] == 0
assert read(flow.directory / "memory_usage.json") == summary["memory_usage"]
assert [name for name, _ in flow.calls] == [
"auto_memory",
"auto_memory",
"index_update",
"agentic_answer",
"answer_judge",
]
question = flow.directory / "questions/0000"
assert read(question / "answer.json") == read(question / "answer-response.json")
result = read(question / "result.json")
assert result == {
"question_id": "q0",
"category": "visual_reasoning",
"question": "What?",
"reference": "answer",
"prediction": "answer",
"metrics": summary["metrics"],
"usage": summary["answer_usage"],
"judge_parse_default_zero": False,
}
@pytest.mark.asyncio
async def test_concurrent_questions_keep_cost_scopes_and_scheduled_denominator(flow):
flow.scenes[0]["questions"].append({**flow.scenes[0]["questions"][0], "question_id": "q1"})
flow.scenes.append({**flow.scenes[0], "id": "uuid_1"})
complete, summary = await flow.execute()
assert complete and summary["completed"] == 4
assert summary["token_usage"]["records"] == 6 and summary["token_usage"]["total_tokens"] == 720
assert summary["mean_tokens_per_user"]["total_tokens"] == 360
assert summary["mean_tokens_per_question"]["total_tokens"] == 180
@pytest.mark.asyncio
async def test_failure_means_use_scheduled_not_successful_denominators(flow):
flow.scenes[0]["questions"].append({**flow.scenes[0]["questions"][0], "question_id": "q1"})
flow.fail = "agentic_answer"
_, summary = await flow.execute()
assert summary["completed"] == 0 and summary["expected"] == 2
assert summary["token_usage"]["total_tokens"] == 360
assert summary["mean_tokens_per_question"]["total_tokens"] == 180
@pytest.mark.asyncio
async def test_empty_scheduled_population_preserves_unknown_means(flow):
flow.scenes.clear()
_, summary = await flow.execute()
assert summary["usage_denominators"] == {"scheduled_users": 0, "expected_questions": 0}
assert all(value is None for value in summary["mean_tokens_per_user"].values())
assert all(value is None for value in summary["mean_tokens_per_question"].values())
assert summary["token_usage"]["records"] == 0
@pytest.mark.asyncio
async def test_summary_ignores_legacy_answer_and_result_usage_copies(flow, monkeypatch):
original = run.close_application
async def close(app, directory, **kwargs):
await original(app, directory, **kwargs)
if app is not None and app.judge:
for name in ("answer.json", "result.json"):
path = flow.directory / "questions/0000" / name
data = read(path)
data["usage"] = {"total_tokens": 999999}
run.save(path, data)
monkeypatch.setattr(run, "close_application", close)
_, summary = await flow.execute()
assert summary["token_usage"]["total_tokens"] == 240
assert summary["answer_usage"]["records"] == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("constructor", [False, True])
async def test_startup_and_cleanup_failures_are_separate(flow, constructor):
flow.constructor_fail, flow.startup_fail = constructor, not constructor
flow.cleanup_fail = True
complete, summary = await flow.execute()
assert not complete and summary["token_usage"]["total_tokens"] is None
assert read(flow.directory / "error.json")["stage"] == "startup"
assert (flow.directory / "cleanup-error.json").exists() is not constructor
assert all(app.closed for app in flow.apps)
@pytest.mark.asyncio
async def test_cancelled_request_finalizes_once_and_closes_owned_apps(flow):
flow.fail, flow.cancel = "auto_memory", True
with pytest.raises(asyncio.CancelledError):
await flow.execute()
receipt = read(flow.directory / "sessions/00000.json")
assert receipt["response"] is None and receipt["usage"]["total_tokens"] == 120
assert read(flow.directory / "error.json")["error_type"] == "CancelledError"
assert all(app.closed for app in flow.apps)
assert read(flow.directory.parents[1] / "summary.json")["memory_usage"]["total_tokens"] == 120
def test_error_artifacts_remove_secrets_payloads_and_locals(monkeypatch):
secret = "offline-sensitive-value"
monkeypatch.setenv("LLM_API_KEY", secret)
binary = "YWJj" * 100
response = Response(
success=False,
answer=f"input length error api_key={secret} data:image/png;base64,{binary}",
metadata={
"headers": {"X-Private": "hidden-header"},
"request": {"body": "hidden-body"},
"credential": {"token": "hidden-credential"},
"traceback": "hidden-local",
"status_code": 400,
},
)
try:
raise ValueError(f"https://user:password@example.invalid/?key={secret} Bearer token-value sk-private-key")
except ValueError as error:
record = error_record(error, stage="answer", user_id="uuid_0", question_id="q0", response=response)
serialized = json.dumps([record, response_record(response)])
for value in (
secret,
binary,
"hidden-header",
"hidden-body",
"hidden-credential",
"hidden-local",
"token-value",
"sk-private-key",
"user:password",
):
assert value not in serialized
assert "input length" in record["reason"]
assert set(record["chain"][0]["traceback"][0]) == {"file", "line", "function"}
assert ImageInputError("../../private.png", ValueError("bad")).image_path == "[invalid image path]"
successful = Response(answer="URL https://example.test/ and literal sk-example-text", metadata={"raw": binary})
assert response_record(successful) == successful.model_dump()
@pytest.mark.parametrize("name", ["token", "refresh_token", "id_token", "access_token", "auth_token"])
def test_failed_response_token_aliases_are_redacted_without_removing_usage_counters(name):
sentinel = "opaque-new-session"
response = Response(
success=False,
answer=f"{name}='{sentinel}'",
metadata={
name: sentinel,
"input_tokens": 100,
"output_tokens": 20,
"cached_input_tokens": 30,
},
)
record = response_record(response)
assert sentinel not in json.dumps(record)
assert record["metadata"]["input_tokens"] == 100
assert record["metadata"]["output_tokens"] == 20
assert record["metadata"]["cached_input_tokens"] == 30
@pytest.mark.parametrize("header", ["Cookie", "Set-Cookie"])
def test_failed_response_redacts_whole_cookie_header_without_changing_ordinary_prose(header):
response = Response(
success=False,
answer=f"{header}: session=opaque; other=secondsecret\nThe cookie parser failed with status 400.",
)
record = response_record(response)
assert "opaque" not in record["answer"] and "secondsecret" not in record["answer"]
assert "The cookie parser failed with status 400." in record["answer"]
@pytest.mark.asyncio
async def test_http_observer_includes_compressor_subsets_and_null_coverage_without_duplicate_wrapping():
async def send(request):
await asyncio.sleep(0)
return SimpleNamespace(status_code=200, is_stream_consumed=True, json=lambda: {"usage": request})
clients = [SimpleNamespace(send=send) for _ in range(3)]
app = SimpleNamespace(
context=SimpleNamespace(
components={
"as_llm": {
name: SimpleNamespace(model=SimpleNamespace(client=SimpleNamespace(_client=client)))
for name, client in zip(("default", "bench", "compressor"), clients)
}
}
)
)
run.observe_usage(app)
run.observe_usage(app)
async def scoped(amount):
with run.usage_scope() as records:
await clients[0].send(
{"prompt_tokens": amount, "completion_tokens": 20, "prompt_tokens_details": {"cached_tokens": 30}}
)
await clients[2].send(
{"prompt_tokens": 50, "completion_tokens": 10, "completion_tokens_details": {"reasoning_tokens": 4}}
)
return run.aggregate_usage(records)
first, second = await asyncio.gather(scoped(100), scoped(200))
assert first["records"] == second["records"] == 2
assert (first["total_tokens"], second["total_tokens"]) == (180, 280)
assert first["cached_input_tokens"] == 30 and first["noncached_input_tokens"] == 70
assert first["coverage"]["cached_input_tokens"] == 1 and first["reasoning_tokens"] == 4
assert mean_tokens(first, 0)["total_tokens"] is None
@pytest.mark.asyncio
async def test_cleanup_error_does_not_replace_primary_failure(flow):
flow.fail, flow.cleanup_fail = "auto_memory", True
_, summary = await flow.execute()
assert summary["memory_usage"]["total_tokens"] == 120
assert read(flow.directory / "error.json")["stage"] == "memory"
assert read(flow.directory / "cleanup-error.json")["stage"] == "cleanup"
@pytest.mark.asyncio
async def test_cleanup_failure_keeps_valid_scores_but_does_not_report_operational_success(flow):
flow.cleanup_fail = True
complete, summary = await flow.execute()
assert not complete and summary["completed"] == summary["expected"] == 1
assert summary["cleanup_error_count"] == 1 and summary["metrics"]["llm_judge"] == 1.0
assert summary["token_usage"]["total_tokens"] == 240

View file

@ -148,6 +148,7 @@ async def test_candidate_and_judge_start_with_native_settings(tmp_path, monkeypa
monkeypatch.setattr("reme.config.config_parser.find_entry_points", installed)
monkeypatch.setenv("LLM_API_KEY", "offline")
monkeypatch.setenv("LLM_BASE_URL", "https://provider.invalid/v1")
monkeypatch.setenv("JUDGE_MODEL_NAME", "offline-judge-override")
candidate_app = Application(**mobilemem.candidate_config(tmp_path / "candidate", "candidate-model"))
judge_app = Application(**mobilemem.judge_config(tmp_path / "judge"))
try:
@ -155,6 +156,11 @@ async def test_candidate_and_judge_start_with_native_settings(tmp_path, monkeypa
await judge_app.start()
assert set(judge_app.context.jobs) == {"answer_judge"}
assert set(judge_app.context.components) == {"as_llm"}
audit = mobilemem.judge_settings(judge_app)
assert audit["model"] == "offline-judge-override"
assert audit["parameters"] == {"max_tokens": 4096, "temperature": 0, "thinking_enable": False}
assert audit["protocol"] == "mobilemem-omni-membase-default-judge-v1"
assert "credential" not in json.dumps(audit) and "provider.invalid" not in json.dumps(audit)
models = candidate_app.context.components["as_llm"]
for component in models.values():
assert component.kwargs["model"] == "candidate-model"