mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-18 09:42:31 +00:00
fix(benchmark): preserve MobileMem usage and failure diagnostics
This commit is contained in:
parent
c249ce5691
commit
3c6222ff1c
7 changed files with 829 additions and 99 deletions
|
|
@ -43,9 +43,25 @@ MemBase executable-default `mobilemem-omni-judge` rubric, `qwen3.8-max`, tempera
|
|||
|
||||
Each user's `workspace/` retains notes, source JSONL, images and native AgentScope trajectories.
|
||||
`sessions/*.json` and `questions/*/` retain responses, judgments, metrics and technical errors.
|
||||
`summary.json` reports completed/expected questions, scores, memory/answer token totals and averages.
|
||||
Provider usage includes compression; absent cache fields stay unknown. Judge usage is excluded from method cost.
|
||||
Native interrupted replies and failed Jobs remain unscored technical gaps. The runner adds no fallback or retry queue.
|
||||
`summary.json` reports completed/expected questions and scores over successfully scored results only.
|
||||
Native interrupted replies and failed Jobs remain unscored technical gaps, not zero-score answers.
|
||||
The runner adds no fallback or retry queue.
|
||||
|
||||
Usage policy version 2 counts observed successful HTTP requests, including compression and requests made before
|
||||
a candidate history or answer later fails. Memory cost comes from `sessions/*.json` and `index-response.json`;
|
||||
answer cost comes only from `answer-response.json`, including failed responses or `response: null` when a call raises.
|
||||
Copies in `answer.json`, result files and response metadata are not counted again. `memory_usage.json` remains a
|
||||
completion marker: it is written only after the full history and index update succeed.
|
||||
Judge usage is excluded; absent counters stay unknown and each counter includes its observation coverage.
|
||||
The `mean_tokens_per_user` / `mean_tokens_per_question` fields amortize this run's observed cost over scheduled users /
|
||||
expected questions, with explicit denominators and attempt/completion counts. They are not successful-sample averages.
|
||||
Earlier selected-success-only reports retain their original accounting policy.
|
||||
|
||||
Errors retain the failing stage, user/session/question identifiers, safe image paths, sanitized reasons and stack
|
||||
locations; credentials, request headers, image payloads and frame locals are omitted. Cleanup errors are separate;
|
||||
`cleanup_error_count` makes the run exit with status 2 without discarding already scored results.
|
||||
`judge-settings.json` records a credential-free whitelist of the validated Judge model, parameters and protocol,
|
||||
including an optional `JUDGE_MODEL_NAME` override, so scoring can be traced to the effective configuration.
|
||||
|
||||
Sources: [MobileMem evaluation code](https://github.com/zjunlp/MobileMem/tree/fa5cd13c9c659699df9caf650987ba2bea8ca085/omni/eval/eval)
|
||||
and [MemBase judge](https://github.com/zjunlp/MemBase/blob/a9ba9495d48541d3e68588ff097294cc7a1201bc/membase/inference_utils/prompts.py).
|
||||
|
|
|
|||
|
|
@ -41,8 +41,20 @@ temperature 为 0,开启 thinking,输出预算 65,536,上下文为 1,000,0
|
|||
|
||||
每个用户的 `workspace/` 保存笔记、原始 JSONL、图像及原生 AgentScope 轨迹。
|
||||
`sessions/*.json` 与 `questions/*/` 保存响应、评分、指标和技术错误。
|
||||
`summary.json` 汇总完成/应测题数、指标、记忆与回答 token 总量及平均值。
|
||||
用量覆盖 provider 返回的压缩调用;缺失的 cache 字段保持未知。Judge 用量不计入方法成本。
|
||||
原生中断与失败 Job 作为未评分的技术缺口保存;runner 不额外加入 fallback 或重试队列。
|
||||
`summary.json` 汇总完成/应测题数,指标只对成功评分的结果求均值。
|
||||
原生中断与失败 Job 保留为未评分的技术缺口,不作为零分答案;runner 不额外加入 fallback 或重试队列。
|
||||
|
||||
usage policy 第 2 版统计已观测的成功 HTTP 请求,包括压缩调用及摄入/回答最终失败前已经产生的请求。
|
||||
记忆成本只汇总 `sessions/*.json` 和 `index-response.json`;回答成本只汇总 `answer-response.json`,
|
||||
包括失败响应及直接异常时的 `response: null`。`answer.json`、result 和 response metadata 中的副本不重复计费。
|
||||
`memory_usage.json` 仍仅在完整历史摄入及索引更新成功后生成,不能将它误作部分摄入标记。
|
||||
Judge 用量不计入方法成本;缺失字段保持未知,并记录每个字段的观测覆盖数。
|
||||
`mean_tokens_per_user` / `mean_tokens_per_question` 是本次已观测成本按计划用户数/应测题数均摊,
|
||||
summary 明确记录分母和尝试/完成数,不代表成功样本平均成本。旧的 selected-success-only 报告不改写口径。
|
||||
|
||||
错误记录包含失败阶段、用户/会话/题号、安全的图片相对路径、脱敏原因和调用栈位置,不保存凭据、请求 headers、
|
||||
图片载荷或 frame locals;清理错误单独保留,`cleanup_error_count` 使本轮以状态码 2 退出,但不丢弃已评分结果。
|
||||
`judge-settings.json` 从实际校验后的配置中保存 Judge 模型、参数和协议白名单,
|
||||
包括可选的 `JUDGE_MODEL_NAME` 覆盖值,不导出凭据。
|
||||
|
||||
协议来源、数据版本和改编代码的完整 MIT 许可证见 [英文说明](README.md)。
|
||||
|
|
|
|||
163
benchmark/mobilemem/_audit.py
Normal file
163
benchmark/mobilemem/_audit.py
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
"""Small, credential-safe diagnostics and accounting helpers for this runner."""
|
||||
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import traceback
|
||||
|
||||
USAGE_POLICY = {
|
||||
"version": 2,
|
||||
"scope": "observed successful HTTP requests, including failed candidate flows and compression; judge excluded",
|
||||
"memory_sources": ["users/*/sessions/*.json", "users/*/index-response.json"],
|
||||
"answer_source": "users/*/questions/*/answer-response.json",
|
||||
"means": "observed run cost amortized over scheduled users / expected questions, not successful samples",
|
||||
}
|
||||
_SENSITIVE = re.compile(
|
||||
r"authorization|credential|secret|password|api.?key|cookie|headers|"
|
||||
r"(?:^|[_-])(?:access[_-]?token|refresh[_-]?token|id[_-]?token|token)$",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def safe_text(value, limit: int = 1200) -> str:
|
||||
"""Retain short diagnostic text, without credentials, URLs or binary payloads."""
|
||||
text = str(value)
|
||||
for key, secret in os.environ.items():
|
||||
if secret and (_SENSITIVE.search(key) or re.search(r"(?:^|_)TOKEN(?:$|_)", key.upper())):
|
||||
if len(secret) < 12:
|
||||
text = re.sub(r"(?<![\w-])" + re.escape(secret) + r"(?![\w-])", "[redacted]", text)
|
||||
else:
|
||||
text = text.replace(secret, "[redacted]")
|
||||
text = re.sub(
|
||||
r"-----BEGIN [^-]*PRIVATE KEY-----.*?-----END [^-]*PRIVATE KEY-----", "[private key redacted]", text, flags=re.S
|
||||
)
|
||||
text = re.sub(r"(?i)\b(?:request[_ -]?)?headers\s*[:=].*", "headers=[redacted]", text)
|
||||
text = re.sub(r"(?i)\b(?:set-cookie|cookie)\s*:[^\r\n]*", "cookie: [redacted]", text)
|
||||
text = re.sub(r"data:[^\s,;]+(?:;[^\s,]*)?,[^\s\"']+", "[image data redacted]", text, flags=re.I)
|
||||
text = re.sub(r"https?://[^\s\"'<>]+", "[url redacted]", text, flags=re.I)
|
||||
text = re.sub(r"(?<![\w])(?:[A-Za-z]:\\|/)[^\s\"']+", "[path redacted]", text)
|
||||
text = re.sub(r"\b(?:Bearer|Basic)\s+[^\s,;\"'}]+", "[authorization redacted]", text, flags=re.I)
|
||||
text = re.sub(
|
||||
r"(?i)([\"']?(?:api[_-]?key|authorization|password|secret|access[_-]?token|"
|
||||
r"refresh[_-]?token|id[_-]?token|token|cookie)[\"']?\s*[:=]\s*)"
|
||||
r"(?:[\"'][^\"']*[\"']|[^\s,;}]+)",
|
||||
r"\1[redacted]",
|
||||
text,
|
||||
)
|
||||
text = re.sub(r"\b(?:sk-|sk_|ghp_|github_pat_)[A-Za-z0-9_-]+", "[key redacted]", text)
|
||||
text = re.sub(r"[A-Za-z0-9+/=_-]{80,}", "[payload redacted]", text)
|
||||
return text[:limit]
|
||||
|
||||
|
||||
def safe_value(value, depth: int = 0):
|
||||
"""Sanitize failed response envelopes, never serialize arbitrary objects/locals."""
|
||||
if depth > 6:
|
||||
return "[nested data omitted]"
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
safe_text(key, 100): (
|
||||
"[redacted]"
|
||||
if _SENSITIVE.search(str(key)) or str(key).lower() in {"base64", "data", "body", "request", "traceback"}
|
||||
else safe_value(item, depth + 1)
|
||||
)
|
||||
for key, item in list(value.items())[:50]
|
||||
}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [safe_value(item, depth + 1) for item in value[:20]]
|
||||
if value is None or isinstance(value, (bool, int, float)):
|
||||
return value
|
||||
return safe_text(value) if isinstance(value, str) else f"[{type(value).__name__} omitted]"
|
||||
|
||||
|
||||
def response_record(response):
|
||||
"""Keep successful response artifacts compatible; sanitize failed envelopes."""
|
||||
if response is None:
|
||||
return None
|
||||
value = response.model_dump()
|
||||
return value if response.success else safe_value(value)
|
||||
|
||||
|
||||
def safe_image_path(source) -> str:
|
||||
"""Only retain bounded dataset-relative image identifiers in diagnostics."""
|
||||
source = str(source)
|
||||
path = PurePosixPath(source)
|
||||
if not source or len(source) > 300 or path.is_absolute() or ".." in path.parts:
|
||||
return "[invalid image path]"
|
||||
if any(character in source for character in ("\\", ":", "\n", "\r")):
|
||||
return "[invalid image path]"
|
||||
return safe_text(source, 300)
|
||||
|
||||
|
||||
class ImageInputError(ValueError):
|
||||
"""Attach only the dataset-relative image identifier to a preprocessing error."""
|
||||
|
||||
def __init__(self, source, error):
|
||||
super().__init__(safe_text(error))
|
||||
self.image_path = safe_image_path(source)
|
||||
|
||||
|
||||
def error_record(error, *, stage: str, user_id=None, session_id=None, question_id=None, response=None) -> dict:
|
||||
"""Preserve stage, a bounded cause chain and stack locations, never frame locals."""
|
||||
chain, seen = [], set()
|
||||
current = error
|
||||
image_path = None
|
||||
while current is not None and id(current) not in seen and len(chain) < 5:
|
||||
seen.add(id(current))
|
||||
image_path = image_path or getattr(current, "image_path", None)
|
||||
chain.append(
|
||||
{
|
||||
"type": type(current).__name__,
|
||||
"reason": safe_text(current),
|
||||
"traceback": [
|
||||
{"file": Path(frame.filename).name, "line": frame.lineno, "function": frame.name}
|
||||
for frame in traceback.extract_tb(current.__traceback__)[-12:]
|
||||
],
|
||||
},
|
||||
)
|
||||
current = current.__cause__ or (None if current.__suppress_context__ else current.__context__)
|
||||
reason = response.answer if response is not None and not response.success else str(error)
|
||||
return {
|
||||
"stage": stage,
|
||||
"user_id": safe_text(user_id, 120) if user_id is not None else None,
|
||||
"session_id": safe_text(session_id, 120) if session_id is not None else None,
|
||||
"question_id": safe_text(question_id, 120) if question_id is not None else None,
|
||||
"image_path": image_path,
|
||||
"error_type": type(error).__name__,
|
||||
"reason": safe_value(reason),
|
||||
"chain": chain,
|
||||
}
|
||||
|
||||
|
||||
def mean_tokens(total: dict, denominator: int) -> dict:
|
||||
"""Amortize observed counters over the explicit scheduled population."""
|
||||
return {
|
||||
key: total[key] / denominator if denominator and total[key] is not None else None
|
||||
for key in ("input_tokens", "output_tokens", "total_tokens", "cached_input_tokens", "noncached_input_tokens")
|
||||
}
|
||||
|
||||
|
||||
def judge_settings(app) -> dict:
|
||||
"""Export a whitelist from the validated application, not unresolved YAML/env text."""
|
||||
from judge_mobilemem.protocol import MEMBASE_REVISION, SOURCE_REVISION, protocol_id
|
||||
|
||||
model = app.config.components["as_llm"]["judge"].model_dump()
|
||||
mode = app.config.jobs["answer_judge"].model_dump().get("judge_protocol", "membase_default")
|
||||
fields = ("backend", "model", "stream", "context_size", "max_retries", "retry_delay")
|
||||
return {
|
||||
**{key: safe_value(model[key]) for key in fields if key in model},
|
||||
"parameters": safe_value(
|
||||
{
|
||||
key: value
|
||||
for key, value in (model.get("parameters") or {}).items()
|
||||
if key in {"temperature", "max_tokens", "thinking_enable", "top_p"}
|
||||
}
|
||||
),
|
||||
"extra_body": {"enable_thinking": (model.get("extra_body") or {}).get("enable_thinking")},
|
||||
"client_kwargs": {
|
||||
key: value for key, value in (model.get("client_kwargs") or {}).items() if key in {"timeout", "max_retries"}
|
||||
},
|
||||
"protocol": protocol_id(mode),
|
||||
"protocol_mode": mode,
|
||||
"mobilemem_revision": SOURCE_REVISION,
|
||||
"membase_revision": MEMBASE_REVISION,
|
||||
}
|
||||
|
|
@ -24,6 +24,15 @@ import yaml
|
|||
from reme import Application
|
||||
from reme.config import resolve_app_config
|
||||
|
||||
from benchmark.mobilemem._audit import (
|
||||
ImageInputError,
|
||||
USAGE_POLICY,
|
||||
error_record,
|
||||
judge_settings,
|
||||
mean_tokens,
|
||||
response_record,
|
||||
)
|
||||
|
||||
|
||||
def timestamp(value: str) -> str:
|
||||
"""Preserve published start-time precision in AgentScope ISO timestamps."""
|
||||
|
|
@ -128,10 +137,14 @@ def session_messages(session: dict, image_root: Path, aliases=None) -> list[dict
|
|||
if turn["content_type"] == "image" and not refs:
|
||||
raise ValueError("MobileMem image turn has no image reference")
|
||||
for source in refs:
|
||||
try:
|
||||
image = image_block(resolve_image(image_root, source, aliases))
|
||||
except Exception as error: # pylint: disable=broad-exception-caught
|
||||
raise ImageInputError(source, error) from error
|
||||
content.extend(
|
||||
[
|
||||
{"type": "text", "text": f"[Image ID: {source}]"},
|
||||
image_block(resolve_image(image_root, source, aliases)),
|
||||
image,
|
||||
],
|
||||
)
|
||||
messages.append(
|
||||
|
|
@ -329,76 +342,124 @@ def judge_config(workspace: Path) -> dict:
|
|||
|
||||
def check_response(response):
|
||||
"""Keep failed Jobs as technical errors rather than incorrect predictions."""
|
||||
if response.metadata.get("cancelled"):
|
||||
raise asyncio.CancelledError("ReMe Job cancelled")
|
||||
if not response.success:
|
||||
raise RuntimeError("ReMe Job failed; see the saved response artifact.")
|
||||
return response
|
||||
|
||||
|
||||
async def close_application(app, directory, *, user_id=None):
|
||||
"""Finish owned cleanup; keep its diagnostics separate from the primary failure."""
|
||||
if app is None:
|
||||
return
|
||||
closing = asyncio.create_task(app.close())
|
||||
try:
|
||||
await asyncio.shield(closing)
|
||||
except asyncio.CancelledError as cancellation:
|
||||
if closing.cancelled():
|
||||
save(directory / "cleanup-error.json", error_record(cancellation, stage="cleanup", user_id=user_id))
|
||||
return
|
||||
try:
|
||||
await closing
|
||||
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
|
||||
save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id))
|
||||
raise
|
||||
except Exception as error: # pylint: disable=broad-exception-caught
|
||||
save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id))
|
||||
|
||||
|
||||
async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) -> list[dict]:
|
||||
"""Ingest the complete chronological history into one fresh user workspace."""
|
||||
app = Application(**candidate_config(directory / "workspace", model))
|
||||
app, response, session_id = None, None, None
|
||||
stage = "startup"
|
||||
results, memory_usage, aliases = [], [], {}
|
||||
try:
|
||||
app = Application(**candidate_config(directory / "workspace", model))
|
||||
await app.start()
|
||||
observe_usage(app)
|
||||
observe_completion(app)
|
||||
for ordinal, session in enumerate(scene["sessions"]):
|
||||
session_id, response = session["session_id"], None
|
||||
stage = "preprocessing"
|
||||
with usage_scope() as measured:
|
||||
response = await app.run_job(
|
||||
"auto_memory",
|
||||
messages=await asyncio.to_thread(session_messages, session, scene["image_root"], aliases),
|
||||
session_id=session["session_id"],
|
||||
date=timestamp(session["event_start_time"])[:10],
|
||||
include_images=True,
|
||||
include_source_sessions=True,
|
||||
)
|
||||
save(
|
||||
directory / "sessions" / f"{ordinal:05d}.json",
|
||||
{
|
||||
"session_id": session["session_id"],
|
||||
"response": response.model_dump(),
|
||||
"usage": aggregate_usage(measured),
|
||||
},
|
||||
)
|
||||
check_response(response)
|
||||
try:
|
||||
messages = await asyncio.to_thread(session_messages, session, scene["image_root"], aliases)
|
||||
stage = "memory"
|
||||
response = await app.run_job(
|
||||
"auto_memory",
|
||||
messages=messages,
|
||||
session_id=session_id,
|
||||
date=timestamp(session["event_start_time"])[:10],
|
||||
include_images=True,
|
||||
include_source_sessions=True,
|
||||
)
|
||||
check_response(response)
|
||||
finally:
|
||||
save(
|
||||
directory / "sessions" / f"{ordinal:05d}.json",
|
||||
{
|
||||
"session_id": session_id,
|
||||
"response": response_record(response),
|
||||
"usage": aggregate_usage(measured),
|
||||
},
|
||||
)
|
||||
memory_usage.extend(measured)
|
||||
print(f"{scene['id']}: session {ordinal + 1}/{len(scene['sessions'])}", flush=True)
|
||||
check_response(await app.run_job("index_update"))
|
||||
stage, session_id, response = "index", None, None
|
||||
with usage_scope() as measured:
|
||||
try:
|
||||
response = await app.run_job("index_update")
|
||||
check_response(response)
|
||||
finally:
|
||||
save(
|
||||
directory / "index-response.json",
|
||||
{"response": response_record(response), "usage": aggregate_usage(measured)},
|
||||
)
|
||||
memory_usage.extend(measured)
|
||||
# Compatibility marker: never write this for a partial history or failed index.
|
||||
save(directory / "memory_usage.json", aggregate_usage(memory_usage))
|
||||
|
||||
async def answer_question(ordinal, question):
|
||||
question_dir = directory / "questions" / f"{ordinal:04d}"
|
||||
answer_stage, answer_response = "answer", None
|
||||
try:
|
||||
async with query_slots:
|
||||
with usage_scope() as measured:
|
||||
response = await app.run_job(
|
||||
"agentic_answer",
|
||||
query=query_prompt(question["question"], "Use the available evidence tools."),
|
||||
scene_id=scene["id"],
|
||||
include_images=True,
|
||||
include_source_sessions=True,
|
||||
max_iters=30,
|
||||
)
|
||||
answer_record = {
|
||||
"question_id": question["question_id"],
|
||||
"response": response.model_dump(),
|
||||
"usage": aggregate_usage(measured),
|
||||
}
|
||||
save(question_dir / "answer-response.json", answer_record)
|
||||
check_response(response)
|
||||
try:
|
||||
answer_response = await app.run_job(
|
||||
"agentic_answer",
|
||||
query=query_prompt(question["question"], "Use the available evidence tools."),
|
||||
scene_id=scene["id"],
|
||||
include_images=True,
|
||||
include_source_sessions=True,
|
||||
max_iters=30,
|
||||
)
|
||||
check_response(answer_response)
|
||||
finally:
|
||||
answer_record = {
|
||||
"question_id": question["question_id"],
|
||||
"response": response_record(answer_response),
|
||||
"usage": aggregate_usage(measured),
|
||||
}
|
||||
save(question_dir / "answer-response.json", answer_record)
|
||||
save(question_dir / "answer.json", answer_record)
|
||||
prediction = response.answer
|
||||
judgment = await judge.run_job(
|
||||
"answer_judge",
|
||||
question=question["question"],
|
||||
reference=question["answer"],
|
||||
prediction=prediction,
|
||||
evidence=evidence_values(question),
|
||||
)
|
||||
save(question_dir / "judge.json", judgment.model_dump())
|
||||
check_response(judgment)
|
||||
prediction = answer_response.answer
|
||||
answer_stage, answer_response = "judge", None
|
||||
try:
|
||||
answer_response = await judge.run_job(
|
||||
"answer_judge",
|
||||
question=question["question"],
|
||||
reference=question["answer"],
|
||||
prediction=prediction,
|
||||
evidence=evidence_values(question),
|
||||
)
|
||||
check_response(answer_response)
|
||||
finally:
|
||||
save(question_dir / "judge.json", response_record(answer_response))
|
||||
answer_stage = "metrics"
|
||||
metrics = score_answer(question["answer"], prediction)
|
||||
metrics["llm_judge"] = judgment.metadata["llm_judge_score"]
|
||||
metrics["llm_judge"] = answer_response.metadata["llm_judge_score"]
|
||||
result = {
|
||||
"question_id": question["question_id"],
|
||||
"category": question["question_type"],
|
||||
|
|
@ -407,43 +468,61 @@ async def evaluate_user(scene, directory: Path, model: str, judge, query_slots)
|
|||
"prediction": prediction,
|
||||
"metrics": metrics,
|
||||
"usage": aggregate_usage(measured),
|
||||
"judge_parse_default_zero": judgment.metadata["parse_default_zero"],
|
||||
"judge_parse_default_zero": answer_response.metadata["parse_default_zero"],
|
||||
}
|
||||
save(question_dir / "result.json", result)
|
||||
results.append(result)
|
||||
print(f"{scene['id']}: {question['question_id']} completed", flush=True)
|
||||
except Exception as error: # pylint: disable=broad-exception-caught
|
||||
save(question_dir / "error.json", {"error_type": type(error).__name__})
|
||||
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
|
||||
save(
|
||||
question_dir / "error.json",
|
||||
error_record(
|
||||
error,
|
||||
stage=answer_stage,
|
||||
user_id=scene["id"],
|
||||
question_id=question["question_id"],
|
||||
response=answer_response,
|
||||
),
|
||||
)
|
||||
if isinstance(error, asyncio.CancelledError):
|
||||
raise
|
||||
print(f"{scene['id']}: {question['question_id']} failed ({type(error).__name__})", flush=True)
|
||||
|
||||
await asyncio.gather(*(answer_question(i, question) for i, question in enumerate(scene["questions"])))
|
||||
except Exception as error: # pylint: disable=broad-exception-caught
|
||||
save(directory / "error.json", {"error_type": type(error).__name__})
|
||||
print(f"{scene['id']}: ingestion failed ({type(error).__name__})", flush=True)
|
||||
stage, response = "answers", None
|
||||
tasks = [asyncio.create_task(answer_question(i, question)) for i, question in enumerate(scene["questions"])]
|
||||
try:
|
||||
await asyncio.gather(*tasks)
|
||||
finally:
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
|
||||
save(
|
||||
directory / "error.json",
|
||||
error_record(error, stage=stage, user_id=scene["id"], session_id=session_id, response=response),
|
||||
)
|
||||
if isinstance(error, asyncio.CancelledError):
|
||||
raise
|
||||
print(f"{scene['id']}: {stage} failed ({type(error).__name__})", flush=True)
|
||||
finally:
|
||||
await app.close()
|
||||
save(directory / "summary.json", {"completed": len(results), "expected": len(scene["questions"])})
|
||||
try:
|
||||
await close_application(app, directory, user_id=scene["id"])
|
||||
finally:
|
||||
save(directory / "summary.json", {"completed": len(results), "expected": len(scene["questions"])})
|
||||
return results
|
||||
|
||||
|
||||
async def run(scenes, output: Path, model: str, user_concurrency: int, query_concurrency: int) -> bool:
|
||||
"""Run independent users concurrently and save official metrics plus method cost."""
|
||||
judge = Application(**judge_config(output / "judge-workspace"))
|
||||
rows = []
|
||||
user_slots, query_slots = asyncio.Semaphore(user_concurrency), asyncio.Semaphore(query_concurrency)
|
||||
try:
|
||||
await judge.start()
|
||||
|
||||
async def user_job(scene):
|
||||
async with user_slots:
|
||||
rows.extend(await evaluate_user(scene, output / "users" / scene["id"], model, judge, query_slots))
|
||||
|
||||
await asyncio.gather(*(user_job(scene) for scene in scenes))
|
||||
finally:
|
||||
await judge.close()
|
||||
def summarize_run(scenes, output: Path, model: str) -> dict:
|
||||
"""Read each canonical receipt/result once, including a run interrupted after partial progress."""
|
||||
expected = sum(len(scene["questions"]) for scene in scenes)
|
||||
memory = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/memory_usage.json")]
|
||||
answers = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/answer.json")]
|
||||
rows = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/result.json")]
|
||||
sessions = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/sessions/*.json")]
|
||||
indexes = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/index-response.json")]
|
||||
memory = [row["usage"] for row in sessions + indexes]
|
||||
answers = [
|
||||
json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/answer-response.json")
|
||||
]
|
||||
answer_usage = [row["usage"] for row in answers]
|
||||
total = aggregate_usage(memory + answer_usage)
|
||||
summary = {
|
||||
|
|
@ -457,31 +536,58 @@ async def run(scenes, output: Path, model: str, user_concurrency: int, query_con
|
|||
"memory_usage": aggregate_usage(memory),
|
||||
"answer_usage": aggregate_usage(answer_usage),
|
||||
"token_usage": total,
|
||||
"mean_tokens_per_user": {
|
||||
key: total[key] / len(memory) if memory and total[key] is not None else None
|
||||
for key in (
|
||||
"input_tokens",
|
||||
"output_tokens",
|
||||
"total_tokens",
|
||||
"cached_input_tokens",
|
||||
"noncached_input_tokens",
|
||||
)
|
||||
},
|
||||
"mean_tokens_per_question": {
|
||||
key: total[key] / len(answers) if answers and total[key] is not None else None
|
||||
for key in (
|
||||
"input_tokens",
|
||||
"output_tokens",
|
||||
"total_tokens",
|
||||
"cached_input_tokens",
|
||||
"noncached_input_tokens",
|
||||
)
|
||||
"usage_policy": USAGE_POLICY,
|
||||
"cleanup_error_count": len(list(output.glob("users/*/cleanup-error.json")))
|
||||
+ int((output / "cleanup-error.json").exists()),
|
||||
"usage_denominators": {"scheduled_users": len(scenes), "expected_questions": expected},
|
||||
"usage_counts": {
|
||||
"attempted_users": len(list(output.glob("users/*/summary.json"))),
|
||||
"attempted_sessions": len(sessions),
|
||||
"successful_sessions": sum(bool((row.get("response") or {}).get("success")) for row in sessions),
|
||||
"completed_ingestion_users": len(list(output.glob("users/*/memory_usage.json"))),
|
||||
"attempted_answers": len(answers),
|
||||
"successful_answers": sum(bool((row.get("response") or {}).get("success")) for row in answers),
|
||||
"scored_questions": len(rows),
|
||||
},
|
||||
"mean_tokens_per_user": mean_tokens(total, len(scenes)),
|
||||
"mean_tokens_per_question": mean_tokens(total, expected),
|
||||
"judge_parse_default_zero": sum(row["judge_parse_default_zero"] for row in rows),
|
||||
}
|
||||
save(output / "summary.json", summary)
|
||||
return summary
|
||||
|
||||
|
||||
async def run(scenes, output: Path, model: str, user_concurrency: int, query_concurrency: int) -> bool:
|
||||
"""Run independent users concurrently and save official metrics plus method cost."""
|
||||
judge = None
|
||||
user_slots, query_slots = asyncio.Semaphore(user_concurrency), asyncio.Semaphore(query_concurrency)
|
||||
try:
|
||||
judge = Application(**judge_config(output / "judge-workspace"))
|
||||
save(output / "judge-settings.json", judge_settings(judge))
|
||||
await judge.start()
|
||||
|
||||
async def user_job(scene):
|
||||
async with user_slots:
|
||||
await evaluate_user(scene, output / "users" / scene["id"], model, judge, query_slots)
|
||||
|
||||
tasks = [asyncio.create_task(user_job(scene)) for scene in scenes]
|
||||
try:
|
||||
await asyncio.gather(*tasks)
|
||||
finally:
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught
|
||||
save(output / "error.json", error_record(error, stage="run"))
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
await close_application(judge, output)
|
||||
finally:
|
||||
summary = summarize_run(scenes, output, model)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True)
|
||||
return len(rows) == expected
|
||||
return summary["completed"] == summary["expected"] and summary["cleanup_error_count"] == 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
|
|
|||
|
|
@ -16,9 +16,13 @@ and default-zero handling of successful responses without a valid label.
|
|||
remain failed Jobs. Judge usage is evaluation cost, not method cost.
|
||||
|
||||
Defaults are `qwen3.8-max`, temperature 0, thinking disabled and 4,096 output tokens.
|
||||
`JUDGE_MODEL_NAME` can override the Judge model independently of the candidate.
|
||||
Candidate methods and their workspaces are not dependencies of this plugin.
|
||||
The benchmark runner invokes this Job in a separate application and saves its
|
||||
prompt, provider output and verdict alongside the candidate prediction.
|
||||
prompt, provider output and verdict alongside the candidate prediction. Its
|
||||
`judge-settings.json` records the validated model, generation/retry settings and
|
||||
protocol identifier through a credential-free whitelist, rather than recording
|
||||
only the default model name.
|
||||
|
||||
Sources: [MemBase executable judge](https://github.com/zjunlp/MemBase/blob/a9ba9495d48541d3e68588ff097294cc7a1201bc/membase/inference_utils/prompts.py)
|
||||
and [MobileMem published strict rubric](https://github.com/zjunlp/MobileMem/blob/fa5cd13c9c659699df9caf650987ba2bea8ca085/omni/eval/eval/question_answering_and_judge_prompts.txt).
|
||||
|
|
|
|||
423
tests/unit/test_mobilemem_audit.py
Normal file
423
tests/unit/test_mobilemem_audit.py
Normal file
|
|
@ -0,0 +1,423 @@
|
|||
"""Failure-path receipts and cost accounting, without provider or server calls."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,protected-access
|
||||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from benchmark.mobilemem import run
|
||||
from benchmark.mobilemem._audit import ImageInputError, error_record, mean_tokens, response_record
|
||||
from reme.schema import Response
|
||||
|
||||
|
||||
def read(path):
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
@pytest.fixture(name="flow")
|
||||
def fixture_flow(tmp_path, monkeypatch):
|
||||
scenes = [
|
||||
{
|
||||
"id": "uuid_0",
|
||||
"image_root": tmp_path,
|
||||
"sessions": [{"session_id": "s0", "event_start_time": "2025-01-01T09:00:00"}],
|
||||
"questions": [
|
||||
{"question_id": "q0", "question": "What?", "answer": "answer", "question_type": "visual_reasoning"},
|
||||
],
|
||||
},
|
||||
]
|
||||
state = SimpleNamespace(
|
||||
scenes=scenes,
|
||||
apps=[],
|
||||
fail=None,
|
||||
direct=False,
|
||||
cancel=False,
|
||||
cleanup_fail=False,
|
||||
startup_fail=False,
|
||||
constructor_fail=False,
|
||||
calls=[],
|
||||
)
|
||||
|
||||
class FakeApp:
|
||||
"""Mock one owned candidate or Judge application."""
|
||||
|
||||
def __init__(self, *, judge):
|
||||
if state.constructor_fail and not judge:
|
||||
raise ValueError("candidate construction failed")
|
||||
self.judge, self.closed = judge, False
|
||||
state.apps.append(self)
|
||||
|
||||
async def start(self):
|
||||
if state.startup_fail and not self.judge:
|
||||
raise ValueError("candidate startup failed")
|
||||
|
||||
async def close(self):
|
||||
self.closed = True
|
||||
if state.cleanup_fail and not self.judge:
|
||||
raise ValueError("candidate cleanup failed")
|
||||
|
||||
async def run_job(self, name, **kwargs):
|
||||
identity = kwargs.get("session_id", kwargs.get("query", ""))
|
||||
state.calls.append((name, kwargs))
|
||||
if name == "answer_judge":
|
||||
assert run._CURRENT.get() is None # Judge traffic cannot join candidate cost.
|
||||
elif name != "index_update":
|
||||
run._CURRENT.get().append(run.normalized_usage({"prompt_tokens": 100, "completion_tokens": 20}))
|
||||
await asyncio.sleep(0) # Force interleaving of independently scoped requests.
|
||||
if state.fail in (name, identity):
|
||||
if state.cancel:
|
||||
raise asyncio.CancelledError("cancelled offline call")
|
||||
if state.direct:
|
||||
raise ValueError("direct offline failure")
|
||||
return Response(success=False, answer="HTTP 400 input length exceeds limit")
|
||||
if name == "answer_judge":
|
||||
return Response(answer="1", metadata={"llm_judge_score": 1.0, "parse_default_zero": False})
|
||||
# This portable counter is deliberately different: never sum it with HTTP receipts.
|
||||
return Response(answer="answer", metadata={"usage": {"total_tokens": 999999}})
|
||||
|
||||
monkeypatch.setattr(run, "Application", FakeApp)
|
||||
monkeypatch.setattr(run, "candidate_config", lambda *_: {"judge": False})
|
||||
monkeypatch.setattr(run, "judge_config", lambda *_: {"judge": True})
|
||||
monkeypatch.setattr(run, "judge_settings", lambda _: {"model": "offline", "protocol": "offline"})
|
||||
monkeypatch.setattr(run, "session_messages", lambda *_: [{"original": "unchanged"}])
|
||||
monkeypatch.setattr(run, "observe_usage", lambda _: None)
|
||||
monkeypatch.setattr(run, "observe_completion", lambda _: None)
|
||||
|
||||
async def execute():
|
||||
complete = await run.run(scenes, tmp_path / "run", "candidate", 4, 16)
|
||||
return complete, read(tmp_path / "run/summary.json")
|
||||
|
||||
state.execute = execute
|
||||
state.directory = tmp_path / "run/users/uuid_0"
|
||||
return state
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_second_memory_keeps_both_requests(flow):
|
||||
flow.scenes[0]["sessions"].append({"session_id": "s1", "event_start_time": "2025-01-02T09:00:00"})
|
||||
flow.fail = "s1"
|
||||
complete, summary = await flow.execute()
|
||||
assert not complete and summary["completed"] == 0
|
||||
assert summary["memory_usage"]["total_tokens"] == summary["token_usage"]["total_tokens"] == 240
|
||||
assert summary["token_usage"]["records"] == 2
|
||||
assert not (flow.directory / "memory_usage.json").exists()
|
||||
assert read(flow.directory / "sessions/00001.json")["response"]["success"] is False
|
||||
error = read(flow.directory / "error.json")
|
||||
assert (error["stage"], error["user_id"], error["session_id"]) == ("memory", "uuid_0", "s1")
|
||||
assert "input length" in error["reason"] and all(app.closed for app in flow.apps)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("direct", [False, True])
|
||||
async def test_failed_answer_keeps_memory_and_answer_cost_once(flow, direct):
|
||||
flow.fail, flow.direct = "agentic_answer", direct
|
||||
complete, summary = await flow.execute()
|
||||
assert not complete and summary["completed"] == 0 and summary["metrics"]["llm_judge"] is None
|
||||
assert summary["token_usage"]["total_tokens"] == 240
|
||||
assert summary["memory_usage"]["total_tokens"] == summary["answer_usage"]["total_tokens"] == 120
|
||||
assert summary["usage_counts"]["attempted_answers"] == 1
|
||||
assert summary["usage_counts"]["successful_answers"] == 0
|
||||
assert summary["mean_tokens_per_question"]["total_tokens"] == 240
|
||||
assert summary["usage_denominators"] == {"scheduled_users": 1, "expected_questions": 1}
|
||||
assert summary["usage_policy"]["version"] == 2
|
||||
assert (flow.directory / "memory_usage.json").exists()
|
||||
question = flow.directory / "questions/0000"
|
||||
receipt = read(question / "answer-response.json")
|
||||
assert receipt["usage"]["total_tokens"] == 120
|
||||
assert (receipt["response"] is None) is direct
|
||||
assert not (question / "answer.json").exists() and not (question / "result.json").exists()
|
||||
assert read(question / "error.json")["stage"] == "answer"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_memory_exception_keeps_null_response_receipt(flow):
|
||||
flow.fail, flow.direct = "auto_memory", True
|
||||
_, summary = await flow.execute()
|
||||
assert summary["memory_usage"]["total_tokens"] == 120
|
||||
receipt = read(flow.directory / "sessions/00000.json")
|
||||
assert receipt["response"] is None and receipt["usage"]["records"] == 1
|
||||
assert not (flow.directory / "memory_usage.json").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preprocessing_failure_keeps_earlier_memory_and_image_identity(flow, monkeypatch):
|
||||
flow.scenes[0]["sessions"].append({"session_id": "s1", "event_start_time": "2025-01-02T09:00:00"})
|
||||
|
||||
def messages(session, *_):
|
||||
if session["session_id"] == "s1":
|
||||
try:
|
||||
raise FileNotFoundError("missing original image")
|
||||
except FileNotFoundError as error:
|
||||
raise ImageInputError("image/uid0/photo.png", error) from error
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(run, "session_messages", messages)
|
||||
_, summary = await flow.execute()
|
||||
assert summary["memory_usage"]["total_tokens"] == 120
|
||||
receipt = read(flow.directory / "sessions/00001.json")
|
||||
assert receipt["response"] is None and receipt["usage"]["records"] == 0
|
||||
error = read(flow.directory / "error.json")
|
||||
assert error["stage"] == "preprocessing" and error["image_path"] == "image/uid0/photo.png"
|
||||
assert [cause["type"] for cause in error["chain"]] == ["ImageInputError", "FileNotFoundError"]
|
||||
assert not (flow.directory / "memory_usage.json").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_failure_is_saved_without_complete_memory_marker(flow):
|
||||
flow.fail = "index_update"
|
||||
_, summary = await flow.execute()
|
||||
assert summary["memory_usage"]["total_tokens"] == 120
|
||||
assert read(flow.directory / "index-response.json")["response"]["success"] is False
|
||||
assert read(flow.directory / "error.json")["stage"] == "index"
|
||||
assert not (flow.directory / "memory_usage.json").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("direct", [False, True])
|
||||
async def test_judge_failure_preserves_answer_without_double_counting(flow, direct):
|
||||
flow.fail, flow.direct = "answer_judge", direct
|
||||
complete, summary = await flow.execute()
|
||||
assert not complete and summary["token_usage"]["total_tokens"] == 240
|
||||
assert summary["answer_usage"]["records"] == 1
|
||||
question = flow.directory / "questions/0000"
|
||||
assert read(question / "answer.json") == read(question / "answer-response.json")
|
||||
assert not (question / "result.json").exists()
|
||||
assert read(question / "error.json")["stage"] == "judge"
|
||||
assert (read(question / "judge.json") is None) is direct
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_success_old_payloads_metrics_and_usage_stay_compatible(flow):
|
||||
flow.scenes[0]["sessions"].append({"session_id": "s1", "event_start_time": "2025-01-02T09:00:00"})
|
||||
complete, summary = await flow.execute()
|
||||
assert complete and summary["completed"] == summary["expected"] == 1
|
||||
assert summary["metrics"] == {"f1": 1.0, "bleu_1": 1.0, "llm_judge": 1.0}
|
||||
assert summary["token_usage"]["total_tokens"] == 360
|
||||
assert summary["memory_usage"]["total_tokens"] == 240
|
||||
assert summary["answer_usage"]["total_tokens"] == 120
|
||||
assert summary["mean_tokens_per_user"]["total_tokens"] == summary["mean_tokens_per_question"]["total_tokens"] == 360
|
||||
assert summary["token_usage"]["cached_input_tokens"] is None
|
||||
assert summary["token_usage"]["coverage"]["cached_input_tokens"] == 0
|
||||
assert read(flow.directory / "memory_usage.json") == summary["memory_usage"]
|
||||
assert [name for name, _ in flow.calls] == [
|
||||
"auto_memory",
|
||||
"auto_memory",
|
||||
"index_update",
|
||||
"agentic_answer",
|
||||
"answer_judge",
|
||||
]
|
||||
question = flow.directory / "questions/0000"
|
||||
assert read(question / "answer.json") == read(question / "answer-response.json")
|
||||
result = read(question / "result.json")
|
||||
assert result == {
|
||||
"question_id": "q0",
|
||||
"category": "visual_reasoning",
|
||||
"question": "What?",
|
||||
"reference": "answer",
|
||||
"prediction": "answer",
|
||||
"metrics": summary["metrics"],
|
||||
"usage": summary["answer_usage"],
|
||||
"judge_parse_default_zero": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_questions_keep_cost_scopes_and_scheduled_denominator(flow):
|
||||
flow.scenes[0]["questions"].append({**flow.scenes[0]["questions"][0], "question_id": "q1"})
|
||||
flow.scenes.append({**flow.scenes[0], "id": "uuid_1"})
|
||||
complete, summary = await flow.execute()
|
||||
assert complete and summary["completed"] == 4
|
||||
assert summary["token_usage"]["records"] == 6 and summary["token_usage"]["total_tokens"] == 720
|
||||
assert summary["mean_tokens_per_user"]["total_tokens"] == 360
|
||||
assert summary["mean_tokens_per_question"]["total_tokens"] == 180
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_means_use_scheduled_not_successful_denominators(flow):
|
||||
flow.scenes[0]["questions"].append({**flow.scenes[0]["questions"][0], "question_id": "q1"})
|
||||
flow.fail = "agentic_answer"
|
||||
_, summary = await flow.execute()
|
||||
assert summary["completed"] == 0 and summary["expected"] == 2
|
||||
assert summary["token_usage"]["total_tokens"] == 360
|
||||
assert summary["mean_tokens_per_question"]["total_tokens"] == 180
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_scheduled_population_preserves_unknown_means(flow):
|
||||
flow.scenes.clear()
|
||||
_, summary = await flow.execute()
|
||||
assert summary["usage_denominators"] == {"scheduled_users": 0, "expected_questions": 0}
|
||||
assert all(value is None for value in summary["mean_tokens_per_user"].values())
|
||||
assert all(value is None for value in summary["mean_tokens_per_question"].values())
|
||||
assert summary["token_usage"]["records"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summary_ignores_legacy_answer_and_result_usage_copies(flow, monkeypatch):
|
||||
original = run.close_application
|
||||
|
||||
async def close(app, directory, **kwargs):
|
||||
await original(app, directory, **kwargs)
|
||||
if app is not None and app.judge:
|
||||
for name in ("answer.json", "result.json"):
|
||||
path = flow.directory / "questions/0000" / name
|
||||
data = read(path)
|
||||
data["usage"] = {"total_tokens": 999999}
|
||||
run.save(path, data)
|
||||
|
||||
monkeypatch.setattr(run, "close_application", close)
|
||||
_, summary = await flow.execute()
|
||||
assert summary["token_usage"]["total_tokens"] == 240
|
||||
assert summary["answer_usage"]["records"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("constructor", [False, True])
|
||||
async def test_startup_and_cleanup_failures_are_separate(flow, constructor):
|
||||
flow.constructor_fail, flow.startup_fail = constructor, not constructor
|
||||
flow.cleanup_fail = True
|
||||
complete, summary = await flow.execute()
|
||||
assert not complete and summary["token_usage"]["total_tokens"] is None
|
||||
assert read(flow.directory / "error.json")["stage"] == "startup"
|
||||
assert (flow.directory / "cleanup-error.json").exists() is not constructor
|
||||
assert all(app.closed for app in flow.apps)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_request_finalizes_once_and_closes_owned_apps(flow):
|
||||
flow.fail, flow.cancel = "auto_memory", True
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await flow.execute()
|
||||
receipt = read(flow.directory / "sessions/00000.json")
|
||||
assert receipt["response"] is None and receipt["usage"]["total_tokens"] == 120
|
||||
assert read(flow.directory / "error.json")["error_type"] == "CancelledError"
|
||||
assert all(app.closed for app in flow.apps)
|
||||
assert read(flow.directory.parents[1] / "summary.json")["memory_usage"]["total_tokens"] == 120
|
||||
|
||||
|
||||
def test_error_artifacts_remove_secrets_payloads_and_locals(monkeypatch):
|
||||
secret = "offline-sensitive-value"
|
||||
monkeypatch.setenv("LLM_API_KEY", secret)
|
||||
binary = "YWJj" * 100
|
||||
response = Response(
|
||||
success=False,
|
||||
answer=f"input length error api_key={secret} data:image/png;base64,{binary}",
|
||||
metadata={
|
||||
"headers": {"X-Private": "hidden-header"},
|
||||
"request": {"body": "hidden-body"},
|
||||
"credential": {"token": "hidden-credential"},
|
||||
"traceback": "hidden-local",
|
||||
"status_code": 400,
|
||||
},
|
||||
)
|
||||
try:
|
||||
raise ValueError(f"https://user:password@example.invalid/?key={secret} Bearer token-value sk-private-key")
|
||||
except ValueError as error:
|
||||
record = error_record(error, stage="answer", user_id="uuid_0", question_id="q0", response=response)
|
||||
serialized = json.dumps([record, response_record(response)])
|
||||
for value in (
|
||||
secret,
|
||||
binary,
|
||||
"hidden-header",
|
||||
"hidden-body",
|
||||
"hidden-credential",
|
||||
"hidden-local",
|
||||
"token-value",
|
||||
"sk-private-key",
|
||||
"user:password",
|
||||
):
|
||||
assert value not in serialized
|
||||
assert "input length" in record["reason"]
|
||||
assert set(record["chain"][0]["traceback"][0]) == {"file", "line", "function"}
|
||||
assert ImageInputError("../../private.png", ValueError("bad")).image_path == "[invalid image path]"
|
||||
successful = Response(answer="URL https://example.test/ and literal sk-example-text", metadata={"raw": binary})
|
||||
assert response_record(successful) == successful.model_dump()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["token", "refresh_token", "id_token", "access_token", "auth_token"])
|
||||
def test_failed_response_token_aliases_are_redacted_without_removing_usage_counters(name):
|
||||
sentinel = "opaque-new-session"
|
||||
response = Response(
|
||||
success=False,
|
||||
answer=f"{name}='{sentinel}'",
|
||||
metadata={
|
||||
name: sentinel,
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 20,
|
||||
"cached_input_tokens": 30,
|
||||
},
|
||||
)
|
||||
record = response_record(response)
|
||||
assert sentinel not in json.dumps(record)
|
||||
assert record["metadata"]["input_tokens"] == 100
|
||||
assert record["metadata"]["output_tokens"] == 20
|
||||
assert record["metadata"]["cached_input_tokens"] == 30
|
||||
|
||||
|
||||
@pytest.mark.parametrize("header", ["Cookie", "Set-Cookie"])
|
||||
def test_failed_response_redacts_whole_cookie_header_without_changing_ordinary_prose(header):
|
||||
response = Response(
|
||||
success=False,
|
||||
answer=f"{header}: session=opaque; other=secondsecret\nThe cookie parser failed with status 400.",
|
||||
)
|
||||
record = response_record(response)
|
||||
assert "opaque" not in record["answer"] and "secondsecret" not in record["answer"]
|
||||
assert "The cookie parser failed with status 400." in record["answer"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_observer_includes_compressor_subsets_and_null_coverage_without_duplicate_wrapping():
|
||||
async def send(request):
|
||||
await asyncio.sleep(0)
|
||||
return SimpleNamespace(status_code=200, is_stream_consumed=True, json=lambda: {"usage": request})
|
||||
|
||||
clients = [SimpleNamespace(send=send) for _ in range(3)]
|
||||
app = SimpleNamespace(
|
||||
context=SimpleNamespace(
|
||||
components={
|
||||
"as_llm": {
|
||||
name: SimpleNamespace(model=SimpleNamespace(client=SimpleNamespace(_client=client)))
|
||||
for name, client in zip(("default", "bench", "compressor"), clients)
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
run.observe_usage(app)
|
||||
run.observe_usage(app)
|
||||
|
||||
async def scoped(amount):
|
||||
with run.usage_scope() as records:
|
||||
await clients[0].send(
|
||||
{"prompt_tokens": amount, "completion_tokens": 20, "prompt_tokens_details": {"cached_tokens": 30}}
|
||||
)
|
||||
await clients[2].send(
|
||||
{"prompt_tokens": 50, "completion_tokens": 10, "completion_tokens_details": {"reasoning_tokens": 4}}
|
||||
)
|
||||
return run.aggregate_usage(records)
|
||||
|
||||
first, second = await asyncio.gather(scoped(100), scoped(200))
|
||||
assert first["records"] == second["records"] == 2
|
||||
assert (first["total_tokens"], second["total_tokens"]) == (180, 280)
|
||||
assert first["cached_input_tokens"] == 30 and first["noncached_input_tokens"] == 70
|
||||
assert first["coverage"]["cached_input_tokens"] == 1 and first["reasoning_tokens"] == 4
|
||||
assert mean_tokens(first, 0)["total_tokens"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cleanup_error_does_not_replace_primary_failure(flow):
|
||||
flow.fail, flow.cleanup_fail = "auto_memory", True
|
||||
_, summary = await flow.execute()
|
||||
assert summary["memory_usage"]["total_tokens"] == 120
|
||||
assert read(flow.directory / "error.json")["stage"] == "memory"
|
||||
assert read(flow.directory / "cleanup-error.json")["stage"] == "cleanup"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cleanup_failure_keeps_valid_scores_but_does_not_report_operational_success(flow):
|
||||
flow.cleanup_fail = True
|
||||
complete, summary = await flow.execute()
|
||||
assert not complete and summary["completed"] == summary["expected"] == 1
|
||||
assert summary["cleanup_error_count"] == 1 and summary["metrics"]["llm_judge"] == 1.0
|
||||
assert summary["token_usage"]["total_tokens"] == 240
|
||||
|
|
@ -148,6 +148,7 @@ async def test_candidate_and_judge_start_with_native_settings(tmp_path, monkeypa
|
|||
monkeypatch.setattr("reme.config.config_parser.find_entry_points", installed)
|
||||
monkeypatch.setenv("LLM_API_KEY", "offline")
|
||||
monkeypatch.setenv("LLM_BASE_URL", "https://provider.invalid/v1")
|
||||
monkeypatch.setenv("JUDGE_MODEL_NAME", "offline-judge-override")
|
||||
candidate_app = Application(**mobilemem.candidate_config(tmp_path / "candidate", "candidate-model"))
|
||||
judge_app = Application(**mobilemem.judge_config(tmp_path / "judge"))
|
||||
try:
|
||||
|
|
@ -155,6 +156,11 @@ async def test_candidate_and_judge_start_with_native_settings(tmp_path, monkeypa
|
|||
await judge_app.start()
|
||||
assert set(judge_app.context.jobs) == {"answer_judge"}
|
||||
assert set(judge_app.context.components) == {"as_llm"}
|
||||
audit = mobilemem.judge_settings(judge_app)
|
||||
assert audit["model"] == "offline-judge-override"
|
||||
assert audit["parameters"] == {"max_tokens": 4096, "temperature": 0, "thinking_enable": False}
|
||||
assert audit["protocol"] == "mobilemem-omni-membase-default-judge-v1"
|
||||
assert "credential" not in json.dumps(audit) and "provider.invalid" not in json.dumps(audit)
|
||||
models = candidate_app.context.components["as_llm"]
|
||||
for component in models.values():
|
||||
assert component.kwargs["model"] == "candidate-model"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue