fix(benchmark): retain MemLens usage and auditable failures

This commit is contained in:
wang-qisen 2026-10-19 22:51:21 +08:00
parent 962d5cd840
commit 14c459cfd9
8 changed files with 928 additions and 168 deletions

View file

@ -40,9 +40,12 @@ three retries, a 60-second delay and a 600-second HTTP timeout; nested retries
are disabled. The runner does not repair model outputs.
The independent [MemLens Judge](../../plugins/memlens-judge/README.md) uses the
official category/subtype-specific binary accuracy protocol, Qwen3.8-Max,
official category/subtype-specific binary accuracy protocol, Qwen3.8-Max by default,
thinking off, temperature 0 and 256 output tokens. SubEM and F1 follow the
official code. Reference answers, question categories and subtypes enter only scoring.
`JUDGE_MODEL_NAME` overrides the judge model independently of the candidate.
`settings.json` records the effective judge model, generation parameters and
protocol after plugin defaults and environment expansion, without credentials.
## Results
@ -51,9 +54,26 @@ official code. Reference answers, question categories and subtypes enter only sc
answers, judgments, metrics and coverage status. The workspace's `mem_session/`
contains native Agent JSONL trajectories. Existing output groups are not overwritten.
Usage includes successful memory/answer requests and compression, excluding
Judge costs and failed attempts. Missing cache/reasoning counters remain unknown.
New runs use usage policy `successful-http-requests-v2`: usage includes observed
successful HTTP requests in memory/answer scopes, including compression and requests
completed before a later Job or history failure. Each scope is counted once.
Judge requests and unsuccessful HTTP attempts are excluded. This is measured request
usage, not a full-sample cost estimate or provider billing total. Unknown counters
remain `null`; `records` and per-counter `coverage` show how many requests were measured.
Expected, scored and unscored question counts make coverage gaps explicit.
Technical failures are coverage gaps; valid empty or iteration-limit answers are scored.
Failure artifacts retain the failed Response before validation and record the stage,
scene/session/question IDs, sanitized error chain and traceback locations without locals.
These artifacts omit credentials, request headers, raw configuration and image data.
Failed responses use separate `diagnostics/*-response.json` files; `answer.json` continues to
indicate a successful answer Job and retains the original prediction. Per-scope
`diagnostics/*-usage.json` receipts survive direct exceptions and cancellation, while
`status.json.memory_complete` records successful ingestion and indexing independently.
Cleanup errors retain scores and usage, increment `cleanup_error_count`, and cause
a nonzero CLI exit even when all questions were scored.
Historical reports using selected successful Job/history usage retain their original
values and scope; they are not rewritten under the new policy.
## Sources and license

View file

@ -38,8 +38,10 @@ session 正文。每题新建 30 轮回答 Agent,工具为 `search`、`read`
为 600 秒;不叠加 SDK/Agent 重试,也不修补模型输出。
独立的 [MemLens Judge](../../plugins/memlens-judge/README.md) 使用官方按
category/subtype 区分的二元评分:Qwen3.8-Max、thinking 关闭、temperature
category/subtype 区分的二元评分:默认 Qwen3.8-Max、thinking 关闭、temperature
0、输出预算 256。辅助指标为官方 SubEM、F1。答案与题型字段仅进入评分。
`JUDGE_MODEL_NAME` 可独立覆盖 Judge 模型;`settings.json` 记录插件默认值和
环境变量展开后实际生效的 Judge 模型、生成参数与协议,不保存凭据或服务端点。
## 结果
@ -48,8 +50,20 @@ category/subtype 区分的二元评分:Qwen3.8-Max、thinking 关闭、tempe
指标及完成状态。workspace 下的 `mem_session/` 保留原生 Agent JSONL 轨迹。
已有结果目录不会被覆盖。
Token 统计包含成功记忆/回答调用及压缩调用,不计 Judge 和失败尝试。
缺失的 cache/reasoning 字段保持未知;技术失败计为覆盖缺口,合法空答案与
达到轮数上限的回答仍评分。
新运行使用 `successful-http-requests-v2` 口径:统计记忆/回答作用域中观测到的
成功 HTTP 请求,包括压缩,以及所属 Job 或历史处理随后失败前已成功的请求;
每个作用域只计一次。不计 Judge 请求及未成功的 HTTP 尝试。这是实测请求用量,
并非完整样本成本估计或供应商账单。未知计数保持 `null`,`records` 与各计数的
`coverage` 标示观测覆盖;预期、已评分、未评分题数分别记录。
技术失败计为覆盖缺口,合法空答案与达到轮数上限的回答仍评分。失败诊断记录阶段、
场景/session/题目 ID、脱敏错误链和不含局部变量的堆栈位置,不保存凭据、
请求头、原始配置或图片数据。失败 Response 在检查前保存到独立的
`diagnostics/*-response.json`;`answer.json` 仍只表示成功回答,保留原始预测。
`diagnostics/*-usage.json` 在直接异常或取消时也保留已观测用量;
`status.json.memory_complete` 独立表示历史摄入和索引完成。
清理失败保留评分及用量、增加 `cleanup_error_count`,并使 CLI 返回非零状态。
历史报告沿用其原有的选定成功 Job/历史统计口径,不按新口径重写数据。
数据来源、上游版本和完整授权文本见[英文 README](README.md#sources-and-license)。

View file

@ -0,0 +1,130 @@
"""Small, credential-free snapshots and diagnostics for benchmark artifacts."""
import os
from pathlib import Path
import re
import traceback
from urllib.parse import urlsplit, urlunsplit
_PRIVATE_KEY = re.compile(
r"credential|api[_-]?key|private[_-]?key|authorization|password|secret|cookie|access[_-]?token|refresh[_-]?token|"
r"(^|[_-])token$|headers|request[_-]?body|^environment$|^config(?:uration)?$|^messages$",
re.IGNORECASE,
)
_ASSIGNMENT = re.compile(
r"(?i)([\"']?(?:api[_-]?key|private[_-]?key|authorization|password|secret|(?:access|refresh|id)[_-]?token|token)"
r"[\"']?\s*[:=]\s*)(?:\"[^\"]*\"|'[^']*'|[^\s,;}]+)",
)
_MODEL_FIELDS = (
"backend",
"model",
"stream",
"context_size",
"max_retries",
"retry_delay",
)
_MODEL_OPTIONS = {
"parameters": (
"temperature",
"top_p",
"max_tokens",
"max_completion_tokens",
"thinking_enable",
),
"extra_body": ("enable_thinking",),
"client_kwargs": ("timeout", "max_retries"),
}
def _safe_text(value: str) -> str:
for key, secret in sorted(os.environ.items(), key=lambda item: len(item[1]), reverse=True):
if secret and _PRIVATE_KEY.search(key):
value = value.replace(secret, "[REDACTED]")
def safe_url(match):
try:
url = urlsplit(match.group())
host = url.netloc.rsplit("@", 1)[-1]
return urlunsplit((url.scheme, host, url.path, "[REDACTED]" if url.query else "", ""))
except ValueError:
return "[REDACTED_URL]"
value = re.sub(r"https?://[^\s\"'<>]+", safe_url, value)
value = re.sub(r"(?s)-----BEGIN [^-]*PRIVATE KEY-----.*?-----END [^-]*PRIVATE KEY-----", "[REDACTED_KEY]", value)
value = re.sub(r"(?im)(\b(?:set[-_])?cookie[\"']?\s*[:=]\s*)[^\r\n]+", r"\1[REDACTED]", value)
value = re.sub(r"(?i)\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+", "[REDACTED_AUTH]", value)
value = _ASSIGNMENT.sub(r"\1[REDACTED]", value)
value = re.sub(r"(?i)data:[^;,\s]+;base64,[A-Za-z0-9+/=\s]+", "[REDACTED_IMAGE]", value)
value = re.sub(r"\b[A-Za-z0-9+/]{80,}={0,2}", "[REDACTED_DATA]", value)
# Exceptions sometimes embed complete request/config dictionaries in their text.
return re.sub(
r"(?is)(\b(?:config|configuration|request_headers|headers|request_body|messages)\b[\"']?\s*[:=]).*",
r"\1[REDACTED]",
value,
)
def safe_value(value):
"""Redact diagnostic data recursively without serializing arbitrary objects."""
if isinstance(value, dict):
return {
_safe_text(str(key)): (
"[REDACTED]"
if _PRIVATE_KEY.search(str(key)) or (key == "data" and value.get("type") == "base64")
else safe_value(item)
)
for key, item in value.items()
}
if isinstance(value, (list, tuple)):
return [safe_value(item) for item in value]
if isinstance(value, str):
return _safe_text(value)
if value is None or isinstance(value, (bool, int, float)):
return value
return f"<{type(value).__name__}>"
def error_details(exc: BaseException, seen: set[int] | None = None) -> dict:
"""Keep the error chain and frame locations, never frame locals or source text."""
seen = set() if seen is None else seen
seen.add(id(exc))
result = {
"type": type(exc).__name__,
"message": str(exc),
"traceback": [
{
"file": Path(frame.filename).name,
"line": frame.lineno,
"function": frame.name,
}
for frame in traceback.extract_tb(exc.__traceback__)
],
}
cause = exc.__cause__ or (None if exc.__suppress_context__ else exc.__context__)
if cause is not None and id(cause) not in seen:
result["cause"] = error_details(cause, seen)
return safe_value(result)
def model_settings(config) -> dict:
"""Whitelist reproducibility fields from a validated ApplicationConfig."""
models = {}
for name, component in config.components.get("as_llm", {}).items():
spec = component.model_dump()
snapshot = {key: spec[key] for key in _MODEL_FIELDS if key in spec}
for key, allowed in _MODEL_OPTIONS.items():
options = spec.get(key) or {}
snapshot[key] = {name: options[name] for name in allowed if name in options}
models[name] = snapshot
return safe_value(models)
def judge_settings(app) -> dict:
"""Record the actual plugin/env-expanded judge model and official protocol."""
from judge_memlens.protocol import PROTOCOL_ID, SOURCE_REVISION
return {
"protocol_id": PROTOCOL_ID,
"source_revision": SOURCE_REVISION,
"models": model_settings(app.config),
}

View file

@ -14,9 +14,12 @@ from pathlib import Path
import re
import subprocess
import string
import sys
import yaml
from benchmark.memlens.artifacts import error_details, judge_settings, model_settings, safe_value
# Adapted query/metric code retains its upstream MIT notice in README.md.
SYSTEM_PROMPT = (
"Provide answers based on the given conversation history. "
@ -161,18 +164,6 @@ def query_prompt(question: dict, context: str) -> str:
)
def parse_model_output(text: str) -> str:
"""Native special-token/think-trace parsing; retained raw outputs remain separate."""
if not text:
return ""
if "</think>" in text:
text = text.split("</think>", 1)[1]
text = re.sub(r"<\|[^>]*\|>", "", text)
text = re.sub(r"<\|begin_of_box\|>(.*?)<\|end_of_box\|>", r"\1", text, flags=re.DOTALL)
text = text.replace("<|begin_of_box|>", "").replace("<|end_of_box|>", "")
return text.replace("[CONTENT_FILTER_REJECTED]", "").strip()
def _normalize(text: str) -> str:
"""Native SubEM/F1 normalization, preserving decimal points and date slashes."""
text = text.lower()
@ -192,6 +183,8 @@ def _normalize(text: str) -> str:
def score_answer(question: dict, prediction: str) -> dict:
"""Official inline auxiliary SubEM/F1; primary accuracy comes from the LLM judge."""
from judge_memlens.protocol import parse_model_output
predicted, expected = _normalize(parse_model_output(prediction)), _normalize(question["answer"])
prediction_tokens, answer_tokens = predicted.split(), expected.split()
overlap = sum((Counter(prediction_tokens) & Counter(answer_tokens)).values())
@ -250,7 +243,7 @@ def total_usage(records: list[dict]) -> dict:
@contextmanager
def collect_usage():
def collect_usage(target: list | None = None, receipt: Path | None = None):
"""Keep concurrent calls' usage separate and include their compression calls."""
records = []
token = _CURRENT.set(records)
@ -258,6 +251,10 @@ def collect_usage():
yield records
finally:
_CURRENT.reset(token)
if target is not None:
target.extend(records)
if receipt is not None:
save(receipt, total_usage(records))
def observe_usage(app) -> None:
@ -295,7 +292,7 @@ def observe_usage(app) -> None:
def save(path: Path, value) -> None:
"""Save results, never model credentials or application secrets."""
"""Save source/results unchanged; callers sanitize diagnostics and config snapshots."""
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, ensure_ascii=False, indent=2, default=str) + "\n", encoding="utf-8")
@ -331,141 +328,207 @@ def check_response(response):
if asyncio.current_task().cancelling():
raise asyncio.CancelledError
if not response.success:
raise RuntimeError("ReMe Job failed; see this scene's application log.")
detail = response.answer or response.metadata.get("error") or "No failure detail provided."
raise RuntimeError(f"ReMe Job failed; response detail: {safe_value(detail)}")
return response
async def close_application(app) -> None:
"""Finish owned cleanup even when cancellation arrives during close."""
closing = asyncio.create_task(app.close())
cancellation = None
while True:
try:
await asyncio.shield(closing)
break
except asyncio.CancelledError as exc:
if closing.done():
raise
cancellation = exc
except Exception as exc:
if cancellation is not None:
raise cancellation from exc
raise
if cancellation is not None:
raise cancellation
async def evaluate_scene(scene, number, model, group, judge, slots):
"""Accumulate one isolated history, then answer and judge its question."""
from reme import Application
label = re.sub(r"[^\w.-]+", "_", scene["id"])[:100]
case = group / "reme_image" / "scenes" / f"{number:04d}-{label}"
app = Application(**app_config(case / "ingestion/attempt-001/workspace", model))
app = None
memory, answers, results = [], [], []
memory_complete = False
save(case / "source.json", {"id": scene["id"], "sessions": len(scene["sessions"])})
status = {"complete": False, "memory_complete": False, "stage": "initialize", "scene_id": scene["id"]}
try:
save(case / "source.json", {"id": scene["id"], "sessions": len(scene["sessions"])})
app = Application(**app_config(case / "ingestion/attempt-001/workspace", model))
status["stage"] = "start"
await app.start()
observe_usage(app)
for ordinal, session in enumerate(scene["sessions"]):
directory = case / "ingestion/attempt-001"
status.update(stage="preprocess", session_id=session["id"])
async with slots:
with collect_usage() as requests:
response = check_response(
await app.run_job(
"auto_memory",
messages=await asyncio.to_thread(session_messages, session),
session_id=session["id"],
date=session["timestamp"][:10],
include_images=True,
include_source_sessions=True,
),
with collect_usage(memory, directory / "diagnostics" / f"{ordinal:05d}-usage.json") as requests:
messages = await asyncio.to_thread(session_messages, session)
status["stage"] = "auto_memory"
response = await app.run_job(
"auto_memory",
messages=messages,
session_id=session["id"],
date=session["timestamp"][:10],
include_images=True,
include_source_sessions=True,
)
memory.extend(requests)
if not response.success:
save(
directory / "diagnostics" / f"{ordinal:05d}-response.json",
safe_value({"response": response.model_dump()}),
)
check_response(response)
save(
directory / "sessions" / f"{ordinal:05d}.json",
{
"session_id": session["id"],
"response": response.model_dump(),
"usage": total_usage(requests),
},
)
status.pop("session_id", None)
status["stage"] = "index_update"
response = await app.run_job("index_update")
if not response.success:
save(
case / "ingestion/attempt-001/sessions" / f"{ordinal:05d}.json",
{
"session_id": session["id"],
"response": response.model_dump(),
"usage": total_usage(requests),
},
case / "ingestion/attempt-001/diagnostics/index-response.json",
safe_value({"response": response.model_dump()}),
)
check_response(await app.run_job("index_update"))
memory_complete = True
check_response(response)
status["memory_complete"] = True
for index, question in enumerate(scene["questions"]):
directory = case / "questions" / f"{index:04d}"
query = question
status.update(stage="agentic_answer", question_id=question["id"])
async with slots:
with collect_usage() as requests:
response = check_response(
await app.run_job(
"agentic_answer",
query=query_prompt(query, "Use the available tools to retrieve relevant evidence."),
query_time=query["question_date"],
query_images=[],
scene_id=scene["id"],
include_images=True,
include_source_sessions=True,
max_iters=30,
),
with collect_usage(answers, directory / "diagnostics/answer-usage.json") as requests:
response = await app.run_job(
"agentic_answer",
query=query_prompt(question, "Use the available tools to retrieve relevant evidence."),
query_time=question["question_date"],
query_images=[],
scene_id=scene["id"],
include_images=True,
include_source_sessions=True,
max_iters=30,
)
answers.extend(requests)
save(
directory / "answer.json",
{
"question_id": query["id"],
"prediction": response.answer,
"usage": total_usage(requests),
"agent": response.metadata,
},
)
if not response.success:
save(
directory / "diagnostics/answer-response.json",
safe_value({"response": response.model_dump()}),
)
check_response(response)
save(
directory / "answer.json",
{
"question_id": question["id"],
"prediction": response.answer,
"usage": total_usage(requests),
"agent": response.metadata,
},
)
status["stage"] = "judge"
async with slots:
judgment = check_response(
await judge.run_job(
"answer_judge",
question=query["text"],
reference=question["answer"],
prediction=response.answer,
category=query["category"],
subtype=question.get("subtype", ""),
old_answer=question.get("old_answer", ""),
),
judgment = await judge.run_job(
"answer_judge",
question=question["text"],
reference=question["answer"],
prediction=response.answer,
category=question["category"],
subtype=question.get("subtype", ""),
old_answer=question.get("old_answer", ""),
)
if not judgment.success:
save(directory / "diagnostics/judge-response.json", safe_value({"response": judgment.model_dump()}))
check_response(judgment)
status["stage"] = "score"
metrics = score_answer(question, response.answer)
metrics["llm_judge"] = judgment.metadata["llm_judge_score"]
save(directory / "judge.json", judgment.metadata)
record = {
"question_id": query["id"],
"category": query["category"],
"question": query["text"],
"question_id": question["id"],
"category": question["category"],
"question": question["text"],
"reference": question["answer"],
"prediction": response.answer,
"metrics": metrics,
}
save(directory / "result.json", record)
results.append(record)
save(case / "status.json", {"complete": True})
status.update(complete=True, stage="complete")
except asyncio.CancelledError as exc:
status["error"] = error_details(exc)
raise
except Exception as exc: # pylint: disable=broad-exception-caught
save(case / "status.json", {"complete": False, "error_type": type(exc).__name__})
status["error"] = error_details(exc)
finally:
await app.close()
return results, memory if memory_complete else [], answers
try:
if app is not None:
await close_application(app)
except (Exception, asyncio.CancelledError) as exc: # pylint: disable=broad-exception-caught
if "error" in status:
status["cleanup_error"] = error_details(exc)
else:
status.update(complete=False, stage="cleanup", error=error_details(exc))
if isinstance(exc, asyncio.CancelledError):
raise
finally:
status.update(memory_usage=total_usage(memory), answer_usage=total_usage(answers))
if "error" in status:
status["error_type"] = status["error"]["type"]
save(case / "status.json", safe_value(status))
return results, memory, answers
async def run(settings: dict, model: str, length: str, limit: int | None = None) -> dict:
"""Run one model in a new output group; existing runs are never overwritten."""
from reme import Application
from reme.schema import ApplicationConfig
scenes = load_memlens(Path(settings["dataset"]), length)
if limit is not None:
scenes = scenes[:limit]
group = Path(settings["output"]).expanduser().resolve() / f"memlens-{length}" / model
group.mkdir(parents=True, exist_ok=False)
effective = app_config(group / "unused-workspace", model)
save(
group / "settings.json",
{
"method": "reme_image",
"model": model,
"context_length": length,
"dataset": str(Path(settings["dataset"]).expanduser().resolve()),
"limit_scenes": limit,
"include_source_sessions": True,
"include_images": True,
"answer_max_iters": 30,
"config": settings,
"models": {
name: {key: value for key, value in spec.items() if key != "credential"}
for name, spec in effective["components"]["as_llm"].items()
},
"reme_commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
},
)
slots = asyncio.Semaphore(settings["query_concurrency"])
scene_slots = asyncio.Semaphore(settings["scenario_concurrency"])
judge = Application(**app_config(group / "judge-workspace", model, judge=True))
judge = None
status = {"complete": False, "stage": "preprocess"}
try:
scenes = load_memlens(Path(settings["dataset"]), length)
if limit is not None:
scenes = scenes[:limit]
status["stage"] = "configure"
effective = ApplicationConfig(**app_config(group / "unused-workspace", model))
judge = Application(**app_config(group / "judge-workspace", model, judge=True))
save(
group / "settings.json",
{
"method": "reme_image",
"model": model,
"context_length": length,
"dataset": str(Path(settings["dataset"]).expanduser().resolve()),
"limit_scenes": limit,
"include_source_sessions": True,
"include_images": True,
"answer_max_iters": 30,
"concurrency": {key: settings[key] for key in ("scenario_concurrency", "query_concurrency")},
"models": model_settings(effective),
"judge": judge_settings(judge),
"usage_policy_version": "successful-http-requests-v2",
"reme_commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
},
)
slots = asyncio.Semaphore(settings["query_concurrency"])
scene_slots = asyncio.Semaphore(settings["scenario_concurrency"])
status["stage"] = "judge_start"
await judge.start()
status["stage"] = "evaluate"
async def evaluate(number, scene):
async with scene_slots:
@ -474,28 +537,64 @@ async def run(settings: dict, model: str, length: str, limit: int | None = None)
print(f"Finished {scene['id']}: {len(result[0])}/{len(scene['questions'])} scored", flush=True)
return result
outputs = await asyncio.gather(*(evaluate(index, scene) for index, scene in enumerate(scenes)))
tasks = [asyncio.create_task(evaluate(index, scene)) for index, scene in enumerate(scenes)]
try:
outputs = await asyncio.gather(*tasks)
finally:
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
status["stage"] = "summary"
results = [row for output in outputs for row in output[0]]
memory = [row for output in outputs for row in output[1]]
answers = [row for output in outputs for row in output[2]]
expected = sum(len(scene["questions"]) for scene in scenes)
statuses = [json.loads(path.read_text()) for path in (group / "reme_image/scenes").glob("*/status.json")]
cleanup_errors = sum("cleanup_error" in row or row["stage"] == "cleanup" for row in statuses)
summary = {
"method": "reme_image",
"model": model,
"context_length": length,
"expected_scenes": len(scenes),
"completed_scenes": sum(row["complete"] for row in statuses),
"memory_complete_scenes": sum(row["memory_complete"] for row in statuses),
"cleanup_error_count": cleanup_errors,
"expected_questions": expected,
"scored_questions": len(results),
"unscored_questions": expected - len(results),
"metrics": {
key: (sum(row["metrics"][key] for row in results) / len(results) if results else None)
for key in ("llm_judge", "sub_em", "f1")
},
"memory_usage": total_usage(memory),
"answer_usage": total_usage(answers),
"token_usage": total_usage(memory + answers),
"usage_policy_version": "successful-http-requests-v2",
"usage_policy": (
"observed successful HTTP requests in memory/answer scopes, including compression and requests "
"before Job/history failure; judge excluded; not full-sample or billed cost"
),
}
save(group / "summary.json", summary)
status.update(complete=len(results) == expected and not cleanup_errors, stage="complete")
except (Exception, asyncio.CancelledError) as exc:
status["error"] = error_details(exc)
raise
finally:
await judge.close()
results = [row for output in outputs for row in output[0]]
memory = [row for output in outputs for row in output[1]]
answers = [row for output in outputs for row in output[2]]
summary = {
"method": "reme_image",
"model": model,
"context_length": length,
"expected_questions": sum(len(scene["questions"]) for scene in scenes),
"scored_questions": len(results),
"metrics": {
key: sum(row["metrics"][key] for row in results) / len(results) if results else None
for key in ("llm_judge", "sub_em", "f1")
},
"memory_usage": total_usage(memory),
"answer_usage": total_usage(answers),
"token_usage": total_usage(memory + answers),
"usage_policy": "successful memory/answer calls, including compression; judge excluded",
}
save(group / "summary.json", summary)
try:
if judge is not None:
await close_application(judge)
except (Exception, asyncio.CancelledError) as exc:
if "error" in status:
status["cleanup_error"] = error_details(exc)
if isinstance(exc, asyncio.CancelledError):
raise
else:
status.update(complete=False, stage="judge_cleanup", error=error_details(exc))
raise
finally:
save(group / "status.json", safe_value(status))
return summary
@ -508,17 +607,23 @@ def main():
parser.add_argument("--limit-scenes", type=int)
parser.add_argument("--output", type=Path)
arguments = parser.parse_args()
settings = yaml.safe_load(os.path.expandvars(arguments.config.read_text(encoding="utf-8")))
if arguments.output:
settings["output"] = str(arguments.output)
if arguments.limit_scenes is not None and arguments.limit_scenes < 1:
parser.error("--limit-scenes must be positive")
for key in ("query_concurrency", "scenario_concurrency"):
if not isinstance(settings[key], int) or settings[key] < 1:
parser.error(f"{key} must be positive")
summary = asyncio.run(run(settings, arguments.model, arguments.context_length, arguments.limit_scenes))
try:
settings = yaml.safe_load(os.path.expandvars(arguments.config.read_text(encoding="utf-8")))
if arguments.output:
settings["output"] = str(arguments.output)
if arguments.limit_scenes is not None and arguments.limit_scenes < 1:
parser.error("--limit-scenes must be positive")
for key in ("query_concurrency", "scenario_concurrency"):
if not isinstance(settings[key], int) or settings[key] < 1:
parser.error(f"{key} must be positive")
summary = asyncio.run(run(settings, arguments.model, arguments.context_length, arguments.limit_scenes))
except (Exception, asyncio.CancelledError, KeyboardInterrupt) as exc: # pylint: disable=broad-exception-caught
print(json.dumps({"complete": False, "error": error_details(exc)}, ensure_ascii=False), file=sys.stderr)
return 130 if isinstance(exc, (asyncio.CancelledError, KeyboardInterrupt)) else 2
print(json.dumps(summary, ensure_ascii=False, indent=2))
return 0 if summary["scored_questions"] == summary["expected_questions"] else 2
return (
0 if summary["scored_questions"] == summary["expected_questions"] and not summary["cleanup_error_count"] else 2
)
if __name__ == "__main__":

View file

@ -15,7 +15,12 @@ long-answer preprocessing and over-500-word zero rule are preserved. A successfu
but unparseable judge output receives the official zero score; API errors instead
remain failed jobs for resumable judging of the same saved prediction.
This comparison uses `qwen3.8-max`, temperature 0 and thinking disabled. The output
The default judge is `qwen3.8-max`, with temperature 0 and thinking disabled.
Set `JUDGE_MODEL_NAME` to override the judge model independently of the candidate.
The benchmark's `settings.json` records the effective judge model, generation
parameters and protocol from the validated application configuration, after plugin
defaults and environment expansion. The snapshot excludes credentials and endpoints.
The output
budget is 256 tokens from the released OpenAI-compatible API judge branch
(`max_completion_tokens`), not the separate vLLM branch's 2,048-token setting.
These judge settings are independent of the candidate reader's thinking mode.

View file

@ -29,6 +29,11 @@ from reme.steps.file_io.read_image import ReadImageStep
from benchmark.memlens import run as memlens
@pytest.fixture(autouse=True)
def memlens_judge_path(monkeypatch):
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "plugins/memlens-judge/src"))
def test_candidate_declares_required_image_dependency():
manifest = Path(__file__).resolve().parents[2] / "plugins/memlens/pyproject.toml"
project = tomllib.loads(manifest.read_text(encoding="utf-8"))["project"]
@ -95,12 +100,10 @@ def test_official_metric_normalization():
@pytest.fixture(name="candidate")
def fixture_candidate(tmp_path, monkeypatch):
name = "memlens"
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "plugins" / name / "src"))
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "plugins/memlens/src"))
monkeypatch.chdir(tmp_path)
memory = import_module(f"reme_{name}.auto_memory")
answer = import_module(f"reme_{name}.agentic_answer")
prefix = "MemLens"
memory = import_module("reme_memlens.auto_memory")
answer = import_module("reme_memlens.agentic_answer")
app = SimpleNamespace(
registry=R,
metadata={},
@ -111,23 +114,23 @@ def fixture_candidate(tmp_path, monkeypatch):
wrapper = AsAgentWrapper(backend="agentscope", as_llm="", app_context=app)
wrapper.reply = AsyncMock(return_value={"result": "answer", "session_id": "new", "usage": {"input_tokens": 7}})
store = LocalFileStore(embedding_store="", app_context=app)
return name, memory, answer, prefix, app, wrapper, store, tmp_path
return memory, answer, app, wrapper, store, tmp_path
def test_candidate_registration_is_application_local(candidate):
name, memory, answer, prefix, *_ = candidate
plugin = _load_plugin(name, EntryPoint(name=name, value=f"reme_{name}", group="reme.plugins"))
memory, answer, *_ = candidate
plugin = _load_plugin("memlens", EntryPoint(name="memlens", value="reme_memlens", group="reme.plugins"))
registry = create_application_registry()
PluginManager([plugin]).register(registry)
assert registry.get("step", f"{name}_auto_memory_step") is getattr(memory, f"{prefix}AutoMemoryStep")
assert registry.get("step", f"{name}_agentic_answer_step") is getattr(answer, f"{prefix}AgenticAnswerStep")
assert registry.get("step", "memlens_auto_memory_step") is memory.MemLensAutoMemoryStep
assert registry.get("step", "memlens_agentic_answer_step") is answer.MemLensAgenticAnswerStep
@pytest.mark.asyncio
@pytest.mark.parametrize("sources", [True, False])
async def test_memory_saves_originals_and_native_image_links(candidate, sources):
_, memory, _, prefix, app, wrapper, store, workspace = candidate
step = getattr(memory, f"{prefix}AutoMemoryStep")(app_context=app, agent_wrapper=wrapper, file_store=store)
memory, _, app, wrapper, store, workspace = candidate
step = memory.MemLensAutoMemoryStep(app_context=app, agent_wrapper=wrapper, file_store=store)
step._list_session_note = AsyncMock(return_value=None)
picture = BytesIO()
Image.new("RGB", (1, 1), (12, 34, 56)).save(picture, format="PNG")
@ -165,7 +168,7 @@ async def test_memory_saves_originals_and_native_image_links(candidate, sources)
@pytest.mark.asyncio
async def test_read_image_returns_real_formatter_image_and_rejects_escape(candidate):
_, _, answer, prefix, app, wrapper, store, workspace = candidate
_, answer, app, wrapper, store, workspace = candidate
path = workspace / "session/images/one/photo.png"
path.parent.mkdir(parents=True)
Image.new("RGB", (2, 2), (12, 34, 56)).save(path)
@ -176,7 +179,7 @@ async def test_read_image_returns_real_formatter_image_and_rejects_escape(candid
return step.context.response
app.jobs["read_image"] = AsyncMock(side_effect=native_read)
step = getattr(answer, f"{prefix}AgenticAnswerStep")(app_context=app, agent_wrapper=wrapper, file_store=store)
step = answer.MemLensAgenticAnswerStep(app_context=app, agent_wrapper=wrapper, file_store=store)
tool = answer.image_reader(step)
result = await tool.call(path="session/images/one/photo.png")
assert result.state == ToolResultState.SUCCESS and isinstance(result.content[1], DataBlock)
@ -200,7 +203,7 @@ async def test_read_image_returns_real_formatter_image_and_rejects_escape(candid
@pytest.mark.asyncio
@pytest.mark.parametrize("sources", [True, False])
async def test_answer_has_fresh_state_bm25_scope_and_no_gold(candidate, sources):
_, _, answer, prefix, app, wrapper, store, workspace = candidate
_, answer, app, wrapper, store, workspace = candidate
for relative in ["daily/note.md", "digest/note.md", "session/dialog/one.jsonl", "mem_session/private.md"]:
path = workspace / relative
path.parent.mkdir(parents=True, exist_ok=True)
@ -224,7 +227,7 @@ async def test_answer_has_fresh_state_bm25_scope_and_no_gold(candidate, sources)
read = AsyncMock(return_value=Response(answer="evidence"))
read.name, read.description, read.parameters = "read", "read", {"type": "object", "properties": {}}
app.jobs = {"search": job, "read": read}
step = getattr(answer, f"{prefix}AgenticAnswerStep")(app_context=app, agent_wrapper=wrapper, file_store=store)
step = answer.MemLensAgenticAnswerStep(app_context=app, agent_wrapper=wrapper, file_store=store)
sessions, contexts = [], []
for _ in range(2):
response = await step(query="Official question", include_source_sessions=sources)

View file

@ -17,12 +17,10 @@ from reme.plugin_manifest import parse_plugin_manifest
@pytest.fixture(name="judge")
def fixture_judge(monkeypatch):
name = "memlens"
root = Path(__file__).resolve().parents[2] / "plugins" / f"{name}-judge" / "src"
root = Path(__file__).resolve().parents[2] / "plugins/memlens-judge/src"
monkeypatch.syspath_prepend(str(root))
module = import_module(f"judge_{name}.llm_judge")
prefix = "MemLens"
step = getattr(module, f"{prefix}JudgeStep")()
module = import_module("judge_memlens.llm_judge")
step = module.MemLensJudgeStep()
step.context = RuntimeContext.from_context(
None,
question="QUESTION",
@ -38,12 +36,12 @@ def fixture_judge(monkeypatch):
usage=ChatUsage(100, 10, 0.01, cache_input_tokens=30),
),
)
return name, step, import_module(f"judge_{name}.protocol"), root
return step, import_module("judge_memlens.protocol"), root
@pytest.mark.asyncio
async def test_judge_job_uses_only_official_prompt_and_separate_usage(judge):
name, step, protocol, root = judge
step, protocol, root = judge
response = await step.execute()
assert response.success and response.metadata["llm_judge_score"] == 1
assert response.metadata["usage"]["cache_input_tokens"] == 30
@ -51,10 +49,10 @@ async def test_judge_job_uses_only_official_prompt_and_separate_usage(judge):
expected = protocol.judge_prompt("QUESTION", "REFERENCE", "PREDICTION", **kwargs)
assert step.as_llm.call_args.args[0][0].get_text_content() == expected
assert len(step.as_llm.call_args.args[0]) == 1 and not step.as_llm.call_args.kwargs
text = (root / f"judge_{name}" / "plugin.yaml").read_text()
parse_plugin_manifest(text, plugin_name=f"{name}-judge")
text = (root / "judge_memlens/plugin.yaml").read_text()
parse_plugin_manifest(text, plugin_name="memlens-judge")
manifest = yaml.safe_load(text)["application_defaults"]
assert manifest["jobs"]["answer_judge"]["steps"] == [{"backend": f"{name}_judge_step", "as_llm": "judge"}]
assert manifest["jobs"]["answer_judge"]["steps"] == [{"backend": "memlens_judge_step", "as_llm": "judge"}]
model = manifest["components"]["as_llm"]["judge"]
assert model["extra_body"] == {"enable_thinking": False} and model["parameters"]["temperature"] == 0
assert model["parameters"]["max_tokens"] == 256

View file

@ -0,0 +1,485 @@
"""Offline failure, accounting and artifact contracts for the MemLens runner."""
# pylint: disable=missing-function-docstring,protected-access
import asyncio
import json
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from benchmark.memlens import run as runner
from benchmark.memlens.artifacts import error_details, judge_settings, safe_value
from reme.schema import Response
@pytest.fixture(autouse=True)
def memlens_judge_path(monkeypatch):
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "plugins/memlens-judge/src"))
@pytest.fixture(name="scene_run")
def fixture_scene_run(tmp_path, monkeypatch):
import reme
scene = {
"id": "scene-1",
"sessions": [{"id": "session-1", "timestamp": "2026-01-01"}],
"questions": [
{
"id": "question-1",
"text": "Question",
"question_date": "2026-01-02",
"answer": "Answer",
"category": "information_extraction",
},
],
}
app = SimpleNamespace(start=AsyncMock(), close=AsyncMock(), run_job=AsyncMock())
judge = SimpleNamespace(run_job=AsyncMock(return_value=Response(metadata={"llm_judge_score": 1})))
monkeypatch.setattr(reme, "Application", lambda **_: app)
monkeypatch.setattr(runner, "app_config", lambda *_, **__: {})
monkeypatch.setattr(runner, "observe_usage", lambda _: None)
monkeypatch.setattr(runner, "session_messages", lambda _: [{"role": "user", "content": "Text"}])
case = tmp_path / "reme_image/scenes/0000-scene-1"
async def evaluate():
return await runner.evaluate_scene(scene, 0, "candidate", tmp_path, judge, asyncio.Semaphore(1))
return scene, app, judge, case, evaluate
def measured(tokens=120):
return runner.normalized_usage({"input_tokens": tokens - 20, "output_tokens": 20})
@pytest.mark.asyncio
async def test_index_failure_retains_completed_session_usage(scene_run):
_, app, _, case, evaluate = scene_run
async def job(name, **_):
if name == "auto_memory":
runner._CURRENT.get().append(measured())
return Response(answer="Memory")
return Response(success=False, answer="Index unavailable")
app.run_job.side_effect = job
results, memory, answers = await evaluate()
assert results == answers == []
assert runner.total_usage(memory)["total_tokens"] == 120
status = json.loads((case / "status.json").read_text())
assert status["stage"] == "index_update" and status["scene_id"] == "scene-1"
assert "response detail: Index unavailable" in status["error"]["message"]
assert status["error_type"] == "RuntimeError"
assert status["memory_complete"] is False
assert len(list((case / "ingestion/attempt-001/sessions").glob("*.json"))) == 1
artifact = json.loads((case / "ingestion/attempt-001/diagnostics/index-response.json").read_text())
assert artifact["response"]["success"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize("direct", [False, True])
async def test_failed_memory_keeps_its_successful_requests_and_diagnostics(scene_run, direct):
_, app, _, case, evaluate = scene_run
async def job(*_, **__):
runner._CURRENT.get().append(measured())
if direct:
try:
raise ValueError("Original provider failure")
except ValueError as exc:
raise RuntimeError("Memory failed") from exc
return Response(success=False, metadata={"error": "Memory failed"})
app.run_job.side_effect = job
_, memory, _ = await evaluate()
assert runner.total_usage(memory)["total_tokens"] == 120
status = json.loads((case / "status.json").read_text())
assert status["stage"] == "auto_memory" and status["session_id"] == "session-1"
assert status["memory_complete"] is False
assert not (case / "ingestion/attempt-001/sessions/00000.json").exists()
assert not list((case / "ingestion/attempt-001/sessions").glob("*.json"))
assert json.loads((case / "ingestion/attempt-001/diagnostics/00000-usage.json").read_text())["total_tokens"] == 120
if direct:
assert status["error"]["cause"]["message"] == "Original provider failure"
assert status["error"]["traceback"] and "locals" not in json.dumps(status)
else:
artifact = json.loads((case / "ingestion/attempt-001/diagnostics/00000-response.json").read_text())
assert artifact["response"]["success"] is False
app.close.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize("failed_job", ["agentic_answer", "judge"])
async def test_failed_answer_or_judge_keeps_answer_usage_once(scene_run, failed_job):
_, app, judge, case, evaluate = scene_run
async def job(name, **_):
if name in {"auto_memory", "agentic_answer"}:
runner._CURRENT.get().append(measured())
return Response(answer="Answer", success=name != failed_job)
app.run_job.side_effect = job
if failed_job == "judge":
judge.run_job.return_value = Response(success=False, metadata={"error": "Judge failed"})
results, memory, answers = await evaluate()
assert results == [] and len(memory) == len(answers) == 1
assert runner.total_usage(answers)["total_tokens"] == 120
status = json.loads((case / "status.json").read_text())
assert status["question_id"] == "question-1" and status["stage"] == failed_job
assert status["memory_complete"] is True
path = "judge-response.json" if failed_job == "judge" else "answer-response.json"
assert json.loads((case / "questions/0000/diagnostics" / path).read_text())["response"]["success"] is False
assert (case / "questions/0000/answer.json").exists() is (failed_job == "judge")
assert json.loads((case / "questions/0000/diagnostics/answer-usage.json").read_text())["total_tokens"] == 120
@pytest.mark.asyncio
@pytest.mark.parametrize("stage", ["preprocess", "start", "cleanup", "cancel"])
async def test_direct_lifecycle_failures_are_recorded(scene_run, monkeypatch, stage):
_, app, _, case, evaluate = scene_run
app.run_job.return_value = Response(answer="Answer")
if stage == "start":
app.start.side_effect = RuntimeError("startup failed")
elif stage == "cleanup":
app.close.side_effect = RuntimeError("cleanup failed")
elif stage == "cancel":
app.run_job.side_effect = asyncio.CancelledError()
else:
def broken(_):
raise ValueError("Invalid image")
monkeypatch.setattr(runner, "session_messages", broken)
if stage == "cancel":
with pytest.raises(asyncio.CancelledError):
await evaluate()
else:
await evaluate()
status = json.loads((case / "status.json").read_text())
assert status["complete"] is False
assert status["stage"] == ("auto_memory" if stage == "cancel" else stage)
assert status["error"]["type"] == (
"CancelledError" if stage == "cancel" else "RuntimeError" if stage in {"start", "cleanup"} else "ValueError"
)
@pytest.mark.asyncio
async def test_observer_separates_concurrent_scopes_including_compression_and_unknown_counters():
def response(usage, status=200):
return SimpleNamespace(status_code=status, is_stream_consumed=True, json=lambda: {"usage": usage})
client = SimpleNamespace(send=AsyncMock(side_effect=response))
compressor = SimpleNamespace(send=AsyncMock(side_effect=response))
app = SimpleNamespace(
context=SimpleNamespace(
components={
"as_llm": {
"default": SimpleNamespace(model=SimpleNamespace(client=SimpleNamespace(_client=client))),
"compressor": SimpleNamespace(model=SimpleNamespace(client=SimpleNamespace(_client=compressor))),
}
}
)
)
runner.observe_usage(app)
async def request(tokens):
with runner.collect_usage() as records:
await client.send({"prompt_tokens": tokens, "completion_tokens": 20})
await asyncio.sleep(0)
await compressor.send({})
return records
first, second = await asyncio.gather(request(100), request(200))
assert runner.total_usage(first)["total_tokens"] == 120
assert runner.total_usage(second)["total_tokens"] == 220
assert runner.total_usage(first)["records"] == 2
assert runner.total_usage(first)["coverage"]["total_tokens"] == 1
assert runner.total_usage(first)["cached_input_tokens"] is None
assert runner._CURRENT.get() is None
@pytest.mark.asyncio
@pytest.mark.parametrize("direct", [False, True])
async def test_failure_artifacts_redact_secrets_and_request_payloads(scene_run, monkeypatch, direct):
_, app, _, case, evaluate = scene_run
secret = "ENV_SECRET_SENTINEL_314159"
monkeypatch.setenv("LLM_API_KEY", secret)
detail = (
f"provider failed {secret}; api_key=INLINE_SENTINEL "
"Bearer AUTH_SENTINEL https://USER_SENTINEL:PASS_SENTINEL@provider.test/v1?key=URL_SENTINEL "
f"data:image/png;base64,{'QmFzZTY0' * 20}"
)
if direct:
app.run_job.side_effect = RuntimeError(detail)
else:
app.run_job.return_value = Response(
success=False,
metadata={
"error": detail,
"request_headers": {"arbitrary": "HEADER_SENTINEL"},
"config": {"arbitrary": "CONFIG_SENTINEL"},
"source": {"type": "base64", "data": "SHORT_IMAGE_SENTINEL"},
},
)
await evaluate()
artifacts = "".join(path.read_text() for path in case.rglob("*.json"))
for value in (
secret,
"INLINE_SENTINEL",
"AUTH_SENTINEL",
"USER_SENTINEL",
"PASS_SENTINEL",
"URL_SENTINEL",
"HEADER_SENTINEL",
"CONFIG_SENTINEL",
"SHORT_IMAGE_SENTINEL",
"QmFzZTY0",
):
assert value not in artifacts
assert "provider failed" in artifacts and "[REDACTED" in artifacts
@pytest.mark.asyncio
async def test_successful_predictions_and_references_remain_verbatim(scene_run):
scene, app, judge, case, evaluate = scene_run
answer = "token=ordinary-text https://example.test/path?a=b data:image/png;base64," + "QmFzZTY0" * 20
scene["questions"][0]["answer"] = answer
app.run_job.return_value = Response(answer=answer)
results, _, _ = await evaluate()
assert results[0]["prediction"] == results[0]["reference"] == answer
assert judge.run_job.call_args.kwargs["prediction"] == answer
for name in ("answer.json", "result.json"):
assert json.loads((case / "questions/0000" / name).read_text())["prediction"] == answer
assert json.loads((case / "status.json").read_text())["memory_complete"] is True
@pytest.mark.asyncio
async def test_cancellation_finalizes_observed_requests_before_propagating(scene_run):
_, app, _, case, evaluate = scene_run
async def canceled(*_, **__):
runner._CURRENT.get().append(measured())
raise asyncio.CancelledError()
app.run_job.side_effect = canceled
with pytest.raises(asyncio.CancelledError):
await evaluate()
assert json.loads((case / "status.json").read_text())["memory_usage"]["total_tokens"] == 120
assert json.loads((case / "ingestion/attempt-001/diagnostics/00000-usage.json").read_text())["total_tokens"] == 120
app.close.assert_awaited_once()
@pytest.mark.asyncio
async def test_cancellation_during_cleanup_finishes_owned_close(scene_run):
_, app, _, case, evaluate = scene_run
app.run_job.return_value = Response(answer="Answer")
started, release, finished = asyncio.Event(), asyncio.Event(), asyncio.Event()
async def close():
started.set()
await release.wait()
finished.set()
app.close.side_effect = close
task = asyncio.create_task(evaluate())
await asyncio.wait_for(started.wait(), timeout=2)
task.cancel()
await asyncio.sleep(0)
assert not finished.is_set()
release.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=2)
assert finished.is_set()
status = json.loads((case / "status.json").read_text())
assert status["error_type"] == "CancelledError" and status["stage"] == "cleanup"
@pytest.mark.parametrize("field", ["cookie", "set-cookie", "private_key", "id_token", "refresh_token", "token"])
def test_diagnostic_credentials_are_redacted_without_removing_counters(field):
diagnostic = safe_value({field: "DIAGNOSTIC_SECRET_SENTINEL", "input_tokens": 120, "output_tokens": 20})
assert diagnostic == {field: "[REDACTED]", "input_tokens": 120, "output_tokens": 20}
assert "DIAGNOSTIC_SECRET_SENTINEL" not in json.dumps(
error_details(RuntimeError(f"{field}: DIAGNOSTIC_SECRET_SENTINEL"))
)
def test_diagnostic_private_key_blocks_are_redacted():
text = "-----BEGIN PRIVATE KEY-----\nPEM_SENTINEL\n-----END PRIVATE KEY-----"
assert safe_value(text) == "[REDACTED_KEY]"
@pytest.mark.parametrize("header", ["Cookie", "Set-Cookie"])
def test_diagnostic_cookie_headers_redact_the_entire_value(header):
assert safe_value(f"{header}: session=FIRST_SENTINEL; other=SECOND_SENTINEL") == f"{header}: [REDACTED]"
@pytest.mark.asyncio
async def test_judge_snapshot_uses_validated_plugin_and_environment_config(tmp_path, monkeypatch):
from importlib.metadata import EntryPoint
from reme import Application
from reme.plugin import PluginManager, _load_plugin
plugin = _load_plugin(
"memlens-judge",
EntryPoint(
name="memlens-judge",
value="judge_memlens",
group="reme.plugins",
),
)
manager = PluginManager([plugin])
monkeypatch.setattr(PluginManager, "discover", classmethod(lambda cls, _: manager))
monkeypatch.setenv("JUDGE_MODEL_NAME", "judge-override-for-test")
monkeypatch.setenv("LLM_API_KEY", "SNAPSHOT_SECRET_SENTINEL")
monkeypatch.setenv("LLM_BASE_URL", "https://private.test/v1?key=URL_SENTINEL")
inputs = runner.app_config(tmp_path / "judge", "candidate-model", judge=True)
assert inputs["components"] == {}
app = Application(**inputs)
snapshot = judge_settings(app)
model = snapshot["models"]["judge"]
assert model["model"] == "judge-override-for-test"
assert model["parameters"] == {
"temperature": 0,
"thinking_enable": False,
"max_tokens": 256,
}
assert model["extra_body"] == {"enable_thinking": False}
assert model["client_kwargs"] == {"timeout": 600, "max_retries": 0}
assert snapshot["protocol_id"] == "memlens-task-binary-api-qwen-v1"
assert "SENTINEL" not in json.dumps(snapshot) and "credential" not in json.dumps(snapshot)
await app.close()
@pytest.mark.asyncio
async def test_summary_versions_request_usage_and_whitelists_settings(scene_run, monkeypatch):
import reme
from reme.schema import ApplicationConfig
scene, _, _, case, _ = scene_run
root = case.parents[2]
judge = SimpleNamespace(
start=AsyncMock(),
close=AsyncMock(),
config=ApplicationConfig(workspace_dir=root),
)
monkeypatch.setattr(reme, "Application", lambda **_: judge)
monkeypatch.setattr(runner, "load_memlens", lambda *_: [scene])
monkeypatch.setattr(runner, "evaluate_scene", AsyncMock(return_value=([], [measured()], [])))
summary = await runner.run(
{
"dataset": str(root / "unused"),
"output": str(root / "output"),
"query_concurrency": 1,
"scenario_concurrency": 1,
"unexpected": "RAW_CONFIG_SENTINEL",
},
"candidate",
"128k",
)
assert summary["usage_policy_version"] == "successful-http-requests-v2"
assert summary["expected_questions"] == summary["unscored_questions"] == 1
assert summary["scored_questions"] == 0 and summary["token_usage"]["total_tokens"] == 120
settings = (root / "output/memlens-128k/candidate/settings.json").read_text()
assert "RAW_CONFIG_SENTINEL" not in settings
assert json.loads(settings)["judge"]["protocol_id"] == "memlens-task-binary-api-qwen-v1"
@pytest.mark.asyncio
@pytest.mark.parametrize("stage", ["preprocess", "judge_start", "judge_cleanup", "cancel"])
async def test_run_lifecycle_keeps_safe_status(tmp_path, monkeypatch, stage):
import reme
from reme.schema import ApplicationConfig
judge = SimpleNamespace(start=AsyncMock(), close=AsyncMock(), config=ApplicationConfig(workspace_dir=tmp_path))
monkeypatch.setattr(reme, "Application", lambda **_: judge)
monkeypatch.setattr(runner, "app_config", lambda *_, **__: {})
monkeypatch.setattr(runner, "load_memlens", lambda *_: [])
monkeypatch.setenv("LLM_API_KEY", "RUN_SECRET_SENTINEL")
error = RuntimeError("RUN_SECRET_SENTINEL failed")
if stage == "preprocess":
def broken(*_):
raise error
monkeypatch.setattr(runner, "load_memlens", broken)
elif stage == "judge_start":
judge.start.side_effect = error
elif stage == "judge_cleanup":
judge.close.side_effect = error
else:
judge.start.side_effect = asyncio.CancelledError()
with pytest.raises(asyncio.CancelledError if stage == "cancel" else RuntimeError):
await runner.run(
{
"dataset": str(tmp_path / "unused"),
"output": str(tmp_path / "output"),
"query_concurrency": 1,
"scenario_concurrency": 1,
},
"candidate",
"128k",
)
text = (tmp_path / "output/memlens-128k/candidate/status.json").read_text()
assert "RUN_SECRET_SENTINEL" not in text
status = json.loads(text)
assert status["complete"] is False
assert status["stage"] == ("judge_start" if stage == "cancel" else stage)
def test_cli_reports_sanitized_direct_failure(monkeypatch, capsys):
monkeypatch.setattr("sys.argv", ["memlens"])
monkeypatch.setattr(runner, "run", AsyncMock(side_effect=RuntimeError("provider api_key=CLI_SECRET_SENTINEL")))
assert runner.main() == 2
diagnostic = capsys.readouterr().err
assert "CLI_SECRET_SENTINEL" not in diagnostic
assert json.loads(diagnostic)["error"]["message"] == "provider api_key=[REDACTED]"
@pytest.mark.asyncio
async def test_scene_cleanup_failure_retains_scores_but_fails_operational_status(scene_run, monkeypatch):
import reme
from reme.schema import ApplicationConfig
scene, app, judge, case, _ = scene_run
root = case.parents[2]
judge.start, judge.close = AsyncMock(), AsyncMock()
judge.config = ApplicationConfig(workspace_dir=root)
applications = iter([judge, app])
monkeypatch.setattr(reme, "Application", lambda **_: next(applications))
monkeypatch.setattr(runner, "load_memlens", lambda *_: [scene])
app.run_job.return_value = Response(answer="Answer")
app.close.side_effect = RuntimeError("Cleanup failed")
summary = await runner.run(
{
"dataset": str(root / "unused"),
"output": str(root / "output"),
"query_concurrency": 1,
"scenario_concurrency": 1,
},
"candidate",
"128k",
)
assert summary["scored_questions"] == summary["expected_questions"] == 1
assert summary["metrics"] == {"llm_judge": 1, "sub_em": 1, "f1": 1}
assert summary["cleanup_error_count"] == summary["memory_complete_scenes"] == 1
assert summary["completed_scenes"] == 0
assert json.loads((root / "output/memlens-128k/candidate/status.json").read_text())["complete"] is False
def test_cli_cleanup_failure_is_nonzero_even_when_all_questions_scored(monkeypatch):
monkeypatch.setattr("sys.argv", ["memlens"])
monkeypatch.setattr(
runner,
"run",
AsyncMock(
return_value={
"scored_questions": 1,
"expected_questions": 1,
"cleanup_error_count": 1,
}
),
)
assert runner.main() == 2