fix(benchmark): preserve MemLens diagnostic schemas during redaction

This commit is contained in:
wang-qisen 2026-10-20 12:19:54 +08:00
parent 14c459cfd9
commit 5139829dd6
3 changed files with 142 additions and 17 deletions

View file

@ -39,7 +39,10 @@ _MODEL_OPTIONS = {
def _safe_text(value: str) -> str:
for key, secret in sorted(os.environ.items(), key=lambda item: len(item[1]), reverse=True):
if secret and _PRIVATE_KEY.search(key):
value = value.replace(secret, "[REDACTED]")
if len(secret) < 12:
value = re.sub(r"(?<![\w-])" + re.escape(secret) + r"(?![\w-])", "[REDACTED]", value)
else:
value = value.replace(secret, "[REDACTED]")
def safe_url(match):
try:
@ -84,18 +87,37 @@ def safe_value(value):
return f"<{type(value).__name__}>"
def status_record(status: dict) -> dict:
"""Keep runner-owned schema/enums/counters intact; errors are already sanitized."""
return {
key: safe_value(value) if key in {"scene_id", "session_id", "question_id"} else value
for key, value in status.items()
}
def response_record(response) -> dict:
"""Keep the Response schema while sanitizing its untrusted payload and extras."""
raw = response.model_dump()
return {
**safe_value({key: value for key, value in raw.items() if key not in {"success", "answer", "metadata"}}),
"success": raw["success"],
"answer": safe_value(raw["answer"]),
"metadata": safe_value(raw["metadata"]),
}
def error_details(exc: BaseException, seen: set[int] | None = None) -> dict:
"""Keep the error chain and frame locations, never frame locals or source text."""
seen = set() if seen is None else seen
seen.add(id(exc))
result = {
"type": type(exc).__name__,
"message": str(exc),
"message": _safe_text(str(exc)),
"traceback": [
{
"file": Path(frame.filename).name,
"file": _safe_text(Path(frame.filename).name),
"line": frame.lineno,
"function": frame.name,
"function": _safe_text(frame.name),
}
for frame in traceback.extract_tb(exc.__traceback__)
],
@ -103,7 +125,7 @@ def error_details(exc: BaseException, seen: set[int] | None = None) -> dict:
cause = exc.__cause__ or (None if exc.__suppress_context__ else exc.__context__)
if cause is not None and id(cause) not in seen:
result["cause"] = error_details(cause, seen)
return safe_value(result)
return result
def model_settings(config) -> dict:
@ -111,12 +133,12 @@ def model_settings(config) -> dict:
models = {}
for name, component in config.components.get("as_llm", {}).items():
spec = component.model_dump()
snapshot = {key: spec[key] for key in _MODEL_FIELDS if key in spec}
snapshot = {key: safe_value(spec[key]) for key in _MODEL_FIELDS if key in spec}
for key, allowed in _MODEL_OPTIONS.items():
options = spec.get(key) or {}
snapshot[key] = {name: options[name] for name in allowed if name in options}
models[name] = snapshot
return safe_value(models)
snapshot[key] = {name: safe_value(options[name]) for name in allowed if name in options}
models[_safe_text(name)] = snapshot
return models
def judge_settings(app) -> dict:

View file

@ -18,7 +18,14 @@ import sys
import yaml
from benchmark.memlens.artifacts import error_details, judge_settings, model_settings, safe_value
from benchmark.memlens.artifacts import (
error_details,
judge_settings,
model_settings,
response_record,
safe_value,
status_record,
)
# Adapted query/metric code retains its upstream MIT notice in README.md.
SYSTEM_PROMPT = (
@ -386,7 +393,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
if not response.success:
save(
directory / "diagnostics" / f"{ordinal:05d}-response.json",
safe_value({"response": response.model_dump()}),
{"response": response_record(response)},
)
check_response(response)
save(
@ -403,7 +410,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
if not response.success:
save(
case / "ingestion/attempt-001/diagnostics/index-response.json",
safe_value({"response": response.model_dump()}),
{"response": response_record(response)},
)
check_response(response)
status["memory_complete"] = True
@ -425,7 +432,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
if not response.success:
save(
directory / "diagnostics/answer-response.json",
safe_value({"response": response.model_dump()}),
{"response": response_record(response)},
)
check_response(response)
save(
@ -449,7 +456,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
old_answer=question.get("old_answer", ""),
)
if not judgment.success:
save(directory / "diagnostics/judge-response.json", safe_value({"response": judgment.model_dump()}))
save(directory / "diagnostics/judge-response.json", {"response": response_record(judgment)})
check_response(judgment)
status["stage"] = "score"
metrics = score_answer(question, response.answer)
@ -486,7 +493,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
status.update(memory_usage=total_usage(memory), answer_usage=total_usage(answers))
if "error" in status:
status["error_type"] = status["error"]["type"]
save(case / "status.json", safe_value(status))
save(case / "status.json", status_record(status))
return results, memory, answers
@ -594,7 +601,7 @@ async def run(settings: dict, model: str, length: str, limit: int | None = None)
status.update(complete=False, stage="judge_cleanup", error=error_details(exc))
raise
finally:
save(group / "status.json", safe_value(status))
save(group / "status.json", status_record(status))
return summary

View file

@ -10,7 +10,7 @@ from unittest.mock import AsyncMock
import pytest
from benchmark.memlens import run as runner
from benchmark.memlens.artifacts import error_details, judge_settings, safe_value
from benchmark.memlens.artifacts import error_details, judge_settings, model_settings, response_record, safe_value
from reme.schema import Response
@ -315,6 +315,102 @@ def test_diagnostic_cookie_headers_redact_the_entire_value(header):
assert safe_value(f"{header}: session=FIRST_SENTINEL; other=SECOND_SENTINEL") == f"{header}: [REDACTED]"
def test_short_environment_secret_does_not_replace_ordinary_word_substrings(monkeypatch):
monkeypatch.setenv("LLM_API_KEY", "a")
assert safe_value("candidate stage q_a1 default") == "candidate stage q_a1 default"
assert safe_value("received credential: a") == "received credential: [REDACTED]"
assert safe_value({"a": "a", "api_key": "a"}) == {"[REDACTED]": "[REDACTED]", "api_key": "[REDACTED]"}
def test_arbitrary_response_keys_and_values_still_redact_known_secrets(monkeypatch):
secret = "ENV_SECRET_SENTINEL_314159"
monkeypatch.setenv("LLM_API_KEY", secret)
response = Response(
success=False,
answer=f"provider failed {secret}",
metadata={f"debug-{secret}": f"detail-{secret}", "private_key": "PRIVATE_SENTINEL"},
**{f"extra-{secret}": secret},
)
record = response_record(response)
assert secret not in json.dumps(record) and "PRIVATE_SENTINEL" not in json.dumps(record)
assert record["success"] is False and "provider failed" in record["answer"]
@pytest.mark.asyncio
@pytest.mark.parametrize("secret", ["response", "success", "answer", "metadata"])
async def test_failed_response_schema_survives_environment_name_collision(scene_run, monkeypatch, secret):
_, app, _, case, evaluate = scene_run
monkeypatch.setenv("LLM_API_KEY", secret)
app.run_job.return_value = Response(success=False, answer="Provider unavailable", metadata={"code": 503})
await evaluate()
record = json.loads((case / "ingestion/attempt-001/diagnostics/00000-response.json").read_text())
assert record["response"]["success"] is False
assert record["response"]["answer"] == "Provider unavailable"
assert record["response"]["metadata"] == {"code": 503}
@pytest.mark.parametrize("secret", ["a", "model", "temperature"])
def test_model_snapshot_preserves_whitelisted_schema(monkeypatch, secret):
monkeypatch.setenv("LLM_API_KEY", secret)
spec = {"backend": "agentscope", "model": "candidate", "parameters": {"temperature": 0, "max_tokens": 256}}
config = SimpleNamespace(components={"as_llm": {"default": SimpleNamespace(model_dump=lambda: spec)}})
snapshot = model_settings(config)["default"]
assert snapshot["backend"] == "agentscope" and snapshot["model"] == "candidate"
assert snapshot["parameters"] == {"temperature": 0, "max_tokens": 256}
@pytest.mark.asyncio
@pytest.mark.parametrize("secret", ["a", "stage", "complete", "cleanup", "input_tokens"])
@pytest.mark.parametrize("cleanup_failure", [False, True])
async def test_run_summary_survives_environment_schema_collision(scene_run, monkeypatch, secret, cleanup_failure):
import reme
from reme.schema import ApplicationConfig
scene, app, judge, case, _ = scene_run
root = case.parents[2]
monkeypatch.setenv("LLM_API_KEY", secret)
monkeypatch.setattr(runner, "app_config", lambda workspace, *_, **__: {"workspace_dir": str(workspace)})
monkeypatch.setattr(runner, "load_memlens", lambda *_: [scene])
judge.start, judge.close = AsyncMock(), AsyncMock()
judge.config = ApplicationConfig(workspace_dir=root / "judge")
applications = iter([judge, app])
monkeypatch.setattr(reme, "Application", lambda **_: next(applications))
async def job(name, **_):
if name != "index_update":
runner._CURRENT.get().append(measured())
return Response(answer="Answer")
app.run_job.side_effect = job
if cleanup_failure:
app.close.side_effect = RuntimeError("Close failed")
summary = await runner.run(
{
"dataset": str(root / "unused"),
"output": str(root / "output"),
"query_concurrency": 1,
"scenario_concurrency": 1,
},
"candidate",
"128k",
)
group = root / "output/memlens-128k/candidate"
status = json.loads((group / "reme_image/scenes/0000-scene-1/status.json").read_text())
assert status["stage"] == ("cleanup" if cleanup_failure else "complete")
assert status["scene_id"] == "scene-1" and status["question_id"] == "question-1"
assert status["memory_complete"] is True and status["complete"] is not cleanup_failure
assert status["memory_usage"]["input_tokens"] == 100 and status["answer_usage"]["total_tokens"] == 120
assert summary["scored_questions"] == summary["memory_complete_scenes"] == 1
assert summary["token_usage"]["total_tokens"] == 240 and summary["token_usage"]["records"] == 2
assert summary["cleanup_error_count"] == int(cleanup_failure)
assert summary["completed_scenes"] == int(not cleanup_failure)
assert json.loads((group / "summary.json").read_text()) == summary
assert json.loads((group / "status.json").read_text())["complete"] is not cleanup_failure
if cleanup_failure:
assert status["error_type"] == status["error"]["type"] == "RuntimeError"
assert status["error"]["message"] == "Close failed"
@pytest.mark.asyncio
async def test_judge_snapshot_uses_validated_plugin_and_environment_config(tmp_path, monkeypatch):
from importlib.metadata import EntryPoint