From 5139829dd69d0d44060a99879f2e2d55cfce660e Mon Sep 17 00:00:00 2001 From: wang-qisen Date: Tue, 20 Oct 2026 12:19:54 +0800 Subject: [PATCH] fix(benchmark): preserve MemLens diagnostic schemas during redaction --- benchmark/memlens/artifacts.py | 40 +++++++-- benchmark/memlens/run.py | 21 +++-- tests/unit/test_memlens_runner_failures.py | 98 +++++++++++++++++++++- 3 files changed, 142 insertions(+), 17 deletions(-) diff --git a/benchmark/memlens/artifacts.py b/benchmark/memlens/artifacts.py index cbe4bfe7..9b448a15 100644 --- a/benchmark/memlens/artifacts.py +++ b/benchmark/memlens/artifacts.py @@ -39,7 +39,10 @@ _MODEL_OPTIONS = { def _safe_text(value: str) -> str: for key, secret in sorted(os.environ.items(), key=lambda item: len(item[1]), reverse=True): if secret and _PRIVATE_KEY.search(key): - value = value.replace(secret, "[REDACTED]") + if len(secret) < 12: + value = re.sub(r"(?" +def status_record(status: dict) -> dict: + """Keep runner-owned schema/enums/counters intact; errors are already sanitized.""" + return { + key: safe_value(value) if key in {"scene_id", "session_id", "question_id"} else value + for key, value in status.items() + } + + +def response_record(response) -> dict: + """Keep the Response schema while sanitizing its untrusted payload and extras.""" + raw = response.model_dump() + return { + **safe_value({key: value for key, value in raw.items() if key not in {"success", "answer", "metadata"}}), + "success": raw["success"], + "answer": safe_value(raw["answer"]), + "metadata": safe_value(raw["metadata"]), + } + + def error_details(exc: BaseException, seen: set[int] | None = None) -> dict: """Keep the error chain and frame locations, never frame locals or source text.""" seen = set() if seen is None else seen seen.add(id(exc)) result = { "type": type(exc).__name__, - "message": str(exc), + "message": _safe_text(str(exc)), "traceback": [ { - "file": Path(frame.filename).name, + "file": _safe_text(Path(frame.filename).name), "line": frame.lineno, - "function": frame.name, + "function": _safe_text(frame.name), } for frame in traceback.extract_tb(exc.__traceback__) ], @@ -103,7 +125,7 @@ def error_details(exc: BaseException, seen: set[int] | None = None) -> dict: cause = exc.__cause__ or (None if exc.__suppress_context__ else exc.__context__) if cause is not None and id(cause) not in seen: result["cause"] = error_details(cause, seen) - return safe_value(result) + return result def model_settings(config) -> dict: @@ -111,12 +133,12 @@ def model_settings(config) -> dict: models = {} for name, component in config.components.get("as_llm", {}).items(): spec = component.model_dump() - snapshot = {key: spec[key] for key in _MODEL_FIELDS if key in spec} + snapshot = {key: safe_value(spec[key]) for key in _MODEL_FIELDS if key in spec} for key, allowed in _MODEL_OPTIONS.items(): options = spec.get(key) or {} - snapshot[key] = {name: options[name] for name in allowed if name in options} - models[name] = snapshot - return safe_value(models) + snapshot[key] = {name: safe_value(options[name]) for name in allowed if name in options} + models[_safe_text(name)] = snapshot + return models def judge_settings(app) -> dict: diff --git a/benchmark/memlens/run.py b/benchmark/memlens/run.py index 237c38eb..15833aa9 100644 --- a/benchmark/memlens/run.py +++ b/benchmark/memlens/run.py @@ -18,7 +18,14 @@ import sys import yaml -from benchmark.memlens.artifacts import error_details, judge_settings, model_settings, safe_value +from benchmark.memlens.artifacts import ( + error_details, + judge_settings, + model_settings, + response_record, + safe_value, + status_record, +) # Adapted query/metric code retains its upstream MIT notice in README.md. SYSTEM_PROMPT = ( @@ -386,7 +393,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots): if not response.success: save( directory / "diagnostics" / f"{ordinal:05d}-response.json", - safe_value({"response": response.model_dump()}), + {"response": response_record(response)}, ) check_response(response) save( @@ -403,7 +410,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots): if not response.success: save( case / "ingestion/attempt-001/diagnostics/index-response.json", - safe_value({"response": response.model_dump()}), + {"response": response_record(response)}, ) check_response(response) status["memory_complete"] = True @@ -425,7 +432,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots): if not response.success: save( directory / "diagnostics/answer-response.json", - safe_value({"response": response.model_dump()}), + {"response": response_record(response)}, ) check_response(response) save( @@ -449,7 +456,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots): old_answer=question.get("old_answer", ""), ) if not judgment.success: - save(directory / "diagnostics/judge-response.json", safe_value({"response": judgment.model_dump()})) + save(directory / "diagnostics/judge-response.json", {"response": response_record(judgment)}) check_response(judgment) status["stage"] = "score" metrics = score_answer(question, response.answer) @@ -486,7 +493,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots): status.update(memory_usage=total_usage(memory), answer_usage=total_usage(answers)) if "error" in status: status["error_type"] = status["error"]["type"] - save(case / "status.json", safe_value(status)) + save(case / "status.json", status_record(status)) return results, memory, answers @@ -594,7 +601,7 @@ async def run(settings: dict, model: str, length: str, limit: int | None = None) status.update(complete=False, stage="judge_cleanup", error=error_details(exc)) raise finally: - save(group / "status.json", safe_value(status)) + save(group / "status.json", status_record(status)) return summary diff --git a/tests/unit/test_memlens_runner_failures.py b/tests/unit/test_memlens_runner_failures.py index b91d8843..43bc32b1 100644 --- a/tests/unit/test_memlens_runner_failures.py +++ b/tests/unit/test_memlens_runner_failures.py @@ -10,7 +10,7 @@ from unittest.mock import AsyncMock import pytest from benchmark.memlens import run as runner -from benchmark.memlens.artifacts import error_details, judge_settings, safe_value +from benchmark.memlens.artifacts import error_details, judge_settings, model_settings, response_record, safe_value from reme.schema import Response @@ -315,6 +315,102 @@ def test_diagnostic_cookie_headers_redact_the_entire_value(header): assert safe_value(f"{header}: session=FIRST_SENTINEL; other=SECOND_SENTINEL") == f"{header}: [REDACTED]" +def test_short_environment_secret_does_not_replace_ordinary_word_substrings(monkeypatch): + monkeypatch.setenv("LLM_API_KEY", "a") + assert safe_value("candidate stage q_a1 default") == "candidate stage q_a1 default" + assert safe_value("received credential: a") == "received credential: [REDACTED]" + assert safe_value({"a": "a", "api_key": "a"}) == {"[REDACTED]": "[REDACTED]", "api_key": "[REDACTED]"} + + +def test_arbitrary_response_keys_and_values_still_redact_known_secrets(monkeypatch): + secret = "ENV_SECRET_SENTINEL_314159" + monkeypatch.setenv("LLM_API_KEY", secret) + response = Response( + success=False, + answer=f"provider failed {secret}", + metadata={f"debug-{secret}": f"detail-{secret}", "private_key": "PRIVATE_SENTINEL"}, + **{f"extra-{secret}": secret}, + ) + record = response_record(response) + assert secret not in json.dumps(record) and "PRIVATE_SENTINEL" not in json.dumps(record) + assert record["success"] is False and "provider failed" in record["answer"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("secret", ["response", "success", "answer", "metadata"]) +async def test_failed_response_schema_survives_environment_name_collision(scene_run, monkeypatch, secret): + _, app, _, case, evaluate = scene_run + monkeypatch.setenv("LLM_API_KEY", secret) + app.run_job.return_value = Response(success=False, answer="Provider unavailable", metadata={"code": 503}) + await evaluate() + record = json.loads((case / "ingestion/attempt-001/diagnostics/00000-response.json").read_text()) + assert record["response"]["success"] is False + assert record["response"]["answer"] == "Provider unavailable" + assert record["response"]["metadata"] == {"code": 503} + + +@pytest.mark.parametrize("secret", ["a", "model", "temperature"]) +def test_model_snapshot_preserves_whitelisted_schema(monkeypatch, secret): + monkeypatch.setenv("LLM_API_KEY", secret) + spec = {"backend": "agentscope", "model": "candidate", "parameters": {"temperature": 0, "max_tokens": 256}} + config = SimpleNamespace(components={"as_llm": {"default": SimpleNamespace(model_dump=lambda: spec)}}) + snapshot = model_settings(config)["default"] + assert snapshot["backend"] == "agentscope" and snapshot["model"] == "candidate" + assert snapshot["parameters"] == {"temperature": 0, "max_tokens": 256} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("secret", ["a", "stage", "complete", "cleanup", "input_tokens"]) +@pytest.mark.parametrize("cleanup_failure", [False, True]) +async def test_run_summary_survives_environment_schema_collision(scene_run, monkeypatch, secret, cleanup_failure): + import reme + from reme.schema import ApplicationConfig + + scene, app, judge, case, _ = scene_run + root = case.parents[2] + monkeypatch.setenv("LLM_API_KEY", secret) + monkeypatch.setattr(runner, "app_config", lambda workspace, *_, **__: {"workspace_dir": str(workspace)}) + monkeypatch.setattr(runner, "load_memlens", lambda *_: [scene]) + judge.start, judge.close = AsyncMock(), AsyncMock() + judge.config = ApplicationConfig(workspace_dir=root / "judge") + applications = iter([judge, app]) + monkeypatch.setattr(reme, "Application", lambda **_: next(applications)) + + async def job(name, **_): + if name != "index_update": + runner._CURRENT.get().append(measured()) + return Response(answer="Answer") + + app.run_job.side_effect = job + if cleanup_failure: + app.close.side_effect = RuntimeError("Close failed") + summary = await runner.run( + { + "dataset": str(root / "unused"), + "output": str(root / "output"), + "query_concurrency": 1, + "scenario_concurrency": 1, + }, + "candidate", + "128k", + ) + group = root / "output/memlens-128k/candidate" + status = json.loads((group / "reme_image/scenes/0000-scene-1/status.json").read_text()) + assert status["stage"] == ("cleanup" if cleanup_failure else "complete") + assert status["scene_id"] == "scene-1" and status["question_id"] == "question-1" + assert status["memory_complete"] is True and status["complete"] is not cleanup_failure + assert status["memory_usage"]["input_tokens"] == 100 and status["answer_usage"]["total_tokens"] == 120 + assert summary["scored_questions"] == summary["memory_complete_scenes"] == 1 + assert summary["token_usage"]["total_tokens"] == 240 and summary["token_usage"]["records"] == 2 + assert summary["cleanup_error_count"] == int(cleanup_failure) + assert summary["completed_scenes"] == int(not cleanup_failure) + assert json.loads((group / "summary.json").read_text()) == summary + assert json.loads((group / "status.json").read_text())["complete"] is not cleanup_failure + if cleanup_failure: + assert status["error_type"] == status["error"]["type"] == "RuntimeError" + assert status["error"]["message"] == "Close failed" + + @pytest.mark.asyncio async def test_judge_snapshot_uses_validated_plugin_and_environment_config(tmp_path, monkeypatch): from importlib.metadata import EntryPoint