From 7f259c5aeb3d1d22d0bd337fe0ac70d66a0d90ce Mon Sep 17 00:00:00 2001 From: wang-qisen Date: Mon, 26 Oct 2026 12:35:37 +0800 Subject: [PATCH] fix(memlens): preserve nested usage fields during diagnostic redaction --- benchmark/memlens/artifacts.py | 21 +++++++++++++----- tests/unit/test_memlens_runner_failures.py | 25 ++++++++++++++++------ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/benchmark/memlens/artifacts.py b/benchmark/memlens/artifacts.py index 9b448a15..18c48749 100644 --- a/benchmark/memlens/artifacts.py +++ b/benchmark/memlens/artifacts.py @@ -34,6 +34,17 @@ _MODEL_OPTIONS = { "extra_body": ("enable_thinking",), "client_kwargs": ("timeout", "max_retries"), } +_USAGE_FIELDS = { + "usage", + "coverage", + "records", + "input_tokens", + "output_tokens", + "total_tokens", + "cached_input_tokens", + "noncached_input_tokens", + "reasoning_tokens", +} def _safe_text(value: str) -> str: @@ -67,19 +78,19 @@ def _safe_text(value: str) -> str: ) -def safe_value(value): +def safe_value(value, *, fixed_keys=()): """Redact diagnostic data recursively without serializing arbitrary objects.""" if isinstance(value, dict): return { - _safe_text(str(key)): ( + (key if key in fixed_keys else _safe_text(str(key))): ( "[REDACTED]" if _PRIVATE_KEY.search(str(key)) or (key == "data" and value.get("type") == "base64") - else safe_value(item) + else safe_value(item, fixed_keys=fixed_keys) ) for key, item in value.items() } if isinstance(value, (list, tuple)): - return [safe_value(item) for item in value] + return [safe_value(item, fixed_keys=fixed_keys) for item in value] if isinstance(value, str): return _safe_text(value) if value is None or isinstance(value, (bool, int, float)): @@ -102,7 +113,7 @@ def response_record(response) -> dict: **safe_value({key: value for key, value in raw.items() if key not in {"success", "answer", "metadata"}}), "success": raw["success"], "answer": safe_value(raw["answer"]), - "metadata": safe_value(raw["metadata"]), + "metadata": safe_value(raw["metadata"], fixed_keys=_USAGE_FIELDS), } diff --git a/tests/unit/test_memlens_runner_failures.py b/tests/unit/test_memlens_runner_failures.py index 423e7b10..210aa39b 100644 --- a/tests/unit/test_memlens_runner_failures.py +++ b/tests/unit/test_memlens_runner_failures.py @@ -287,16 +287,29 @@ def test_diagnostic_credentials_are_redacted_without_removing_counters(field): @pytest.mark.asyncio -@pytest.mark.parametrize("secret", ["response", "success", "answer", "metadata"]) +@pytest.mark.parametrize( + "secret", + ["response", "success", "answer", "metadata", "usage", "coverage", "records", "input_tokens"], +) async def test_failed_response_schema_survives_environment_name_collision(scene_run, monkeypatch, secret): _, app, _, case, evaluate = scene_run - monkeypatch.setenv("LLM_API_KEY", secret) - app.run_job.return_value = Response(success=False, answer="Provider unavailable", metadata={"code": 503}) + monkeypatch.setenv("TEST_PRIVATE_KEY", secret) + usage = {"input_tokens": 100, "output_tokens": 20, "records": 1, "coverage": {"input_tokens": 1}} + app.run_job.return_value = Response( + success=False, + answer=f"Provider unavailable: {secret}", + metadata={"usage": usage, "private_key": "PRIVATE_SENTINEL", f"debug {secret}": f"detail {secret}"}, + ) await evaluate() record = json.loads((case / "ingestion/attempt-001/diagnostics/00000-response.json").read_text()) - assert record["response"]["success"] is False - assert record["response"]["answer"] == "Provider unavailable" - assert record["response"]["metadata"] == {"code": 503} + response = record["response"] + assert Response.model_validate(response).success is False + assert response["answer"] == "Provider unavailable: [REDACTED]" + assert response["metadata"] == { + "usage": usage, + "private_key": "[REDACTED]", + "debug [REDACTED]": "detail [REDACTED]", + } @pytest.mark.parametrize("secret", ["a", "model", "temperature"])