mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-18 09:42:31 +00:00
fix(benchmark): preserve MemLens diagnostic schemas during redaction
This commit is contained in:
parent
14c459cfd9
commit
5139829dd6
3 changed files with 142 additions and 17 deletions
|
|
@ -39,7 +39,10 @@ _MODEL_OPTIONS = {
|
|||
def _safe_text(value: str) -> str:
|
||||
for key, secret in sorted(os.environ.items(), key=lambda item: len(item[1]), reverse=True):
|
||||
if secret and _PRIVATE_KEY.search(key):
|
||||
value = value.replace(secret, "[REDACTED]")
|
||||
if len(secret) < 12:
|
||||
value = re.sub(r"(?<![\w-])" + re.escape(secret) + r"(?![\w-])", "[REDACTED]", value)
|
||||
else:
|
||||
value = value.replace(secret, "[REDACTED]")
|
||||
|
||||
def safe_url(match):
|
||||
try:
|
||||
|
|
@ -84,18 +87,37 @@ def safe_value(value):
|
|||
return f"<{type(value).__name__}>"
|
||||
|
||||
|
||||
def status_record(status: dict) -> dict:
|
||||
"""Keep runner-owned schema/enums/counters intact; errors are already sanitized."""
|
||||
return {
|
||||
key: safe_value(value) if key in {"scene_id", "session_id", "question_id"} else value
|
||||
for key, value in status.items()
|
||||
}
|
||||
|
||||
|
||||
def response_record(response) -> dict:
|
||||
"""Keep the Response schema while sanitizing its untrusted payload and extras."""
|
||||
raw = response.model_dump()
|
||||
return {
|
||||
**safe_value({key: value for key, value in raw.items() if key not in {"success", "answer", "metadata"}}),
|
||||
"success": raw["success"],
|
||||
"answer": safe_value(raw["answer"]),
|
||||
"metadata": safe_value(raw["metadata"]),
|
||||
}
|
||||
|
||||
|
||||
def error_details(exc: BaseException, seen: set[int] | None = None) -> dict:
|
||||
"""Keep the error chain and frame locations, never frame locals or source text."""
|
||||
seen = set() if seen is None else seen
|
||||
seen.add(id(exc))
|
||||
result = {
|
||||
"type": type(exc).__name__,
|
||||
"message": str(exc),
|
||||
"message": _safe_text(str(exc)),
|
||||
"traceback": [
|
||||
{
|
||||
"file": Path(frame.filename).name,
|
||||
"file": _safe_text(Path(frame.filename).name),
|
||||
"line": frame.lineno,
|
||||
"function": frame.name,
|
||||
"function": _safe_text(frame.name),
|
||||
}
|
||||
for frame in traceback.extract_tb(exc.__traceback__)
|
||||
],
|
||||
|
|
@ -103,7 +125,7 @@ def error_details(exc: BaseException, seen: set[int] | None = None) -> dict:
|
|||
cause = exc.__cause__ or (None if exc.__suppress_context__ else exc.__context__)
|
||||
if cause is not None and id(cause) not in seen:
|
||||
result["cause"] = error_details(cause, seen)
|
||||
return safe_value(result)
|
||||
return result
|
||||
|
||||
|
||||
def model_settings(config) -> dict:
|
||||
|
|
@ -111,12 +133,12 @@ def model_settings(config) -> dict:
|
|||
models = {}
|
||||
for name, component in config.components.get("as_llm", {}).items():
|
||||
spec = component.model_dump()
|
||||
snapshot = {key: spec[key] for key in _MODEL_FIELDS if key in spec}
|
||||
snapshot = {key: safe_value(spec[key]) for key in _MODEL_FIELDS if key in spec}
|
||||
for key, allowed in _MODEL_OPTIONS.items():
|
||||
options = spec.get(key) or {}
|
||||
snapshot[key] = {name: options[name] for name in allowed if name in options}
|
||||
models[name] = snapshot
|
||||
return safe_value(models)
|
||||
snapshot[key] = {name: safe_value(options[name]) for name in allowed if name in options}
|
||||
models[_safe_text(name)] = snapshot
|
||||
return models
|
||||
|
||||
|
||||
def judge_settings(app) -> dict:
|
||||
|
|
|
|||
|
|
@ -18,7 +18,14 @@ import sys
|
|||
|
||||
import yaml
|
||||
|
||||
from benchmark.memlens.artifacts import error_details, judge_settings, model_settings, safe_value
|
||||
from benchmark.memlens.artifacts import (
|
||||
error_details,
|
||||
judge_settings,
|
||||
model_settings,
|
||||
response_record,
|
||||
safe_value,
|
||||
status_record,
|
||||
)
|
||||
|
||||
# Adapted query/metric code retains its upstream MIT notice in README.md.
|
||||
SYSTEM_PROMPT = (
|
||||
|
|
@ -386,7 +393,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
|
|||
if not response.success:
|
||||
save(
|
||||
directory / "diagnostics" / f"{ordinal:05d}-response.json",
|
||||
safe_value({"response": response.model_dump()}),
|
||||
{"response": response_record(response)},
|
||||
)
|
||||
check_response(response)
|
||||
save(
|
||||
|
|
@ -403,7 +410,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
|
|||
if not response.success:
|
||||
save(
|
||||
case / "ingestion/attempt-001/diagnostics/index-response.json",
|
||||
safe_value({"response": response.model_dump()}),
|
||||
{"response": response_record(response)},
|
||||
)
|
||||
check_response(response)
|
||||
status["memory_complete"] = True
|
||||
|
|
@ -425,7 +432,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
|
|||
if not response.success:
|
||||
save(
|
||||
directory / "diagnostics/answer-response.json",
|
||||
safe_value({"response": response.model_dump()}),
|
||||
{"response": response_record(response)},
|
||||
)
|
||||
check_response(response)
|
||||
save(
|
||||
|
|
@ -449,7 +456,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
|
|||
old_answer=question.get("old_answer", ""),
|
||||
)
|
||||
if not judgment.success:
|
||||
save(directory / "diagnostics/judge-response.json", safe_value({"response": judgment.model_dump()}))
|
||||
save(directory / "diagnostics/judge-response.json", {"response": response_record(judgment)})
|
||||
check_response(judgment)
|
||||
status["stage"] = "score"
|
||||
metrics = score_answer(question, response.answer)
|
||||
|
|
@ -486,7 +493,7 @@ async def evaluate_scene(scene, number, model, group, judge, slots):
|
|||
status.update(memory_usage=total_usage(memory), answer_usage=total_usage(answers))
|
||||
if "error" in status:
|
||||
status["error_type"] = status["error"]["type"]
|
||||
save(case / "status.json", safe_value(status))
|
||||
save(case / "status.json", status_record(status))
|
||||
return results, memory, answers
|
||||
|
||||
|
||||
|
|
@ -594,7 +601,7 @@ async def run(settings: dict, model: str, length: str, limit: int | None = None)
|
|||
status.update(complete=False, stage="judge_cleanup", error=error_details(exc))
|
||||
raise
|
||||
finally:
|
||||
save(group / "status.json", safe_value(status))
|
||||
save(group / "status.json", status_record(status))
|
||||
return summary
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from unittest.mock import AsyncMock
|
|||
import pytest
|
||||
|
||||
from benchmark.memlens import run as runner
|
||||
from benchmark.memlens.artifacts import error_details, judge_settings, safe_value
|
||||
from benchmark.memlens.artifacts import error_details, judge_settings, model_settings, response_record, safe_value
|
||||
from reme.schema import Response
|
||||
|
||||
|
||||
|
|
@ -315,6 +315,102 @@ def test_diagnostic_cookie_headers_redact_the_entire_value(header):
|
|||
assert safe_value(f"{header}: session=FIRST_SENTINEL; other=SECOND_SENTINEL") == f"{header}: [REDACTED]"
|
||||
|
||||
|
||||
def test_short_environment_secret_does_not_replace_ordinary_word_substrings(monkeypatch):
|
||||
monkeypatch.setenv("LLM_API_KEY", "a")
|
||||
assert safe_value("candidate stage q_a1 default") == "candidate stage q_a1 default"
|
||||
assert safe_value("received credential: a") == "received credential: [REDACTED]"
|
||||
assert safe_value({"a": "a", "api_key": "a"}) == {"[REDACTED]": "[REDACTED]", "api_key": "[REDACTED]"}
|
||||
|
||||
|
||||
def test_arbitrary_response_keys_and_values_still_redact_known_secrets(monkeypatch):
|
||||
secret = "ENV_SECRET_SENTINEL_314159"
|
||||
monkeypatch.setenv("LLM_API_KEY", secret)
|
||||
response = Response(
|
||||
success=False,
|
||||
answer=f"provider failed {secret}",
|
||||
metadata={f"debug-{secret}": f"detail-{secret}", "private_key": "PRIVATE_SENTINEL"},
|
||||
**{f"extra-{secret}": secret},
|
||||
)
|
||||
record = response_record(response)
|
||||
assert secret not in json.dumps(record) and "PRIVATE_SENTINEL" not in json.dumps(record)
|
||||
assert record["success"] is False and "provider failed" in record["answer"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("secret", ["response", "success", "answer", "metadata"])
|
||||
async def test_failed_response_schema_survives_environment_name_collision(scene_run, monkeypatch, secret):
|
||||
_, app, _, case, evaluate = scene_run
|
||||
monkeypatch.setenv("LLM_API_KEY", secret)
|
||||
app.run_job.return_value = Response(success=False, answer="Provider unavailable", metadata={"code": 503})
|
||||
await evaluate()
|
||||
record = json.loads((case / "ingestion/attempt-001/diagnostics/00000-response.json").read_text())
|
||||
assert record["response"]["success"] is False
|
||||
assert record["response"]["answer"] == "Provider unavailable"
|
||||
assert record["response"]["metadata"] == {"code": 503}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("secret", ["a", "model", "temperature"])
|
||||
def test_model_snapshot_preserves_whitelisted_schema(monkeypatch, secret):
|
||||
monkeypatch.setenv("LLM_API_KEY", secret)
|
||||
spec = {"backend": "agentscope", "model": "candidate", "parameters": {"temperature": 0, "max_tokens": 256}}
|
||||
config = SimpleNamespace(components={"as_llm": {"default": SimpleNamespace(model_dump=lambda: spec)}})
|
||||
snapshot = model_settings(config)["default"]
|
||||
assert snapshot["backend"] == "agentscope" and snapshot["model"] == "candidate"
|
||||
assert snapshot["parameters"] == {"temperature": 0, "max_tokens": 256}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("secret", ["a", "stage", "complete", "cleanup", "input_tokens"])
|
||||
@pytest.mark.parametrize("cleanup_failure", [False, True])
|
||||
async def test_run_summary_survives_environment_schema_collision(scene_run, monkeypatch, secret, cleanup_failure):
|
||||
import reme
|
||||
from reme.schema import ApplicationConfig
|
||||
|
||||
scene, app, judge, case, _ = scene_run
|
||||
root = case.parents[2]
|
||||
monkeypatch.setenv("LLM_API_KEY", secret)
|
||||
monkeypatch.setattr(runner, "app_config", lambda workspace, *_, **__: {"workspace_dir": str(workspace)})
|
||||
monkeypatch.setattr(runner, "load_memlens", lambda *_: [scene])
|
||||
judge.start, judge.close = AsyncMock(), AsyncMock()
|
||||
judge.config = ApplicationConfig(workspace_dir=root / "judge")
|
||||
applications = iter([judge, app])
|
||||
monkeypatch.setattr(reme, "Application", lambda **_: next(applications))
|
||||
|
||||
async def job(name, **_):
|
||||
if name != "index_update":
|
||||
runner._CURRENT.get().append(measured())
|
||||
return Response(answer="Answer")
|
||||
|
||||
app.run_job.side_effect = job
|
||||
if cleanup_failure:
|
||||
app.close.side_effect = RuntimeError("Close failed")
|
||||
summary = await runner.run(
|
||||
{
|
||||
"dataset": str(root / "unused"),
|
||||
"output": str(root / "output"),
|
||||
"query_concurrency": 1,
|
||||
"scenario_concurrency": 1,
|
||||
},
|
||||
"candidate",
|
||||
"128k",
|
||||
)
|
||||
group = root / "output/memlens-128k/candidate"
|
||||
status = json.loads((group / "reme_image/scenes/0000-scene-1/status.json").read_text())
|
||||
assert status["stage"] == ("cleanup" if cleanup_failure else "complete")
|
||||
assert status["scene_id"] == "scene-1" and status["question_id"] == "question-1"
|
||||
assert status["memory_complete"] is True and status["complete"] is not cleanup_failure
|
||||
assert status["memory_usage"]["input_tokens"] == 100 and status["answer_usage"]["total_tokens"] == 120
|
||||
assert summary["scored_questions"] == summary["memory_complete_scenes"] == 1
|
||||
assert summary["token_usage"]["total_tokens"] == 240 and summary["token_usage"]["records"] == 2
|
||||
assert summary["cleanup_error_count"] == int(cleanup_failure)
|
||||
assert summary["completed_scenes"] == int(not cleanup_failure)
|
||||
assert json.loads((group / "summary.json").read_text()) == summary
|
||||
assert json.loads((group / "status.json").read_text())["complete"] is not cleanup_failure
|
||||
if cleanup_failure:
|
||||
assert status["error_type"] == status["error"]["type"] == "RuntimeError"
|
||||
assert status["error"]["message"] == "Close failed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_judge_snapshot_uses_validated_plugin_and_environment_config(tmp_path, monkeypatch):
|
||||
from importlib.metadata import EntryPoint
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue