From 8e7e6bc91dd2f2d6ce7415ca5fa0ee9f123aed85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E7=89=9B=E8=82=89?= Date: Fri, 30 Oct 2026 02:23:21 +0800 Subject: [PATCH] feat(home): add case-study agent and query evidence tracing tools - Add home_case_study job with CaseStudyAgentWrapper and a read-only sandboxed investigation step for per-case diagnostic reports - Add home_trace_query_evidence job/step to locate a query's source evidence sessions, memory checkpoints and answer/score previews - Compact evaluation summaries: record per-query failure errors, round scores to four decimals, expose scoring_enabled and observed means - Slim run manifests: drop schema_version and trusted_judge fingerprint, exclude debug/sampling_policy options, keep construction_source only for query-only replay - Simplify history index internals and update guide docs and tests --- plugins/home/README.md | 113 +++++++- plugins/home/examples/prepare.yaml | 1 - plugins/home/pyproject.toml | 2 +- .../home/src/reme_home/agent/case_study.py | 209 ++++++++++++++ .../home/src/reme_home/agent/case_study.yaml | 42 +++ plugins/home/src/reme_home/agent/wrapper.py | 21 ++ .../src/reme_home/evaluation/artifacts.py | 69 +++-- .../src/reme_home/evaluation/compaction.py | 24 ++ .../src/reme_home/evaluation/component.py | 12 +- .../src/reme_home/evaluation/evidence_step.py | 142 ++++++++++ .../home/src/reme_home/evaluation/history.py | 34 +-- .../home/src/reme_home/evaluation/inputs.py | 2 +- plugins/home/src/reme_home/plugin.yaml | 42 +++ .../src/reme_home/preparation/component.py | 1 - .../src/reme_home/preparation/data/basic.py | 33 +-- .../src/reme_home/preparation/data/models.py | 3 +- .../home/src/reme_home/preparation/spec.py | 2 +- .../runtime_docs/validation-artifact-guide.md | 69 ++++- plugins/home/src/reme_home/sampling.py | 1 - plugins/home/tests/test_case_study.py | 267 ++++++++++++++++++ plugins/home/tests/test_data_preparation.py | 6 +- plugins/home/tests/test_evaluation.py | 26 +- .../home/tests/test_evaluation_artifacts.py | 79 +++++- plugins/home/tests/test_evaluation_history.py | 37 ++- plugins/home/tests/test_preparation.py | 2 +- plugins/home/tests/test_query_evidence.py | 221 +++++++++++++++ 26 files changed, 1312 insertions(+), 148 deletions(-) create mode 100644 plugins/home/src/reme_home/agent/case_study.py create mode 100644 plugins/home/src/reme_home/agent/case_study.yaml create mode 100644 plugins/home/src/reme_home/evaluation/evidence_step.py create mode 100644 plugins/home/tests/test_case_study.py create mode 100644 plugins/home/tests/test_query_evidence.py diff --git a/plugins/home/README.md b/plugins/home/README.md index bdf55e25..997d9e84 100644 --- a/plugins/home/README.md +++ b/plugins/home/README.md @@ -140,10 +140,10 @@ All prepared outputs live under `workspace_dir/home/` (paths below are relative - `domain_spec.yaml`: resolved configuration, including actual selected case IDs and the full source commit SHA. -- `datasets/training-set/`: normalized Case/Session/Query JSON and a versioned +- `datasets/training-set/`: normalized Case/Session/Query JSON and a manifest with case IDs and content fingerprints. -- `datasets/debug-cases/`: deterministic small smoke cases and their derivation - record, or an empty `cases/` directory when debug derivation is disabled. +- `datasets/debug-cases/`: deterministic small smoke cases with source case IDs + in their metadata, or an empty `cases/` directory when debug derivation is disabled. - `code/runtime/`: fixed committed ReMe sources, packaging files and configurations, without plugin projects. Uncommitted source edits are excluded. - `code/repo/`: an independent Git repository containing complete selected plugin @@ -251,6 +251,9 @@ cases// /answer.log ``` +Structured scores in query results and summaries use at most four decimal places. +Raw judge text and reference answers are preserved. + The memory tar is exported **before** queries, retains session Git checkpoints, and is independent of the query execution state. Replayed runs keep an independent donor tar and inherited build log without charging construction @@ -322,8 +325,9 @@ because preparation owns multiple workspace output locations. registered as the `home_bounded` agent-wrapper backend when Home is enabled. It reuses ReMe's Job tools, hidden `injected_job_kwargs`, `tool_context_id`, built-in tools, permissions, model configuration, skills, token accounting and -persisted sessions. No agent component or optimization run is enabled automatically. -Configure a component explicitly, alongside the required model and Home plugins: +persisted sessions. Home configures a wrapper for the single-case investigation +Job below; no investigation or optimization runs automatically. Configure other +agents explicitly, alongside the required model and Home plugins: ```yaml components: @@ -571,14 +575,14 @@ only derived caches so they can rebuild from the final source run status. history = app.context.components["home.evaluation_history"]["default"] results = history.list_results() # Typed EvaluationRecord objects, chronological. result = history.get_result(results[-1].run_key) -history.rebuild() # Rebuild derived indexes from saved artifacts. +history.rebuild() # Rebuild sampling observations from saved artifacts. ``` The Component owns the workspace lock for these synchronous filesystem methods; -async host code can run them in a worker thread. Its compact catalog lives at -`home/evaluations/history.json`. Each record points to its run via `run_key` and -contains commit/branch identity, selected IDs, completion time and summary metrics; -questions, answers and memory files stay in their original result directory. +async host code can run them in a worker thread. Successful results are discovered +from each run's manifest and summary; no `history.json` index is written or read. +Legacy index files are ignored and left untouched. Each returned record points to +its run via `run_key`; questions, answers and memory files stay in that result directory. The separate startup ledger advances only after the first successfully initialized sandbox, including when a later worker or execution fails. Extra/replacement @@ -660,11 +664,27 @@ closed for restricted `_allowed_paths` contexts, like the other Home Jobs. ### Memory checkpoint inspection -Two read-only Jobs investigate a case's exported memory Git repository. Locate +Read-only Jobs investigate a case's exported memory Git repository. Locate an evaluation and case with `home_list_validation_results` and -`home_inspect_validation` first. Neither Job changes artifacts, Git refs, the +`home_inspect_validation` first. These Jobs never change artifacts, Git refs, the index, evaluation history or sampling state. +- `home_trace_query_evidence(validation_path, case_id, query_id, + evidence_session_ids=null)` joins a selected query to its source sessions and + memory checkpoints without calling a model. Dataset evidence annotations take + precedence over optional inferred session IDs, including an empty annotation. + Returns `query` (question, expected/actual answer, score, error and + `result_missing`), `evidence_source` (`independent_annotation`, + `diagnostician_inference`, or `unavailable`), `evidence_is_certain`, + `memory_status`, `sessions` (ID, checkpoint and changed files), and `paths` to + the normalized case, memory repository, query results and query artifacts. + A null checkpoint means no matching `session: ` commit was found; missing + results keep scores null. Answer/question previews are limited to 2,000 + characters and errors to 600. At most 50 sessions and 50 changed files per + session are returned, with omitted counts when needed. Session text, diffs, + later history and full answer/judge results remain at the returned paths or + in the inspection/comparison Jobs below. Original transcripts and derived + memories can both be searchable; these coordinates alone do not prove a cause. - `home_inspect_memory_history(validation_path, case_id, path=null, offset=0, limit=20)` returns `status`, `workspace_exists`, `has_git_history`, `head`, `repository`, and newest-first checkpoint `rows` with `row_count` and @@ -696,6 +716,7 @@ After starting the service with the Home configuration, call from another termin ```bash reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1 +reme home_trace_query_evidence validation_path=evaluations/candidate/abcdef0/1 case_id=case1 query_id=query1 reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 case_id=case1 base_ref=HEAD~1 reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 target_validation_path=evaluations/candidate/1234567/1 case_id=case1 ``` @@ -704,5 +725,69 @@ reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 target_ form is `evaluations/candidate/abcdef0/1`. For agent tools, host `injected_job_kwargs` can pin `validation_path` and `case_id`; the wrapper hides injected fields and rejects model overrides. Like the other Home inspection Jobs, these Jobs reject -`_allowed_paths` restrictions rather than bypassing them. Query-to-session evidence -tracing and the case study agent are separate functionality. +`_allowed_paths` restrictions rather than bypassing them. Use these tools directly +or through the single-case investigator below. + +## Single-case investigation + +`home_case_study(validation_path, case_id, focus=...)` runs one fresh diagnostic +agent. It follows dev_meta's case-study workflow: trace judging and answer +failures backward through retrieval, memory construction, source sessions and +the evaluated implementation. The text report cites evidence paths/lines or +JSON fields and distinguishes supported causes, hypotheses and missing evidence; +it does not prescribe fixes. There is no batch fan-out or parallel case-study +dispatcher. Each invocation has its own agent state and timer. + +The `home_case_study` component uses `home_case_study_agent`, a thin subclass of +the bounded wrapper, defaulting to +the host's existing `as_llm: judge` model, 60 iterations and a 30-minute advisory +budget, matching dev_meta's default budgets. Configure a separate host model by +overriding `components.agent_wrapper.home_case_study.as_llm`. ReMe owns model +startup, token accounting and saved session transcripts. The response answer is +the report; metadata includes the validation, case, evaluated commit, session ID +and `usage`. No case scratch directory or intermediate analysis files are created. + +The toolkit includes validation inspection, compact query-evidence tracing, +memory history/comparison, and the three committed-code navigation Jobs. +Validation/case selectors and the evaluated code revision are hidden and enforced +by the host; memory comparison stays within the selected evaluation. General +Read/Grep/Glob, Write/Edit, validation and code-mutation tools are not exposed. + +`AutoBash` and `python_execute` share a required OS sandbox. Commands have no +static whitelist (`is_read_only=false`); filesystem permissions enforce access. +Read-only roots are the selected case artifact directory, normalized source-case +JSON, validation manifest, all of `home/code/`, and `home/docs/`. No host writable +roots are granted. System/interpreter runtime paths and disposable backend +temporary storage remain available. Direct code reads can see current worktree +changes; use the revision-pinned Jobs to establish the evaluated behavior. +`python_execute` accepts either `code` or an existing `file`, optional file-only +`args`, and a `timeout` in seconds (default 60, maximum 600). It uses `python -I -B` +and returns returncode/stdout/stderr/timed_out. Both executors reuse bounded output, +timeouts, cancellation cleanup and network isolation from the platform backend. + +Linux requires Bubblewrap and, with the default 4096-MiB limit, a writable +delegated cgroup v2. macOS requires Seatbelt (`sandbox-exec`); memory limits are +not enforced there. The Job probes sandbox startup before calling the model and +never falls back to an unrestricted executor. The command whitelist parser is +not used by this agent. Sandbox policy, output-report token guidance and executor +memory policy are host Step configuration, not model arguments: + +```yaml +jobs: + home_case_study: + steps: + - backend: home_case_study_step + agent_wrapper: home_case_study + output_token_limit: 2500 + # memory_limit_mib: 0 # Explicitly disable Linux cgroup memory limiting if needed. +``` + +With the configured service running, call from another terminal: + +```bash +reme home_case_study validation_path=evaluations/candidate/abcdef0/1 case_id=case1 'focus=Trace low-scoring queries' +``` + +Calls with `_allowed_paths` restrictions fail closed, as other Home artifact +Jobs do. Structured inspection tools run in the host with their existing +containment checks; only Bash/Python subprocesses run inside the OS sandbox. diff --git a/plugins/home/examples/prepare.yaml b/plugins/home/examples/prepare.yaml index 5a4fbcf1..1299c21d 100644 --- a/plugins/home/examples/prepare.yaml +++ b/plugins/home/examples/prepare.yaml @@ -1,4 +1,3 @@ -schema_version: 1 benchmark: name: beam variant: 100K diff --git a/plugins/home/pyproject.toml b/plugins/home/pyproject.toml index 40bf5fc2..e549f13c 100644 --- a/plugins/home/pyproject.toml +++ b/plugins/home/pyproject.toml @@ -17,7 +17,7 @@ where = ["src"] include = ["reme_home*"] [tool.setuptools.package-data] -reme_home = ["plugin.yaml", "runtime_docs/*.md"] +reme_home = ["plugin.yaml", "runtime_docs/*.md", "agent/*.yaml"] [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" diff --git a/plugins/home/src/reme_home/agent/case_study.py b/plugins/home/src/reme_home/agent/case_study.py new file mode 100644 index 00000000..cccebd1d --- /dev/null +++ b/plugins/home/src/reme_home/agent/case_study.py @@ -0,0 +1,209 @@ +"""One case-local investigation using ReMe Jobs and sandboxed command tools.""" + +import sys +from pathlib import Path + +from agentscope.tool import FunctionTool, Toolkit +from pydantic import Field, field_validator +import yaml + +from reme.steps.base_step import BaseStep, Ref +from reme.utils.file_sandbox import create_file_sandbox_backend + +from ..auto_bash import AutoBash +from ..code.repository import resolve_revision, validate_repository +from ..evaluation.artifacts import contained +from ..evaluation.component import blocking +from ..evaluation.evidence_step import _source_case +from ..evaluation.inputs import entity_id +from ..evaluation.memory_inspection_step import _repository +from ..preparation.spec import SpecModel +from ..workspace import HomeWorkspace +from .wrapper import BoundedAgentWrapper + +PROMPT = Path(__file__).with_suffix(".yaml") +CASE_TOOLS = ( + "home_inspect_validation", + "home_trace_query_evidence", + "home_inspect_memory_history", + "home_compare_memory", +) +CODE_TOOLS = ("home_list_code_tree", "home_search_code", "home_read_code_file") + + +class CaseStudyInput(SpecModel): + """Select one existing validation case; runtime and permissions are host-owned.""" + + validation_path: str = Field(min_length=1) + case_id: str = Field(min_length=1) + focus: str = Field( + default="Investigate the failed or low-scoring queries and their earliest supported causes.", + min_length=1, + ) + + @field_validator("case_id") + @classmethod + def validate_case(cls, value): + """Keep the case inside its manifest-selected artifact directory.""" + return entity_id(value) + + +def prepare_scope(home, spec): + """Resolve evidence and the validation's immutable candidate revision before model calls.""" + run, manifest, _memory = _repository(home, spec.validation_path, spec.case_id) + case_path = contained(run, f"cases/{spec.case_id}", directory=True) + if not case_path.is_dir(): + raise ValueError("Case artifacts do not exist") + source_path, _case = _source_case(home, manifest, spec.case_id) + repo, _record = validate_repository(home) + commit = manifest.get("commit_sha") + if not isinstance(commit, str) or len(commit) != 40 or any(char not in "0123456789abcdef" for char in commit): + raise ValueError("Evaluation manifest must record a full candidate commit_sha") + revision = resolve_revision(repo, commit) + roots = [ + case_path, + source_path, + contained(run, "manifest.json"), + contained(home, "code", directory=True), + contained(home, "docs", directory=True), + ] + return { + "validation_path": str(run), + "case_id": spec.case_id, + "case_path": str(case_path), + "candidate_commit": revision, + "code_repository": str(repo), + "source_case": str(source_path), + "docs": str(home / "docs"), + "read_only_paths": [str(path) for path in roots], + } + + +def python_tool(backend, cwd): + """Run source or an existing script with the same read-only sandbox as AutoBash.""" + + async def python_execute( + code: str | None = None, + timeout: float = 60, + *, + file: str | None = None, + args: list[str] | None = None, + ) -> dict: + """Execute code or file, exclusively; print results to stdout. No host path is writable. + + Args: + code: Python source, mutually exclusive with file. + timeout: Execution timeout in seconds, greater than zero and at most 600. + file: Existing script path, absolute or relative to the case directory; sandbox read rules apply. + args: String arguments for file mode only. + """ + if (code is not None and not isinstance(code, str)) or (file is not None and not isinstance(file, str)): + raise ValueError("code and file must be strings or null") + has_code, has_file = bool(code and code.strip()), bool(file and file.strip()) + if has_code == has_file: + raise ValueError("Provide exactly one of code or file") + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not 0 < timeout <= 600: + raise ValueError("timeout must be greater than zero and at most 600 seconds") + if args is not None and ( + not has_file or not isinstance(args, list) or any(not isinstance(a, str) for a in args) + ): + raise ValueError("args must be a list of strings in file mode only") + command = [sys.executable, "-I", "-B"] + command += ["-c", code] if has_code else [str(Path(cwd) / file), *(args or [])] + result = await backend.exec_shell(command, cwd=cwd, timeout=timeout) + return { + "returncode": result.exit_code, + "stdout": result.stdout.decode(errors="replace"), + "stderr": result.stderr.decode(errors="replace"), + "timed_out": result.exit_code == -1 and result.stderr == b"timed out", + } + + return FunctionTool(python_execute, is_concurrency_safe=False) + + +def create_toolkit(wrapper, scope, backend): + """Expose only read-only Jobs with hidden, enforced scope and two sandboxed executors.""" + tools = [] + for name in CASE_TOOLS + CODE_TOOLS: + job = wrapper.app_context.jobs[name] + if not job.enabled or not job.is_started: + raise RuntimeError(f"Case study requires a started Job: {name}") + injected = ( + {"revision": scope["candidate_commit"]} + if name in CODE_TOOLS + else {"validation_path": scope["validation_path"], "case_id": scope["case_id"]} + ) + if name == "home_compare_memory": + injected["target_validation_path"] = None + tools.append(wrapper._make_tool(job, injected_job_kwargs=injected)) # pylint: disable=protected-access + tools.extend( + [ + AutoBash(cwd=scope["case_path"], backend=backend, is_read_only=False), + python_tool(backend, scope["case_path"]), + ], + ) + for tool in tools: + tool.is_concurrency_safe = False + return Toolkit(tools=tools) + + +class HomeCaseStudyStep(BaseStep): + """Run one fresh agent through the application's bounded wrapper and model lifecycle.""" + + workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace") + agent_wrapper: BoundedAgentWrapper = Ref(BoundedAgentWrapper, "agent_wrapper") + + async def execute(self): + """Prepare scope, require isolation, then return the case report and framework token usage.""" + if self.context.get("_allowed_paths") is not None: + raise ValueError("Home case study requires unrestricted access to its managed artifacts") + if not self.workspace.is_started or not self.agent_wrapper.is_started: + raise RuntimeError("Start the Home workspace and case-study agent wrapper first") + spec = CaseStudyInput.model_validate( + {key: self.context[key] for key in CaseStudyInput.model_fields if key in self.context}, + ) + scope = await blocking(prepare_scope, self.workspace.home_path, spec) + backend = create_file_sandbox_backend( + cwd=scope["case_path"], + read_only_paths=scope["read_only_paths"], + writable_paths=[], + memory_limit_mib=self.kwargs.get("memory_limit_mib"), + ) + # Fail before spending model tokens if the OS sandbox or its runtime cannot start. + probe = await backend.exec_shell( + [sys.executable, "-I", "-B", "-c", "pass"], + cwd=scope["case_path"], + timeout=10, + ) + if not probe.ok(): + raise RuntimeError("Case-study sandbox unavailable: " + probe.stderr.decode(errors="replace")) + toolkit = create_toolkit(self.agent_wrapper, scope, backend) + output_limit = self.kwargs.get("output_token_limit", 2500) + if isinstance(output_limit, bool) or not isinstance(output_limit, int) or output_limit < 1: + raise ValueError("output_token_limit must be a positive integer") + prompts = yaml.safe_load(PROMPT.read_text(encoding="utf-8")) + if not isinstance(prompts, dict) or any( + not isinstance(prompts.get(key), str) or not prompts[key].strip() + for key in ("system_prompt", "task_prompt") + ): + raise ValueError("Case-study YAML requires nonempty system_prompt and task_prompt strings") + prompt = prompts["system_prompt"].format(**scope, output_token_limit=output_limit) + result = await self.agent_wrapper.reply( + prompts["task_prompt"].format(focus=spec.focus), + system_prompt=prompt, + toolkit=toolkit, + builtin_tools=[], + job_tools=[], + skills=[], + resume=None, + session_id=None, + fork_session=False, + output_schema=None, + permission_mode="bypass", + ) + self.context.response.answer = result["result"] + self.context.response.metadata.update( + {key: scope[key] for key in ("validation_path", "case_id", "candidate_commit")}, + ) + self.context.response.metadata.update({key: result[key] for key in ("session_id", "usage") if key in result}) + return self.context.response diff --git a/plugins/home/src/reme_home/agent/case_study.yaml b/plugins/home/src/reme_home/agent/case_study.yaml new file mode 100644 index 00000000..0750393f --- /dev/null +++ b/plugins/home/src/reme_home/agent/case_study.yaml @@ -0,0 +1,42 @@ +system_prompt: |- + Investigate exactly one validation case. Trace observed failures backward from judging and answer generation through + retrieval/ranking, final memory, construction checkpoints, original sessions, and the responsible implementation. + At each layer explain what should have happened, what happened, and the supporting or contradicting evidence. + Continue to the earliest evidence-supported mechanism; "not found", "ranked too low" and "wrong answer" are symptoms. + Compare plausible explanations and state the smallest missing observation when no cause is established. + + Assigned scope: + - Validation: {validation_path} + - Case: {case_id}; authoritative case artifacts: {case_path} + - Normalized source case: {source_case} + - Candidate repository: {code_repository}; evaluated commit: {candidate_commit} + - Read architecture-guide.md and validation-artifact-guide.md in {docs} when needed. + + Investigate no other case. Do not modify code or evidence, run validation, execute ReMe, or prescribe fixes. + Treat construction partial_success as requiring repair even when answers succeed. Distinguish infrastructure, + candidate execution, answer-quality and judging failures without replacing their saved runtime statuses. + + Use home_inspect_validation to select anomalous queries and locate result fields and logs. Start query evidence + tracing with home_trace_query_evidence. Respect evidence_source and evidence_is_certain: inferred or uncertain + session annotations are leads, not established facts. Original transcripts and derived memory may both be searchable; + absence from a summary does not prove that the answerer could not retrieve the source session. + + Use home_inspect_memory_history and home_compare_memory for construction checkpoints and focused diffs. + Use home_list_code_tree, home_search_code and home_read_code_file to examine the evaluated commit; their revision + and case-tool scopes are fixed by the host. Follow returned paths for full source details and omitted evidence. + + AutoBash and python_execute share an OS file sandbox. The case artifacts, source case, validation manifest, + home/docs and all of home/code are readable; no host paths are writable. System tools and interpreter libraries + are also available. No scratch directory or Write/Edit tool is provided. Commands and Python analysis are allowed + without a command whitelist, but network access and writes outside sandbox-internal temporary storage are denied. + Print analysis results instead of saving intermediate files. Python accepts source code or an existing script file. + The directly readable code worktree can differ from the evaluated commit; establish commit alignment before + attributing a failure to code. Prefer the pinned code tools when checking historical behavior. + + Return a concise report, conclusion and strongest evidence first, within approximately {output_token_limit} tokens. + Name affected queries and lifecycle stages, explain the causal chain, include contrary evidence and confidence, + and separate established causes, hypotheses and unknowns. Cite every error and causal claim with an exact + path:line location or path#key.path for structured JSON. Keep citations tight. Make no unsupported cross-case claims. +task_prompt: |- + Pay special attention to: + {focus} diff --git a/plugins/home/src/reme_home/agent/wrapper.py b/plugins/home/src/reme_home/agent/wrapper.py index 60425c3c..7586fe80 100644 --- a/plugins/home/src/reme_home/agent/wrapper.py +++ b/plugins/home/src/reme_home/agent/wrapper.py @@ -3,7 +3,9 @@ from typing import Any from reme.components.agent_wrapper import AsAgentWrapper +from reme.components.as_llm import BaseAsLLM +from ..judge import HomeJudge from .bounded import BoundedReActAgent, RunTimeBudgetTimer @@ -19,3 +21,22 @@ class BoundedAgentWrapper(AsAgentWrapper): budget = options.get("time_budget_minutes") timer = None if budget is None else RunTimeBudgetTimer(budget, agent_label=self.name) return BoundedReActAgent(run_time_budget_timer=timer, **kwargs) + + +class CaseStudyAgentWrapper(BoundedAgentWrapper): + """Validate Home's host configuration before requiring the investigation model. + + A configured model and the host judge start before this wrapper. Optional + model binding preserves Home's missing-judge diagnostic; startup still + rejects an absent investigation model after the judge is validated. + """ + + def __init__(self, as_llm: str = "judge", **kwargs): + super().__init__(as_llm="", **kwargs) + self.home_judge = self.bind("default", HomeJudge, optional=False) + self.as_llm = self.bind(as_llm, BaseAsLLM, optional=True) + + async def _start(self): + await super()._start() + if self.as_llm is None: + raise ValueError("Case-study agent requires a configured as_llm component") diff --git a/plugins/home/src/reme_home/evaluation/artifacts.py b/plugins/home/src/reme_home/evaluation/artifacts.py index 9f504817..28214613 100644 --- a/plugins/home/src/reme_home/evaluation/artifacts.py +++ b/plugins/home/src/reme_home/evaluation/artifacts.py @@ -12,6 +12,7 @@ from uuid import uuid4 from ..preparation.data.models import CaseSpec, QuerySpec from ..workspace import _validate_directory +from .compaction import round_scores def contained(root: Path, value: str | Path, *, directory: bool = False) -> Path: @@ -216,7 +217,7 @@ def summarize(run: Path, cases: list, *, scoring_required: bool = True) -> dict: construction_errors = sum(not result.get("success") for result in job_results) stages["construction"] += construction_status == "failure" or bool(construction_errors) stages["query"] += bool(failed) - compact_queries = {} + compact_queries, query_errors = {}, {} for query in queries: query_status = ( "completed" @@ -227,8 +228,14 @@ def summarize(run: Path, cases: list, *, scoring_required: bool = True) -> dict: compact_queries[query["query_id"]] = { "question_type": query.get("question_type"), "score": query.get("score"), - "status": query_status, } + if query_status != "completed": + error = str(query["error"]) + query_errors[query["query_id"]] = { + "failure_kind": query_status, + "stage": query.get("failure_stage"), + "error": error if len(error) <= 500 else error[:500] + "…", + } rows[case.case_id] = { "construction": { "status": construction_status, @@ -245,6 +252,7 @@ def summarize(run: Path, cases: list, *, scoring_required: bool = True) -> dict: rows[case.case_id]["construction_error_count"] = construction_errors if failed: rows[case.case_id]["query_error_count"] = failed + rows[case.case_id]["query_errors"] = query_errors groups = {} for query in query_results: groups.setdefault(str(query.get("question_type") or "unknown"), []).append(query) @@ -253,36 +261,39 @@ def summarize(run: Path, cases: list, *, scoring_required: bool = True) -> dict: scores = [query["score"] for query in queries if query.get("score") is not None] statistics[name] = { "query_count": len(queries), - "average_score": round(sum(scores) / len(scores), 3) if scores else None, + "average_score": sum(scores) / len(scores) if scores else None, "average_query_total_token_usage": average_tokens(queries), } - return { - "case_count": len(rows), - "case_status_counts": case_counts, - "construction_status_counts": construction_counts, - "failure_stage_case_counts": stages, - "expected_query_count": expected, - "scored_query_count": len(all_scores), - "query_status_counts": { - **query_counts, - "unscored": max(expected - len(all_scores), 0) if scoring_required else 0, + return round_scores( + { + "case_count": len(rows), + "scoring_enabled": scoring_required, + "case_status_counts": case_counts, + "construction_status_counts": construction_counts, + "failure_stage_case_counts": stages, + "expected_query_count": expected, + "scored_query_count": len(all_scores), + "query_status_counts": { + **query_counts, + "unscored": max(expected - len(all_scores), 0) if scoring_required else 0, + }, + "mean_query_score": ( + sum(all_scores) / expected if scoring_required and expected and not case_counts["infra_error"] else None + ), + "observed_mean_query_score": sum(all_scores) / len(all_scores) if all_scores else None, + "low_score_cases": [ + {key: row["mean_score"]} + for key, row in sorted( + rows.items(), + key=lambda item: (item[1]["mean_score"] if item[1]["mean_score"] is not None else 2, item[0]), + ) + if row["mean_score"] is not None + and any(query.get("score") is not None and query["score"] < 1 for query in row["queries"].values()) + ], + "question_type_statistics": statistics, + "cases": rows, }, - "mean_query_score": ( - sum(all_scores) / expected if scoring_required and expected and not case_counts["infra_error"] else None - ), - "observed_mean_query_score": sum(all_scores) / len(all_scores) if all_scores else None, - "low_score_cases": [ - {key: row["mean_score"]} - for key, row in sorted( - rows.items(), - key=lambda item: (item[1]["mean_score"] if item[1]["mean_score"] is not None else 2, item[0]), - ) - if row["mean_score"] is not None - and any(query.get("score") is not None and query["score"] < 1 for query in row["queries"].values()) - ], - "question_type_statistics": statistics, - "cases": rows, - } + ) def rebuild_summary(run: Path) -> dict: diff --git a/plugins/home/src/reme_home/evaluation/compaction.py b/plugins/home/src/reme_home/evaluation/compaction.py index 7db9161b..cdedb3be 100644 --- a/plugins/home/src/reme_home/evaluation/compaction.py +++ b/plugins/home/src/reme_home/evaluation/compaction.py @@ -10,10 +10,34 @@ from pathlib import PurePosixPath from typing import Any +def round_scores(value: Any, *, score: bool = False) -> Any: + """Copy structured scores with four decimal places, preserving text and other numbers.""" + if isinstance(value, dict): + return { + key: round_scores( + child, + score=score + or isinstance(key, str) + and ( + key in {"score", "precision", "recall", "f1", "tau_norm", "low_score_cases"} + or key.endswith("_score") + ), + ) + for key, child in value.items() + } + if isinstance(value, list): + return [round_scores(child, score=score) for child in value] + return round(value, 4) if score and isinstance(value, float) else value + + def compact_query(result: dict) -> dict: """Copy and compact one authoritative query result.""" result = deepcopy(result) _compact_query_result(result) + # Reference answers and provider raw text remain source evidence. + for key in ("score", "judge_result", "answer_result"): + if key in result: + result[key] = round_scores(result[key], score=key == "score") return result diff --git a/plugins/home/src/reme_home/evaluation/component.py b/plugins/home/src/reme_home/evaluation/component.py index 13ecba99..5d136b10 100644 --- a/plugins/home/src/reme_home/evaluation/component.py +++ b/plugins/home/src/reme_home/evaluation/component.py @@ -113,12 +113,9 @@ class HomeEvaluation(BaseComponent): }, ), ) - judge_identity = self.judge.identity(dataset.dataset) snapshot, commit, branch = snapshot_code(home, record) fingerprints = { "dataset": dataset.normalized_fingerprint, - "code": snapshot.sha256, - "trusted_judge": judge_identity["fingerprint"], "construction_config": fingerprint( { "runtime": record["code"]["runtime_fingerprint"], @@ -189,7 +186,6 @@ class HomeEvaluation(BaseComponent): run = None try: manifest = { - "schema_version": 1, "branch_name": branch, "commit_sha": commit, "case_ids": [case.case_id for case in cases], @@ -198,13 +194,17 @@ class HomeEvaluation(BaseComponent): "dataset_kind": spec.dataset_kind, "mode": "debug" if judge is None else "scored", "execution_mode": "query_only_replay" if source else "full_validation", - "construction_source": source, "fingerprints": fingerprints, - "options": spec.model_dump(mode="json", exclude={"case_ids", "query_ids", "reuse_construction_from"}), + "options": spec.model_dump( + mode="json", + exclude={"case_ids", "query_ids", "reuse_construction_from", "debug", "sampling_policy"}, + ), "judge": self.judge.identity(dataset.dataset), "status": "running", "started_at": utc_now().isoformat(), } + if source is not None: + manifest["construction_source"] = source run = self._allocate(branch, commit, manifest) plans = [] for case in cases: diff --git a/plugins/home/src/reme_home/evaluation/evidence_step.py b/plugins/home/src/reme_home/evaluation/evidence_step.py new file mode 100644 index 00000000..54c77fd7 --- /dev/null +++ b/plugins/home/src/reme_home/evaluation/evidence_step.py @@ -0,0 +1,142 @@ +"""Compact, read-only query evidence coordinates from saved evaluation artifacts.""" + +from typing import Annotated + +from pydantic import Field, TypeAdapter, field_validator + +from reme.steps.base_step import BaseStep, Ref + +from ..preparation.spec import SpecModel +from ..workspace import HomeWorkspace +from .artifacts import contained, read_json +from .inputs import entity_id +from .inspection_step import _execute, _object, _preview +from .memory_inspection_step import _head, _repository, git + +SessionIds = list[Annotated[str, Field(min_length=1)]] +MAX_SESSIONS = 50 +MAX_CHANGED_FILES = 50 + + +class TraceQueryEvidenceInput(SpecModel): + """Locate one selected query; inferred evidence is optional and never overrides annotations.""" + + validation_path: str = Field(min_length=1) + case_id: str = Field(min_length=1) + query_id: str = Field(min_length=1) + evidence_session_ids: SessionIds | None = None + + @field_validator("case_id", "query_id") + @classmethod + def validate_entity(cls, value): + """Keep artifact selectors within one directory segment.""" + return entity_id(value) + + +def _source_case(home, manifest, case_id): + kind = manifest.get("dataset_kind") + if kind not in {"training-set", "debug-cases"}: + raise ValueError(f"Unsupported evaluation dataset_kind: {kind!r}") + root = contained(home, f"datasets/{kind}/cases", directory=True) + for candidate in sorted(root.glob("*.json")): + path = contained(root, candidate) + case = read_json(path) + if isinstance(case, dict) and case.get("case_id") == case_id: + return path, case + raise ValueError(f"Normalized case is missing: {case_id}") + + +def _session_checkpoints(repository, head, selected): + if head is None or not selected: + return {} + # Separate commit subjects and records without relying on line delimiters. + raw = git(repository, "log", "--no-show-signature", "--reverse", "--format=%H%x00%s", "-z", head, "--") + fields = raw.split("\0") + checkpoints = {} + for sha, subject in zip(fields[::2], fields[1::2]): + if subject.startswith("session: ") and subject[9:] in selected: + checkpoints[subject[9:]] = sha + return checkpoints + + +def trace_query_evidence(home, spec): + """Join source annotations, committed checkpoint coordinates and query result previews.""" + run, manifest, repository = _repository(home, spec.validation_path, spec.case_id) + if spec.query_id not in manifest.get("query_ids", {}).get(spec.case_id, []): + raise ValueError("query_id is not selected by this evaluation manifest") + source_path, case = _source_case(home, manifest, spec.case_id) + query = next((item for item in case.get("queries", []) if item.get("query_id") == spec.query_id), None) + if query is None: + raise ValueError(f"Normalized case has no query: {spec.query_id}") + metadata = query.get("metadata") or {} + annotated = metadata.get("evidence_session_ids") + selected = TypeAdapter(SessionIds).validate_python( + annotated if annotated is not None else spec.evidence_session_ids or [], + strict=True, + ) + selected = list(dict.fromkeys(selected)) + source = "independent_annotation" if annotated is not None else "diagnostician_inference" + if annotated is None and spec.evidence_session_ids is None: + source = "unavailable" + known = {session["session_id"] for session in case.get("sessions", [])} + if unknown := set(selected) - known: + raise ValueError(f"Evidence sessions are not present in case: {sorted(unknown)}") + + status, head = _head(repository) + checkpoints = _session_checkpoints(repository, head, set(selected)) + sessions = [] + for session_id in selected[:MAX_SESSIONS]: + checkpoint = checkpoints.get(session_id) + files = ( + git(repository, "diff-tree", "--root", "--no-commit-id", "--name-only", "-r", "-z", checkpoint, "--") + .rstrip("\0") + .split("\0") + if checkpoint + else [] + ) + files = [name for name in files if name] + row = {"id": session_id, "checkpoint": checkpoint, "changed_files": files[:MAX_CHANGED_FILES]} + if len(files) > MAX_CHANGED_FILES: + row["omitted_changed_file_count"] = len(files) - MAX_CHANGED_FILES + sessions.append(row) + + results_path = contained(run, f"cases/{spec.case_id}/queries/result.json") + payload = _object(run, str(results_path.relative_to(run))) + result = next( + (item for item in (payload or {}).get("queries", []) if item.get("query_id") == spec.query_id), + None, + ) + outcome = result or {} + response = { + "query": { + "question": _preview(outcome.get("question", query.get("question")), 2000), + "expected": _preview(outcome.get("golden_answer", query.get("golden_answer")), 2000), + "actual": _preview(outcome.get("answer"), 2000), + "score": outcome.get("score"), + "error": _preview(outcome.get("error")), + "result_missing": result is None, + }, + "evidence_source": source, + "evidence_is_certain": metadata.get("evidence_is_certain"), + "memory_status": status, + "sessions": sessions, + "paths": { + "source_case": str(source_path), + "memory_repository": str(repository), + "query_result": str(results_path), + "query_artifacts": str(contained(run, f"cases/{spec.case_id}/queries/{spec.query_id}", directory=True)), + }, + } + if len(selected) > MAX_SESSIONS: + response["omitted_session_count"] = len(selected) - MAX_SESSIONS + return response + + +class HomeTraceQueryEvidenceStep(BaseStep): + """Expose compact evidence navigation through the normal Home Job lifecycle.""" + + workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace") + + async def execute(self): + """Read saved artifacts without model calls or workspace mutations.""" + return await _execute(self, TraceQueryEvidenceInput, trace_query_evidence) diff --git a/plugins/home/src/reme_home/evaluation/history.py b/plugins/home/src/reme_home/evaluation/history.py index 85e42e6d..12b16703 100644 --- a/plugins/home/src/reme_home/evaluation/history.py +++ b/plugins/home/src/reme_home/evaluation/history.py @@ -1,11 +1,9 @@ """Successful evaluation history and rebuildable sampling evidence owned by Home.""" from pathlib import Path -from typing import Mapping, Literal +from typing import Mapping import math -from pydantic import Field - from reme.components.base_component import BaseComponent from ..preparation.data.models import utc_now @@ -33,13 +31,6 @@ class EvaluationRecord(SpecModel): mean_query_score: float | None -class HistoryIndex(SpecModel): - """A disposable index; manifests, summaries and plans remain authoritative.""" - - schema_version: Literal[1] = 1 - results: list[EvaluationRecord] = Field(default_factory=list) - - def successful_result(home: Path, run: Path) -> tuple[dict, dict] | None: """Match the non-debug Response.success contract, including valid low scores.""" manifest = read_json(contained(home, run / "manifest.json")) @@ -129,7 +120,6 @@ def start_validation(home: Path, run: Path, policy: SamplingPolicy, plan: dict | dict(plan) if plan is not None else { - "schema_version": 1, "mode": "explicit", "case_ids": manifest["case_ids"], "query_ids": manifest["query_ids"], @@ -278,17 +268,15 @@ class HomeEvaluationHistory(BaseComponent): raise ValueError("Expected one evaluation result directory") return run - def _index(self, *, rebuild: bool = False) -> HistoryIndex: + def _records(self) -> list[EvaluationRecord]: + """Read source artifacts directly; legacy history.json files are ignored.""" home = self.workspace.home_path - path = contained(home, "evaluations/history.json") - if path.exists() and not rebuild: - return HistoryIndex.model_validate(read_json(path)) records = [] for manifest in contained(home, "evaluations", directory=True).glob("*/*/*/manifest.json"): record = result_record(home, contained(home, manifest.parent, directory=True)) if record is not None: records.append(record) - return HistoryIndex(results=sorted(records, key=lambda record: (record.round, record.run_key))) + return sorted(records, key=lambda record: (record.round, record.run_key)) def sampling_state(self) -> dict: """Return owned, reconstructed-on-demand observations for the next draw.""" @@ -309,18 +297,13 @@ class HomeEvaluationHistory(BaseComponent): record = result_record(home, run) if record is None: return None - index = self._index() - by_key = {item.run_key: item for item in index.results} - by_key[record.run_key] = record update_sampling_state(home, run) - index = HistoryIndex(results=sorted(by_key.values(), key=lambda item: (item.round, item.run_key))) - write_json(contained(home, "evaluations/history.json"), index.model_dump(mode="json")) return record def list_results(self) -> list[EvaluationRecord]: """Return successful results in chronological order, including low-scoring candidates.""" with self._history_lock(): - return self._index().results + return self._records() def get_result(self, run_key: str) -> EvaluationRecord: """Look up a successful result by its stable workspace-relative identity.""" @@ -333,13 +316,10 @@ class HomeEvaluationHistory(BaseComponent): """Rebuild only derived indexes and sampling observations; never rewrite source evidence.""" with self._history_lock(): home = self.workspace.home_path - index = self._index(rebuild=True) rebuild_sampling_state(home) - write_json(contained(home, "evaluations/history.json"), index.model_dump(mode="json")) - return index.results + return self._records() def invalidate(self) -> None: """Discard only derived caches after an unsuccessful registration; retain all source evidence.""" with self._history_lock(): - for name in ("evaluations/history.json", "sampling/state.json"): - contained(self.workspace.home_path, name).unlink(missing_ok=True) + contained(self.workspace.home_path, "sampling/state.json").unlink(missing_ok=True) diff --git a/plugins/home/src/reme_home/evaluation/inputs.py b/plugins/home/src/reme_home/evaluation/inputs.py index 41240474..fbeea7fc 100644 --- a/plugins/home/src/reme_home/evaluation/inputs.py +++ b/plugins/home/src/reme_home/evaluation/inputs.py @@ -104,7 +104,7 @@ def load_inputs(home: Path, spec: EvaluationSpec): raise ValueError("Prepared dataset no longer matches its manifest") if manifest.case_count != len(cases) or manifest.query_count != sum(len(c.queries) for c in cases.values()): raise ValueError("Prepared dataset counts do not match its manifest") - if spec.dataset_kind == "training-set" and manifest.model_dump(mode="json") != record["dataset"]: + if spec.dataset_kind == "training-set" and manifest != DatasetManifest.model_validate(record["dataset"]): raise ValueError("Prepared training manifest has changed") selected = spec.case_ids if spec.case_ids is not None else list(cases) if not selected or set(selected) - cases.keys(): diff --git a/plugins/home/src/reme_home/plugin.yaml b/plugins/home/src/reme_home/plugin.yaml index b5e2c2be..3d0dff8f 100644 --- a/plugins/home/src/reme_home/plugin.yaml +++ b/plugins/home/src/reme_home/plugin.yaml @@ -1,4 +1,7 @@ backends: + home_case_study_step: reme_home.agent.case_study:HomeCaseStudyStep + home_case_study_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper + home_trace_query_evidence_step: reme_home.evaluation.evidence_step:HomeTraceQueryEvidenceStep home_memory_inspection_step: reme_home.evaluation.memory_inspection_step:HomeMemoryInspectionStep home_code: reme_home.code.component:HomeCode home_code_step: reme_home.code.step:HomeCodeStep @@ -16,6 +19,13 @@ backends: application_defaults: components: + agent_wrapper: + home_case_study: + backend: home_case_study_agent + as_llm: judge + time_budget_minutes: 30 + react_config: + max_iters: 60 home.code: default: backend: home_code @@ -41,6 +51,38 @@ application_defaults: default: backend: home_workspace jobs: + home_case_study: + backend: base + description: "Investigate one validation case with read-only sandboxed Bash/Python and commit-pinned evidence tools; return an evidence-supported diagnostic report" + parameters: + type: object + additionalProperties: false + required: [validation_path, case_id] + properties: + validation_path: {type: string, minLength: 1} + case_id: {type: string, minLength: 1} + focus: {type: string, minLength: 1, description: "Questions or failure mechanisms to prioritize"} + steps: + - backend: home_case_study_step + agent_wrapper: home_case_study + output_token_limit: 2500 + home_trace_query_evidence: + backend: base + description: "Locate a selected query's source evidence sessions, memory checkpoints and answer/score previews; use home_compare_memory and returned paths for details" + parameters: + type: object + additionalProperties: false + required: [validation_path, case_id, query_id] + properties: + validation_path: {type: string, minLength: 1} + case_id: {type: string, minLength: 1} + query_id: {type: string, minLength: 1} + evidence_session_ids: + type: [array, "null"] + items: {type: string, minLength: 1} + description: "Optional inferred session IDs, used only when the query has no dataset annotation" + steps: + - backend: home_trace_query_evidence_step home_inspect_memory_history: backend: base description: "List a case's exported memory Git checkpoints, newest first; use home_inspect_validation to locate the case" diff --git a/plugins/home/src/reme_home/preparation/component.py b/plugins/home/src/reme_home/preparation/component.py index 46166222..d0cda079 100644 --- a/plugins/home/src/reme_home/preparation/component.py +++ b/plugins/home/src/reme_home/preparation/component.py @@ -173,7 +173,6 @@ class HomePreparation(BaseComponent): for name in ("architecture-guide.md", "validation-artifact-guide.md"): (docs / name).write_bytes(resources.joinpath(name).read_bytes()) result = { - "schema_version": 1, "preparation_id": uuid4().hex, "created_at": datetime.now(timezone.utc).isoformat(), "config_fingerprint": fingerprint(domain), diff --git a/plugins/home/src/reme_home/preparation/data/basic.py b/plugins/home/src/reme_home/preparation/data/basic.py index 06cd9f48..6832e1e9 100644 --- a/plugins/home/src/reme_home/preparation/data/basic.py +++ b/plugins/home/src/reme_home/preparation/data/basic.py @@ -83,7 +83,6 @@ def derive_debug_cases( case_count = _debug_case_count(len(source_paths)) case_indices = _evenly_spaced_indices(len(source_paths), case_count) debug_cases: list[CaseSpec] = [] - derivations: list[dict[str, object]] = [] for source_case_index in case_indices: case = CaseSpec.model_validate_json(source_paths[source_case_index].read_text(encoding="utf-8")) eligible_indices = _eligible_session_indices(case) @@ -111,37 +110,7 @@ def derive_debug_cases( }, ), ) - derivations.append( - { - "debug_case_id": debug_id, - "source_case_id": case.case_id, - "source_case_index": source_case_index, - "session_indices": session_indices, - "messages_per_session": messages_per_session, - }, - ) - manifest = write_normalized_dataset(output, str(training_manifest["dataset"]), "debug", debug_cases) - (output / "derivation.json").write_bytes( - canonical_json_bytes( - { - "schema_version": 1, - "training_fingerprint": training_manifest["normalized_fingerprint"], - "rules": { - "case_selection_ratio": 0.1, - "minimum_case_count": 2, - "derived_case_count": case_count, - "sessions_per_case": sessions_per_case, - "messages_per_session": messages_per_session, - "case_selection": "evenly_spaced_in_normalized_order", - "session_selection": "evenly_spaced", - "message_selection": "prefix", - }, - "cases": derivations, - }, - ) - + b"\n", - ) - return manifest + return write_normalized_dataset(output, str(training_manifest["dataset"]), "debug", debug_cases) def _debug_case_count(training_case_count: int) -> int: diff --git a/plugins/home/src/reme_home/preparation/data/models.py b/plugins/home/src/reme_home/preparation/data/models.py index 74a9d15b..5ac76094 100644 --- a/plugins/home/src/reme_home/preparation/data/models.py +++ b/plugins/home/src/reme_home/preparation/data/models.py @@ -63,7 +63,8 @@ class CaseSpec(ContractModel): class DatasetManifest(ContractModel): """Selection identity, converter version and normalized content fingerprints.""" - schema_version: int = Field(default=SCHEMA_VERSION, ge=1) + # Accept legacy files without emitting a version that has no reader dispatch. + schema_version: int = Field(default=SCHEMA_VERSION, ge=1, exclude=True) dataset: DatasetName converter_version: int = 1 variant: str | None = None diff --git a/plugins/home/src/reme_home/preparation/spec.py b/plugins/home/src/reme_home/preparation/spec.py index 394b4e23..56a5ba4c 100644 --- a/plugins/home/src/reme_home/preparation/spec.py +++ b/plugins/home/src/reme_home/preparation/spec.py @@ -109,7 +109,7 @@ class DebugSpec(SpecModel): class PreparationSpec(SpecModel): """Public Home preparation configuration, accepted as YAML or a dict.""" - schema_version: Literal[1] = 1 + schema_version: Literal[1] = Field(default=1, exclude=True) benchmark: BenchmarkSpec runtime: RuntimeSpec candidate: CandidateSpec diff --git a/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md b/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md index 32749827..ff6ae516 100644 --- a/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md +++ b/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md @@ -44,6 +44,16 @@ by a sibling session ID. Recognized cumulative daily-index `notes` snapshots bec records become `0`. Detailed construction costs stay in this case artifact. Small scalar containers are written on one line, following `dev_meta` formatting. +Structured scores in query results and summaries use at most four decimal places. +Reference answers, raw judge strings and non-score metrics retain their original values. + +New run manifests omit the unused schema version, duplicate code and judge fingerprints, +`options.debug` (use `dataset_kind`/`mode`), and duplicate sampling policy (use the plan). +The remaining `fingerprints.dataset` and `fingerprints.construction_config` guard memory +reuse compatibility; `judge.fingerprint` identifies the trusted scoring implementation. +Legacy manifests remain readable. Workspace markers, preparation recovery journals and +sampling state retain versions used for validation, rollback and migration. + ## Summary fields The run summary contains only: @@ -51,24 +61,29 @@ The run summary contains only: - `case_count`, `case_status_counts`, `construction_status_counts`, and `failure_stage_case_counts` (construction/query; tolerated construction failures still count as construction errors). -- `expected_query_count`, `scored_query_count`, and `query_status_counts` +- `scoring_enabled`, `expected_query_count`, `scored_query_count`, and `query_status_counts` (completed, candidate failure, infrastructure error, unscored). - `mean_query_score`, `observed_mean_query_score`, `low_score_cases`, `question_type_statistics`, and `cases`. Each `cases[case_id]` contains construction `status`, `success_job_count` and `fail_job_count`; `mean_score`; `average_query_total_token_usage`; and a `queries` -map. Each query entry contains only `question_type`, `score` and `status`. +map. Each query entry contains only `question_type` and `score`. Failed queries +also appear in `query_errors`, keyed by query ID, with `failure_kind` +(`candidate_failure` or `infra_error`), `stage` (null if unavailable), and an +error excerpt limited to 500 characters plus a truncation marker. Full errors +remain in the phase artifacts. A null score alone does not identify a failure: +successful queries also remain unscored when `scoring_enabled` is false. `stage`, `construction_error_count` and `query_error_count` appear only when applicable. There is no redundant case status or per-case coverage accounting. -Questions, answers, errors, Job details and construction token usage stay in +Questions, answers, full errors, Job details and construction token usage stay in phase artifacts, outside the summary. Token summaries contain only average candidate answer **total** tokens per model, divided by all returned queries, including failures and queries without reported usage. Zero and unknown token totals are omitted. Judge/construction costs and input/output token breakdowns are not aggregated. Question-type statistics contain -query count, average of available scores (rounded to three decimals), and the same +query count, average of available scores (rounded to four decimals), and the same answer token averages across cases. Missing/empty types form `unknown`. Candidate failures count as zero in the expected-query score denominator. @@ -126,7 +141,7 @@ not reuse donor answers or scores. `home_evaluate sampling=true` enables adaptive selection; false preserves full or explicit evaluation. Debug always bypasses history. `sampling_policy` is the host policy input, distinct from the boolean toggle. The effective policy is -captured in `manifest.options` and the started plan. +captured in the started plan, or `manifest.selection_plan` for untracked adaptive runs. `record_history=true` (the default) includes a successful full baseline even when `sampling=false`. `record_history=false` runs the same scoring and artifact pipeline @@ -137,8 +152,9 @@ skips history. The response reports effective participation as `record_history`. An untracked adaptive run may read existing observations and saves its draw locally in `manifest.selection_plan`, without a shared started plan or ledger round. -`home.evaluation_history` owns the successful-result catalog at -`home/evaluations/history.json`. Entries require completed scored training runs +`home.evaluation_history` discovers successful results directly from run manifests +and summaries. No `history.json` index is generated or read; legacy copies are ignored. +Entries require completed scored training runs with `success=true` (or equivalent legacy summary counts) and no infrastructure errors. Low scores remain useful. Failed/cancelled/debug/partial runs are excluded; a mixed run with an infrastructure failure contributes no quality observations. @@ -153,8 +169,8 @@ advances staleness/recheck timing and trains EWMA. Adaptive plans preserve their probability clock in `selection_round`. Actual numeric scores alone count as quality observations; missing/unjudged answers never become zero scores. -Missing caches and `HomeEvaluationHistory.rebuild()` reconstruct the catalog and -sampling observations from saved evidence. Rebuilding never modifies datasets, +Missing sampling caches and `HomeEvaluationHistory.rebuild()` reconstruct +sampling observations from saved evidence. Result listings always scan the source artifacts. Rebuilding never modifies datasets, candidate code or memory files. See the Home README for configuration, host per-call policy overrides and enforced agent injection examples. @@ -179,6 +195,24 @@ not rebuild summaries or modify evaluation history, sampling state or artifacts. ## Memory checkpoint tools +Start a failed-query investigation with +`home_trace_query_evidence(validation_path, case_id, query_id)`. +It returns compact question/answer/score/error previews, source evidence session +IDs, matching Git checkpoints, changed files and paths to complete evidence. It +does not call a model or include transcripts, diffs or judge-result bodies. +Dataset `evidence_session_ids` annotations take precedence; only when absent may +you supply inferred `evidence_session_ids`. Respect `evidence_source` and +`evidence_is_certain`: inferred or uncertain sessions are leads, not established +causes. No annotation or supplied IDs yields `evidence_source: unavailable`. +Missing query results are explicit and scores remain null. `memory_status` +distinguishes unavailable histories; a null session checkpoint means no matching +commit was found. Question/answer previews have a 2,000-character limit, errors +600, and session/changed-file lists 50 entries with omitted counts. Follow +`paths.source_case`, `paths.query_result` and `paths.query_artifacts` for details; +use the tools below for history and focused diffs. Both original transcripts and +derived memory may be searchable, so absence from a summary alone proves no +retrieval failure. + After locating one case, call `home_inspect_memory_history` with its `validation_path` and `case_id` to page through exported memory checkpoints. Optional `path` follows a literal memory-workspace-relative file or directory. @@ -201,3 +235,20 @@ printing bytes. Narrow `path` when the 2,000-file, 20-MB snapshot, 5-MB file or These tools neither reconstruct missing exports nor modify Git/evaluation/history artifacts. Raw logs and source JSON remain available through the existing result inspection paths. + +## Single-case diagnostic agent + +Call `home_case_study(validation_path, case_id, focus=...)` to investigate exactly +one case. It returns a text report with evidence citations and uncertainty, using +fresh agent state, bounded iterations and advisory time reminders. There is no +parallel case dispatcher. Scope and evaluated code revision are host-injected +into the structured tools. AutoBash and Python share a mandatory read-only file +sandbox covering the selected case artifacts, source JSON, validation manifest, +home/docs and all home/code, plus required system runtimes. Neither executor has +host-writable roots or a case scratch directory. They allow analysis commands +and Python rather than a static command whitelist. The directly readable code +worktree can differ from the evaluated commit; use pinned code tools to verify +version-sensitive claims. No write/edit, validation or code-mutation tools are +exposed. Sandbox startup failure prevents model execution; it never falls back +to an unrestricted process. See the Home README for model, budget and platform +sandbox configuration. diff --git a/plugins/home/src/reme_home/sampling.py b/plugins/home/src/reme_home/sampling.py index 0f329578..c9fdfd5c 100644 --- a/plugins/home/src/reme_home/sampling.py +++ b/plugins/home/src/reme_home/sampling.py @@ -87,7 +87,6 @@ def create_sampling_plan( if not selected_cases: raise ValueError("sampling budget selected no executable case/query pairs") return { - "schema_version": 1, "mode": "sampled", "round": round_id, "seed": seed, diff --git a/plugins/home/tests/test_case_study.py b/plugins/home/tests/test_case_study.py new file mode 100644 index 00000000..05d12c4a --- /dev/null +++ b/plugins/home/tests/test_case_study.py @@ -0,0 +1,267 @@ +"""Single case investigations use normal ReMe lifecycle and sandboxed executors.""" + +# pylint: disable=missing-function-docstring,missing-class-docstring,redefined-outer-name +# pylint: disable=unused-argument,protected-access + +import asyncio +import json +import sys +from types import SimpleNamespace + +import pytest +from agentscope.message import TextBlock, ToolCallBlock +from agentscope.model import ChatResponse, ChatUsage +from agentscope.tool import ExecResult +from reme_home.agent import case_study +from reme_home.evaluation.artifacts import write_json +from reme_home.preparation.code import git + +from test_bounded_agent import RecordingModel +from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import + FakeFactory, + home_application, + preparation_inputs, + prepared_home, +) +from test_evaluation_inspection import write_run, snapshot + +from reme.utils.local_backend import LocalProcessBackend +from reme.utils.file_sandbox import create_file_sandbox_backend + + +class RecordingSandbox: + """Mock only the subprocess boundary; never claim this is an OS sandbox.""" + + os_name = "posix" + + def __init__(self, **kwargs): + self.policy = kwargs + self.calls = [] + + async def exec_shell(self, command, *, cwd, timeout): + self.calls.append((command, cwd, timeout)) + return ExecResult(exit_code=0, stdout=b"analysis\n", stderr=b"") + + +@pytest.fixture +def sandbox(monkeypatch): + instances = [] + + def create(**kwargs): + backend = RecordingSandbox(**kwargs) + instances.append(backend) + return backend + + monkeypatch.setattr(case_study, "create_file_sandbox_backend", create) + return instances + + +@pytest.fixture +def case_run(prepared): + commit = git(prepared / "code/repo", "rev-parse", "HEAD") + source = sorted((prepared / "datasets/training-set/cases").glob("*.json"))[0] + case = json.loads(source.read_text()) + case_id = case["case_id"] + query_id = case["queries"][0]["query_id"] + run = write_run(prepared, commit=commit, case_ids=[case_id], query_ids={case_id: [query_id]}) + write_json( + run / f"cases/{case_id}/queries/result.json", + { + "queries": [{"query_id": query_id, "answer": "wrong", "score": 0}], + }, + ) + return run, case_id, query_id, commit, source + + +class InvestigationModel(RecordingModel): + def __init__(self, query_id): + super().__init__() + self.query_id = query_id + + async def __call__(self, **kwargs): + self.calls.append(kwargs) + instructions = [ + ("home_trace_query_evidence", {"query_id": self.query_id}), + ("home_read_code_file", {"path": "plugins/beam/src/reme_beam/__init__.py"}), + ("python_execute", {"code": "print('analysis')"}), + ("AutoBash", {"command": "for x in 1 2; do echo $x; done"}), + ] + index = len(self.calls) - 1 + if index < len(instructions): + name, arguments = instructions[index] + content = [ToolCallBlock(id=f"call-{index}", name=name, input=json.dumps(arguments))] + else: + content = [TextBlock(text="Evidence-supported report; root cause remains uncertain.")] + return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0)) + + +@pytest.mark.asyncio +async def test_single_run_real_agent_tools_scope_and_no_scratch(app, prepared, case_run, sandbox): + run, case_id, query_id, commit, source = case_run + wrapper = app.context.components["agent_wrapper"]["home_case_study"] + model = InvestigationModel(query_id) + wrapper.as_llm = SimpleNamespace(model=model) + (prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py").write_text("value = 999\n") + before = snapshot(prepared) + response = await app.run_job( + "home_case_study", + validation_path=str(run), + case_id=case_id, + focus="Trace the wrong answer", + ) + assert response.success, response.answer + assert "Evidence-supported report" in response.answer + assert response.metadata["candidate_commit"] == commit and response.metadata["usage"]["total_tokens"] == 25 + assert response.metadata["session_id"] + assert "value = 1" in str(model.calls[2]["messages"]) + assert "value = 999" not in str(model.calls[2]["messages"]) + schemas = {item["function"]["name"]: item["function"]["parameters"] for item in model.calls[0]["tools"]} + assert set(schemas) == set(case_study.CASE_TOOLS + case_study.CODE_TOOLS) | {"AutoBash", "python_execute"} + assert "case_id" not in schemas["home_trace_query_evidence"]["properties"] + assert "validation_path" not in schemas["home_inspect_validation"]["properties"] + assert "revision" not in schemas["home_read_code_file"]["properties"] + assert "target_validation_path" not in schemas["home_compare_memory"]["properties"] + policy = sandbox[0].policy + assert policy["writable_paths"] == [] and policy["cwd"] == str(run / f"cases/{case_id}") + assert str(prepared / "code") in policy["read_only_paths"] and str(source) in policy["read_only_paths"] + assert str(prepared) not in policy["read_only_paths"] + assert len(sandbox[0].calls) == 3 # sandbox probe, Python, unrestricted Bash + assert sandbox[0].calls[1][0][:4] == [sys.executable, "-I", "-B", "-c"] + assert "for x in 1 2" in str(sandbox[0].calls[2][0]) + assert snapshot(prepared) == before + # Each invocation is fresh, even through the same application-owned wrapper. + wrapper.as_llm = SimpleNamespace(model=InvestigationModel(query_id)) + second = await app.run_job("home_case_study", validation_path=str(run), case_id=case_id) + assert second.success and second.metadata["session_id"] != response.metadata["session_id"] + + +@pytest.mark.asyncio +async def test_scope_override_rejected_by_real_job_tool(app, prepared, case_run, sandbox): + run, case_id, _query_id, _commit, _source = case_run + wrapper = app.context.components["agent_wrapper"]["home_case_study"] + scope = case_study.prepare_scope(prepared, case_study.CaseStudyInput(validation_path=str(run), case_id=case_id)) + toolkit = case_study.create_toolkit(wrapper, scope, RecordingSandbox()) + schemas = await toolkit.get_tool_schemas() + assert len(schemas) == 9 + tool = wrapper._make_tool( + app.context.jobs["home_inspect_validation"], + injected_job_kwargs={ + "validation_path": str(run), + "case_id": case_id, + }, + ) + with pytest.raises(ValueError, match="cannot be provided"): + await tool._func(case_id="another-case") + + +@pytest.mark.asyncio +async def test_invalid_scope_and_sandbox_failure_precede_llm(app, prepared, case_run, sandbox, monkeypatch): + run, case_id, _query_id, _commit, _source = case_run + wrapper = app.context.components["agent_wrapper"]["home_case_study"] + model = InvestigationModel("unused") + wrapper.as_llm = SimpleNamespace(model=model) + response = await app.run_job("home_case_study", validation_path=str(run), case_id="../escape") + assert not response.success and not sandbox + response = await app.run_job( + "home_case_study", + validation_path=str(run), + case_id=case_id, + _allowed_paths=[str(run)], + ) + assert not response.success and not sandbox + + async def denied(self, *args, **kwargs): + return ExecResult(exit_code=1, stdout=b"", stderr=b"sandbox denied") + + monkeypatch.setattr(RecordingSandbox, "exec_shell", denied) + response = await app.run_job("home_case_study", validation_path=str(run), case_id=case_id) + assert not response.success and "sandbox unavailable" in str(response.answer) + assert not model.calls + + def unavailable(**kwargs): + raise RuntimeError("No sandbox backend") + + monkeypatch.setattr(case_study, "create_file_sandbox_backend", unavailable) + response = await app.run_job("home_case_study", validation_path=str(run), case_id=case_id) + assert not response.success and "No sandbox backend" in str(response.answer) and not model.calls + + +@pytest.mark.asyncio +async def test_iteration_limit_returns_final_report(app, prepared, case_run, sandbox): + run, case_id, _query_id, _commit, _source = case_run + wrapper = app.context.components["agent_wrapper"]["home_case_study"] + assert wrapper.kwargs["react_config"]["max_iters"] == 60 + assert wrapper.kwargs["time_budget_minutes"] == 30 + wrapper.kwargs["react_config"] = {"max_iters": 1} + model = RecordingModel() + wrapper.as_llm = SimpleNamespace(model=model) + response = await app.run_job("home_case_study", validation_path=str(run), case_id=case_id) + assert response.success and response.answer == "final report" + assert model.calls[-1]["tools"] == [] + + +@pytest.mark.asyncio +async def test_os_sandbox_read_scope_and_no_writes(tmp_path): + case = tmp_path / "case" + code = tmp_path / "code" + case.mkdir() + code.mkdir() + (code / "source.py").write_text("visible code") + outside = tmp_path / "secret.txt" + outside.write_text("outside scope") + backend = create_file_sandbox_backend( + cwd=str(case), + read_only_paths=[str(case), str(code)], + writable_paths=[], + memory_limit_mib=0, + ) + try: + probe = await backend.exec_shell([sys.executable, "-I", "-B", "-c", "pass"], cwd=str(case), timeout=10) + except (OSError, RuntimeError) as error: + pytest.skip(f"OS sandbox unavailable on this host: {error}") + if not probe.ok(): + pytest.skip("OS sandbox cannot start on this host: " + probe.stderr.decode(errors="replace")) + tool = case_study.python_tool(backend, str(case)) + result = await tool._func(code=f"from pathlib import Path; print(Path({str(code / 'source.py')!r}).read_text())") + assert result["returncode"] == 0 and "visible code" in result["stdout"] + for script in ( + f"open({str(outside)!r}).read()", + "open('written.txt', 'w').write('denied')", + f"open({str(code / 'source.py')!r}, 'w').write('changed')", + ): + result = await tool._func(code=script) + assert result["returncode"] != 0 + bash = case_study.AutoBash(cwd=str(case), backend=backend, is_read_only=False) + chunks = [chunk async for chunk in bash.call(f"cat '{code / 'source.py'}'")] + assert "visible code" in str(chunks) + chunks = [chunk async for chunk in bash.call("echo changed > written.txt")] + assert "Command failed" in str(chunks) + assert not (case / "written.txt").exists() and (code / "source.py").read_text() == "visible code" + + +@pytest.mark.asyncio +async def test_python_code_file_timeout_and_cancellation(tmp_path): + # Execution protocol uses real subprocesses; OS enforcement is tested separately. + tool = case_study.python_tool(LocalProcessBackend(), str(tmp_path)) + result = await tool._func(code="print(1 + 2)") + assert result["stdout"].strip() == "3" and result["returncode"] == 0 + script = tmp_path / "script.py" + script.write_text("import sys; print(sys.argv[1])") + result = await tool._func(file="script.py", args=["argument"]) + assert result["stdout"].strip() == "argument" + result = await tool._func(code="import time; time.sleep(20)", timeout=0.05) + assert result["timed_out"] + task = asyncio.create_task(tool._func(code="import time; time.sleep(20)")) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + for arguments in ( + {}, + {"code": "pass", "file": "x"}, + {"code": "pass", "args": []}, + {"code": "pass", "timeout": float("nan")}, + {"code": "pass", "timeout": True}, + ): + with pytest.raises(ValueError): + await tool._func(**arguments) diff --git a/plugins/home/tests/test_data_preparation.py b/plugins/home/tests/test_data_preparation.py index 0fb6983a..028a1a4a 100644 --- a/plugins/home/tests/test_data_preparation.py +++ b/plugins/home/tests/test_data_preparation.py @@ -1,6 +1,5 @@ """Debug derivation reads only selected cases and preserves deterministic output.""" -import json from pathlib import Path from reme_home.preparation.data.basic import derive_debug_cases, write_normalized_dataset @@ -47,9 +46,8 @@ def test_debug_derivation_reads_only_selected_cases(tmp_path, monkeypatch): assert reads == expected_paths assert manifest.case_ids == [f"debug-{index}" for index in expected_indices] assert manifest.query_count == 3 - derivation = json.loads((output / "derivation.json").read_text()) - assert [case["source_case_index"] for case in derivation["cases"]] == expected_indices - assert all(case["session_indices"] == [0, 2] for case in derivation["cases"]) + assert not (output / "derivation.json").exists() + assert "schema_version" not in manifest.model_dump() for path in sorted((output / "cases").glob("*.json")): case = CaseSpec.model_validate_json(path.read_text()) assert [session.session_id for session in case.sessions] == ["s0", "s2"] diff --git a/plugins/home/tests/test_evaluation.py b/plugins/home/tests/test_evaluation.py index 0fdea79c..a495044d 100644 --- a/plugins/home/tests/test_evaluation.py +++ b/plugins/home/tests/test_evaluation.py @@ -370,19 +370,24 @@ async def test_mode_controls_data_scoring_and_fail_fast_defaults(app, prepared, manifest = json.loads((run / "manifest.json").read_text()) summary = json.loads((run / "summary.json").read_text()) assert manifest["options"]["fail_fast"] is expected_fail_fast - assert manifest["options"]["debug"] is debug + assert "debug" not in manifest["options"] + assert "schema_version" not in manifest + assert set(manifest["fingerprints"]) == {"dataset", "construction_config"} assert "dataset_kind" not in manifest["options"] assert manifest["dataset_kind"] == ("debug-cases" if debug else "training-set") assert manifest["mode"] == ("debug" if debug else "scored") assert set(summary["cases"]) == ({"debug-1", "debug-2"} if debug else {"1", "2"}) assert summary["expected_query_count"] == 2 assert summary["scored_query_count"] == (0 if debug else 2) + assert summary["scoring_enabled"] is not debug assert manifest["status"] == "completed" assert summary["mean_query_score"] == (None if debug else 0.5) assert summary["observed_mean_query_score"] == (None if debug else 0.5) assert summary["query_status_counts"]["unscored"] == 0 for case in summary["cases"].values(): assert case["mean_score"] == (None if debug else 0.5) + assert "query_errors" not in case + assert all("status" not in query for query in case["queries"].values()) assert app.context.components["agent_wrapper"]["judge"].reply.await_count == (0 if debug else 2) @@ -727,3 +732,22 @@ async def test_late_cleanup_failure_survives_without_duplicate_execution_record( assert failure["stage"] == "cleanup" and failure["failure_kind"] == "infra_error" assert "late cleanup failure" in failure["error"] assert not (folder / "execution.json").exists() + + +@pytest.mark.asyncio +async def test_evaluation_reads_legacy_dataset_versions_without_rewriting_them(app, prepared): + paths = [prepared / "datasets/training-set/manifest.json", prepared / "metadata/preparation.json"] + for path in paths: + payload = json.loads(path.read_text()) + payload["schema_version"] = 1 + if path.name == "preparation.json": + payload["dataset"]["schema_version"] = 1 + path.write_text(json.dumps(payload)) + original = [path.read_bytes() for path in paths] + response = await app.run_job("home_evaluate") + assert response.success, response.answer + assert [path.read_bytes() for path in paths] == original + run = Path(response.answer["result_dir"]) + manifest = json.loads((run / "manifest.json").read_text()) + assert "schema_version" not in manifest and "construction_source" not in manifest + assert set(manifest["fingerprints"]) == {"dataset", "construction_config"} diff --git a/plugins/home/tests/test_evaluation_artifacts.py b/plugins/home/tests/test_evaluation_artifacts.py index 4ea0ea3e..42124637 100644 --- a/plugins/home/tests/test_evaluation_artifacts.py +++ b/plugins/home/tests/test_evaluation_artifacts.py @@ -13,6 +13,77 @@ from reme_home.evaluation.compaction import compact_construction, compact_query from reme_home.preparation.data.models import CaseSpec, QuerySpec +def test_scores_are_rounded_without_changing_reference_text_or_token_metrics(tmp_path): + raw = { + "query_id": "q", + "question": "?", + "golden_answer": {"score": 0.123456789}, + "answer": "0.123456789", + "score": 1 / 3, + "error": None, + "judge_result": { + "success": True, + "metadata": { + "llm_judge_score": 1 / 3, + "event_ordering": {"precision": 1 / 3, "f1": 2 / 3, "count": 2}, + "raw_answer_judgement": "0.333333333", + }, + }, + } + result = compact_query(raw) + assert result["score"] == result["judge_result"]["metadata"]["llm_judge_score"] == 0.3333 + assert raw["score"] == 1 / 3 + assert result["judge_result"]["metadata"]["event_ordering"] == {"precision": 0.3333, "f1": 0.6667, "count": 2} + assert result["golden_answer"] == raw["golden_answer"] + assert result["answer"] == raw["answer"] + assert result["judge_result"]["metadata"]["raw_answer_judgement"] == "0.333333333" + case = CaseSpec(case_id="c", sessions=[], queries=[QuerySpec(query_id="q", question="?", golden_answer=None)]) + write_json(tmp_path / "cases/c/memory_construction/result.json", {"status": "success", "jobs": []}) + # Legacy long-precision query files also produce short-precision summaries. + write_json(tmp_path / "cases/c/queries/result.json", {"queries": [raw]}) + summary = summarize(tmp_path, [case]) + assert summary["mean_query_score"] == summary["observed_mean_query_score"] == 0.3333 + assert summary["low_score_cases"] == [{"c": 0.3333}] + assert summary["cases"]["c"]["mean_score"] == summary["cases"]["c"]["queries"]["q"]["score"] == 0.3333 + assert summary["question_type_statistics"]["unknown"]["average_score"] == 0.3333 + + +@pytest.mark.parametrize("scoring_enabled", [True, False]) +def test_summary_separates_query_errors_from_scores(tmp_path, scoring_enabled): + ids = ["zero", "unscored", "candidate", "infra"] + case = CaseSpec( + case_id="c", + sessions=[], + queries=[QuerySpec(query_id=value, question="?", golden_answer=None) for value in ids], + ) + queries = [ + {"query_id": "zero", "score": 0}, + {"query_id": "unscored", "score": None}, + {"query_id": "candidate", "score": None, "error": "bad answer", "failure_stage": "answer"}, + { + "query_id": "infra", + "score": None, + "error": "x" * 600, + "failure_stage": "judge", + "infrastructure_error": True, + }, + ] + write_json(tmp_path / "cases/c/memory_construction/result.json", {"status": "success", "jobs": []}) + source = tmp_path / "cases/c/queries/result.json" + write_json(source, {"queries": queries}) + summary = summarize(tmp_path, [case], scoring_required=scoring_enabled) + assert summary["scoring_enabled"] is scoring_enabled + row = summary["cases"]["c"] + assert all("status" not in query for query in row["queries"].values()) + assert row["queries"]["zero"]["score"] == 0 + assert row["queries"]["unscored"]["score"] is None + assert row["query_errors"] == { + "candidate": {"failure_kind": "candidate_failure", "stage": "answer", "error": "bad answer"}, + "infra": {"failure_kind": "infra_error", "stage": "judge", "error": "x" * 500 + "…"}, + } + assert read_json(source) == {"queries": queries} + + def test_query_compaction_preserves_evidence_without_duplicate_prompts_or_costs(): raw = { "question": "What happened?", @@ -195,13 +266,14 @@ def test_summary_matches_reviewed_dev_meta_fields_and_denominators(tmp_path): summary = summarize(tmp_path, cases) assert summary == { "case_count": 2, + "scoring_enabled": True, "case_status_counts": {"completed": 0, "candidate_failure": 2, "infra_error": 0}, "construction_status_counts": {"success": 0, "partial_success": 1, "failure": 1}, "failure_stage_case_counts": {"construction": 2, "query": 1}, "expected_query_count": 3, "scored_query_count": 1, "query_status_counts": {"completed": 1, "candidate_failure": 1, "infra_error": 0, "unscored": 2}, - "mean_query_score": 0.5 / 3, + "mean_query_score": 0.1667, "observed_mean_query_score": 0.5, "low_score_cases": [{"one": 0.25}], "question_type_statistics": { @@ -213,9 +285,10 @@ def test_summary_matches_reviewed_dev_meta_fields_and_denominators(tmp_path): "mean_score": 0.25, "average_query_total_token_usage": {"answer": 6.0}, "queries": { - "q1": {"question_type": "fact", "score": 0.5, "status": "completed"}, - "q2": {"question_type": "fact", "score": None, "status": "candidate_failure"}, + "q1": {"question_type": "fact", "score": 0.5}, + "q2": {"question_type": "fact", "score": None}, }, + "query_errors": {"q2": {"failure_kind": "candidate_failure", "stage": None, "error": "candidate bug"}}, "construction_error_count": 1, "query_error_count": 1, }, diff --git a/plugins/home/tests/test_evaluation_history.py b/plugins/home/tests/test_evaluation_history.py index bb2d5c98..5b4f6171 100644 --- a/plugins/home/tests/test_evaluation_history.py +++ b/plugins/home/tests/test_evaluation_history.py @@ -42,9 +42,13 @@ async def test_history_catalog_keeps_successful_low_scores_and_deduplicates(app, before = manager.sampling_state() assert manager.record(run) == results[0] assert len(manager.list_results()) == 1 and manager.sampling_state() == before - payload = json.loads((prepared / "evaluations/history.json").read_text()) - assert payload["results"][0]["run_key"] == results[0].run_key - assert "golden_answer" not in json.dumps(payload) + assert not (prepared / "evaluations/history.json").exists() + # A stale or corrupt legacy index cannot hide authoritative results. + legacy = prepared / "evaluations/history.json" + legacy.write_text("obsolete index") + assert manager.list_results() == results + assert manager.rebuild() == results + assert legacy.read_text() == "obsolete index" debug = await app.run_job("home_evaluate", sampling=True, debug=True) assert debug.success and manager.record(Path(debug.answer["result_dir"])) is None assert manager.list_results() == results @@ -106,13 +110,12 @@ async def test_history_and_observations_rebuild_without_touching_artifacts(app, expected, observations = manager.list_results(), manager.sampling_state() source = Path(sampled.answer["result_dir"]) / "summary.json" original = source.read_bytes() - (prepared / "evaluations/history.json").unlink() (prepared / "sampling/state.json").unlink() assert manager.list_results() == expected assert manager.sampling_state() == observations assert manager.rebuild() == expected assert source.read_bytes() == original - assert (prepared / "evaluations/history.json").is_file() + assert not (prepared / "evaluations/history.json").exists() assert (prepared / "sampling/state.json").is_file() @@ -124,7 +127,9 @@ async def test_call_policy_merges_component_defaults_without_mutating_them(app, assert response.success, response.answer options = manifest(response)["options"] assert options["sampling"] is True - assert options["sampling_policy"] == { + assert "sampling_policy" not in options + plan = json.loads((prepared / manifest(response)["sampling"]["plan_path"]).read_text()) + assert plan["policy"] == { "max_cases": 1, "max_queries": 2, "temperature": 0.7, @@ -135,7 +140,8 @@ async def test_call_policy_merges_component_defaults_without_mutating_them(app, following = await app.run_job("home_evaluate", sampling=True) assert following.success and following.answer["case_count"] == 2 assert evaluator.sampling_policy.max_cases == 2 - assert manifest(following)["options"]["sampling_policy"]["max_cases"] == 2 + plan = json.loads((prepared / manifest(following)["sampling"]["plan_path"]).read_text()) + assert plan["policy"]["max_cases"] == 2 for kwargs in ( {"sampling": "true"}, {"sampling": {"max_cases": 1}}, @@ -182,7 +188,7 @@ async def test_standalone_full_evaluation_preserves_history_and_rebuild_excludes if baseline: assert (await app.run_job("home_evaluate")).success records, observations = manager.list_results(), manager.sampling_state() - cache_paths = [prepared / name for name in ("evaluations/history.json", "sampling/state.json")] + cache_paths = [prepared / name for name in ("sampling/state.json",)] caches = [path.read_bytes() if path.exists() else None for path in cache_paths] response = await app.run_job("home_evaluate", sampling=False, record_history=False) assert response.success, response.answer @@ -251,7 +257,7 @@ async def test_untracked_failures_preserve_existing_history(app, prepared, monke assert (await app.run_job("home_evaluate")).success manager = history(app) records, observations = manager.list_results(), manager.sampling_state() - paths = [prepared / name for name in ("evaluations/history.json", "sampling/state.json")] + paths = [prepared / name for name in ("sampling/state.json",)] caches = [path.read_bytes() for path in paths] if failure == "infra": monkeypatch.setattr(FakeFactory, "bad_export", True) @@ -293,7 +299,8 @@ async def test_legacy_cache_rebuild_and_explicit_null_query_budget(app, prepared evaluator.sampling_policy = SamplingPolicy(max_cases=2, max_queries=1) result = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_queries": None}) assert result.success and result.answer["scored_query_count"] == 2 - assert manifest(result)["options"]["sampling_policy"]["max_queries"] is None + plan = json.loads((prepared / manifest(result)["sampling"]["plan_path"]).read_text()) + assert plan["policy"]["max_queries"] is None assert evaluator.sampling_policy.max_queries == 1 manager = history(app) observations = manager.sampling_state() @@ -340,12 +347,12 @@ async def test_cancellation_after_durable_completion_keeps_history_consistent(ap @pytest.mark.asyncio async def test_registration_failure_invalidates_only_caches(app, prepared, monkeypatch): manager = history(app) - original = manager._index + original = manager.record - def fail(): - raise OSError("history index could not be loaded") + def fail(*args): + raise OSError("sampling observations could not be saved") - monkeypatch.setattr(manager, "_index", fail) + monkeypatch.setattr(manager, "record", fail) response = await app.run_job("home_evaluate") assert not response.success runs = list((prepared / "evaluations").glob("*/*/*/manifest.json")) @@ -356,5 +363,5 @@ async def test_registration_failure_invalidates_only_caches(app, prepared, monke assert (run / "summary.json").is_file() and (run / "cases/1/queries/result.json").is_file() assert not (prepared / "sampling/state.json").exists() assert read_state(prepared)["round"] == 1 and read_state(prepared)["successful_round"] == 0 - monkeypatch.setattr(manager, "_index", original) + monkeypatch.setattr(manager, "record", original) assert manager.list_results() == [] and not manager.sampling_state()["cases"] diff --git a/plugins/home/tests/test_preparation.py b/plugins/home/tests/test_preparation.py index f5d1a664..be91daf3 100644 --- a/plugins/home/tests/test_preparation.py +++ b/plugins/home/tests/test_preparation.py @@ -149,7 +149,7 @@ async def test_python_job_prepares_once(app, inputs): case = json.loads((root / "home/datasets/training-set/cases/000000.json").read_text()) assert case["sessions"][0]["messages"][0]["id"] == "1" assert case["queries"][0]["metadata"]["evidence_session_ids"] == ["beam_100K_2_batch1"] - assert (root / "home/datasets/debug-cases/derivation.json").is_file() + assert not (root / "home/datasets/debug-cases/derivation.json").exists() docs = root / "home/docs" resources = resource_files("reme_home").joinpath("runtime_docs") assert sorted(path.name for path in docs.iterdir()) == ["architecture-guide.md", "validation-artifact-guide.md"] diff --git a/plugins/home/tests/test_query_evidence.py b/plugins/home/tests/test_query_evidence.py new file mode 100644 index 00000000..e77e6929 --- /dev/null +++ b/plugins/home/tests/test_query_evidence.py @@ -0,0 +1,221 @@ +"""Compact evidence tracing against real, isolated evaluation and Git artifacts.""" + +# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access + +import json +import shutil + +import pytest +from pydantic import ValidationError +from reme_home.evaluation import evidence_step as evidence_tools +from reme_home.evaluation.artifacts import write_json +from reme_home.preparation.code import git + +from test_memory_inspection import ( # noqa: F401 # pylint: disable=unused-import + FakeFactory, + commit, + evidence, + home_application, + preparation_inputs, + prepared_home, +) +from test_evaluation_inspection import call, snapshot, write_run + + +@pytest.fixture +def query_evidence(evidence): # noqa: F811 + home, run, repo, first, second = evidence + manifest = json.loads((run / "manifest.json").read_text()) + manifest["query_ids"] = {"case1": ["q1"]} + write_json(run / "manifest.json", manifest) + case = { + "case_id": "case1", + "sessions": [{"session_id": "session1", "messages": [{"content": "source transcript"}]}], + "queries": [ + { + "query_id": "q1", + "question": "What is the fact?", + "golden_answer": "old fact", + "metadata": {"evidence_session_ids": ["session1", "session1"], "evidence_is_certain": False}, + }, + ], + } + source = home / "datasets/training-set/cases/000000.json" + write_json(source, case) + write_json( + run / "cases/case1/queries/result.json", + {"queries": [{"query_id": "q1", "answer": "new fact", "score": 0, "judge_result": {"large": "hidden"}}]}, + ) + return home, run, repo, first, second, source, case + + +def trace(home, run, **kwargs): + spec = evidence_tools.TraceQueryEvidenceInput(validation_path=str(run), case_id="case1", query_id="q1", **kwargs) + return evidence_tools.trace_query_evidence(home, spec) + + +def test_compact_annotation_root_checkpoint_and_read_only(query_evidence): + home, run, repo, first, _second, source, _case = query_evidence + (repo / "memory/note.md").write_text("dirty ignored") + before = snapshot(home) + # Independent annotations take precedence, even over nonexistent inferred IDs. + result = trace(home, run, evidence_session_ids=["unknown"]) + assert result["query"] == { + "question": "What is the fact?", + "expected": "old fact", + "actual": "new fact", + "score": 0, + "error": None, + "result_missing": False, + } + assert result["evidence_source"] == "independent_annotation" + assert result["evidence_is_certain"] is False and result["memory_status"] == "available" + assert result["sessions"] == [ + {"id": "session1", "checkpoint": first, "changed_files": ["memory/note.md", "removed.txt"]}, + ] + assert result["paths"]["source_case"] == str(source) + assert result["paths"]["memory_repository"] == str(repo) + assert "source transcript" not in json.dumps(result) and "judge_result" not in json.dumps(result) + assert snapshot(home) == before + + +def test_inference_missing_checkpoint_and_debug_dataset(query_evidence): + home, run, _repo, _first, second, source, case = query_evidence + case["sessions"] += [{"session_id": "session2"}, {"session_id": "not-constructed"}] + case["queries"][0]["metadata"] = {} + debug_source = home / "datasets/debug-cases/cases/000001.json" + write_json(debug_source, case) + source.unlink() + manifest = json.loads((run / "manifest.json").read_text()) + manifest["dataset_kind"] = "debug-cases" + write_json(run / "manifest.json", manifest) + result = trace(home, run, evidence_session_ids=["session2", "not-constructed", "session2"]) + assert result["evidence_source"] == "diagnostician_inference" + assert result["paths"]["source_case"] == str(debug_source) + assert result["sessions"][0]["checkpoint"] == second + assert result["sessions"][0]["changed_files"] == ["added.bin", "memory/note.md", "removed.txt"] + assert result["sessions"][1] == {"id": "not-constructed", "checkpoint": None, "changed_files": []} + assert trace(home, run)["evidence_source"] == "unavailable" + with pytest.raises(ValueError, match="not present"): + trace(home, run, evidence_session_ids=["unknown"]) + + +@pytest.mark.parametrize("annotation", [[], [""], "session1", [123]]) +def test_empty_and_invalid_annotations(query_evidence, annotation): + home, run, _repo, _first, _second, source, case = query_evidence + case["queries"][0]["metadata"]["evidence_session_ids"] = annotation + write_json(source, case) + if not annotation: + result = trace(home, run, evidence_session_ids=["session1"]) + assert not result["sessions"] and result["evidence_source"] == "independent_annotation" + else: + with pytest.raises(ValidationError): + trace(home, run) + + +@pytest.mark.parametrize("state", ["workspace_missing", "history_missing", "history_empty"]) +def test_missing_history_and_results(query_evidence, state): + home, run, repo, _first, _second, _source, _case = query_evidence + shutil.rmtree(repo if state == "workspace_missing" else repo / ".git") + if state == "history_empty": + git(repo, "init") + (run / "cases/case1/queries/result.json").unlink() + result = trace(home, run) + assert result["memory_status"] == state and result["sessions"][0]["checkpoint"] is None + assert result["query"]["result_missing"] and result["query"]["score"] is None + assert result["query"]["actual"] is None and result["query"]["expected"] == "old fact" + + +def test_large_fields_and_lists_are_bounded(query_evidence): + home, run, repo, _first, _second, source, case = query_evidence + for index in range(55): + (repo / f"file-{index:03}").write_text("fact") + checkpoint = commit(repo, "session: session1") + case["sessions"] = [{"session_id": f"s{index}"} for index in range(55)] + case["sessions"] + case["queries"][0]["metadata"]["evidence_session_ids"] = ["session1"] + [f"s{i}" for i in range(55)] + case["queries"][0]["question"] = "x" * 100_000 + write_json(source, case) + result = trace(home, run) + assert len(result["sessions"]) == 50 and result["omitted_session_count"] == 6 + assert result["sessions"][0]["checkpoint"] == checkpoint + assert len(result["sessions"][0]["changed_files"]) == 50 + assert result["sessions"][0]["omitted_changed_file_count"] == 5 + assert len(result["query"]["question"]) < 2100 + assert len(json.dumps(result)) < 15_000 + + +def test_reject_unselected_queries_and_external_sources(query_evidence, tmp_path): + home, run, _repo, _first, _second, source, _case = query_evidence + manifest = json.loads((run / "manifest.json").read_text()) + manifest["query_ids"] = {"case1": []} + write_json(run / "manifest.json", manifest) + with pytest.raises(ValueError, match="not selected"): + trace(home, run) + manifest["query_ids"] = {"case1": ["q1"]} + write_json(run / "manifest.json", manifest) + source.unlink() + source.symlink_to(tmp_path / "external.json") + with pytest.raises(ValueError, match="Symbolic link"): + trace(home, run) + source.unlink() + with pytest.raises(ValueError, match="case is missing"): + trace(home, run) + + +def test_missing_query_invalid_dataset_and_corrupt_history(query_evidence): + home, run, repo, _first, _second, source, case = query_evidence + case["queries"] = [] + write_json(source, case) + with pytest.raises(ValueError, match="has no query"): + trace(home, run) + manifest = json.loads((run / "manifest.json").read_text()) + manifest["dataset_kind"] = "../outside" + write_json(run / "manifest.json", manifest) + with pytest.raises(ValueError, match="dataset_kind"): + trace(home, run) + manifest["dataset_kind"] = "training-set" + write_json(run / "manifest.json", manifest) + case["queries"] = [{"query_id": "q1"}] + write_json(source, case) + (repo / ".git/objects/info/alternates").write_text("/external/objects") + with pytest.raises(ValueError, match="indirect"): + trace(home, run) + + +def test_run_and_case_scope(query_evidence): + home, run, _repo, _first, _second, _source, _case = query_evidence + with pytest.raises(ValueError, match="case_id"): + evidence_tools.trace_query_evidence( + home, + evidence_tools.TraceQueryEvidenceInput(validation_path=str(run), case_id="unknown", query_id="q1"), + ) + with pytest.raises(ValueError, match="escapes"): + trace(home, home.parent / "outside") + + +@pytest.mark.parametrize("kwargs", [{"query_id": "../q"}, {"case_id": "../c"}, {"evidence_session_ids": [""]}]) +def test_invalid_inputs(kwargs): + with pytest.raises(ValidationError): + evidence_tools.TraceQueryEvidenceInput.model_validate( + {"validation_path": "run", "case_id": "case1", "query_id": "q1", **kwargs}, + ) + + +@pytest.mark.asyncio +async def test_registered_job_and_restricted_scope(app, prepared, query_evidence): + home, source_run, _repo, _first, _second, _source, _case = query_evidence + run = write_run(prepared, case_ids=["case1"], query_ids={"case1": ["q1"]}) + shutil.copytree(home / "datasets/training-set", prepared / "datasets/training-set", dirs_exist_ok=True) + shutil.copytree(source_run / "cases", run / "cases") + before = snapshot(prepared) + result = await call(app, "home_trace_query_evidence", validation_path=str(run), case_id="case1", query_id="q1") + assert result["query"]["score"] == 0 and result["sessions"][0]["checkpoint"] + response = await app.run_job( + "home_trace_query_evidence", + validation_path=str(run), + case_id="case1", + query_id="q1", + _allowed_paths=[str(run)], + ) + assert not response.success and "unrestricted" in str(response.answer) + assert snapshot(prepared) == before