From febc84586e00cf6c19b2f52f9dcdf98b7bcdf591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E7=89=9B=E8=82=89?= Date: Fri, 30 Oct 2026 17:17:17 +0800 Subject: [PATCH] feat(home): support multi-case case-study runs with dedicated LLM and Git sandbox - Allow home_case_study to investigate up to eight ordered cases per invocation with a shared focus, one isolated agent per case, while keeping the legacy single-case selector as a deprecated alias - Configure a dedicated home_case_study LLM component instead of the shared judge backend - Wrap ReMe's OS file sandbox with a Home-specific Git runtime grant (developer directory on macOS) and argv-based child environment - Tighten case-study prompts around evidence certainty, code-tool path scoping, citation form, and unresolved-cause reporting - Extend evidence tracing and expand case-study/sandbox tests and docs --- plugins/home/README.md | 50 ++-- .../home/src/reme_home/agent/case_study.py | 229 +++++++++++++++--- .../home/src/reme_home/agent/case_study.yaml | 20 +- plugins/home/src/reme_home/agent/sandbox.py | 76 ++++++ plugins/home/src/reme_home/agent/wrapper.py | 25 +- .../src/reme_home/evaluation/evidence_step.py | 53 +++- plugins/home/src/reme_home/plugin.yaml | 39 ++- .../runtime_docs/validation-artifact-guide.md | 33 ++- plugins/home/tests/test_case_study.py | 214 +++++++++++++++- plugins/home/tests/test_case_study_sandbox.py | 96 ++++++++ plugins/home/tests/test_query_evidence.py | 74 +++++- 11 files changed, 824 insertions(+), 85 deletions(-) create mode 100644 plugins/home/src/reme_home/agent/sandbox.py create mode 100644 plugins/home/tests/test_case_study_sandbox.py diff --git a/plugins/home/README.md b/plugins/home/README.md index 997d9e84..fc8edcc1 100644 --- a/plugins/home/README.md +++ b/plugins/home/README.md @@ -674,14 +674,18 @@ index, evaluation history or sampling state. memory checkpoints without calling a model. Dataset evidence annotations take precedence over optional inferred session IDs, including an empty annotation. Returns `query` (question, expected/actual answer, score, error and - `result_missing`), `evidence_source` (`independent_annotation`, + `result_missing` and `expected_kind`), `locations` with JSON Pointers into source + and saved query files, optional `criteria` previews of saved rubric judgments, + `evidence_source` (`independent_annotation`, `diagnostician_inference`, or `unavailable`), `evidence_is_certain`, - `memory_status`, `sessions` (ID, checkpoint and changed files), and `paths` to + `memory_status`, `sessions` (ID, source JSON Pointer, checkpoint and changed files), and `paths` to the normalized case, memory repository, query results and query artifacts. A null checkpoint means no matching `session: ` commit was found; missing results keep scores null. Answer/question previews are limited to 2,000 characters and errors to 600. At most 50 sessions and 50 changed files per - session are returned, with omitted counts when needed. Session text, diffs, + session are returned, with omitted counts when needed. Criterion previews are + limited to 20 entries and 600 characters per requirement/reason; absent or + mismatched rubric judgments are not paired. Session text, diffs, later history and full answer/judge results remain at the returned paths or in the inspection/comparison Jobs below. Original transcripts and derived memories can both be searchable; these coordinates alone do not prove a cause. @@ -726,26 +730,38 @@ form is `evaluations/candidate/abcdef0/1`. For agent tools, host `injected_job_k `validation_path` and `case_id`; the wrapper hides injected fields and rejects model overrides. Like the other Home inspection Jobs, these Jobs reject `_allowed_paths` restrictions rather than bypassing them. Use these tools directly -or through the single-case investigator below. +or through the case-study batch Job below. -## Single-case investigation +## Case-study batches -`home_case_study(validation_path, case_id, focus=...)` runs one fresh diagnostic -agent. It follows dev_meta's case-study workflow: trace judging and answer -failures backward through retrieval, memory construction, source sessions and +`home_case_study(evaluation_path, case_ids, focus=...)` investigates up to eight +unique case IDs in input order, sharing the evaluation and unchanged focus. +Cases run sequentially, each with fresh agent state, sandbox and timer. The +whole selection is validated before model calls; a case execution failure is +returned individually without cancelling later cases. It follows dev_meta's +case-study workflow: trace judging and answer failures backward through retrieval, +memory construction, source sessions and the evaluated implementation. The text report cites evidence paths/lines or JSON fields and distinguishes supported causes, hypotheses and missing evidence; -it does not prescribe fixes. There is no batch fan-out or parallel case-study -dispatcher. Each invocation has its own agent state and timer. +it does not prescribe fixes. There is no parallel case-study dispatcher. The `home_case_study` component uses `home_case_study_agent`, a thin subclass of -the bounded wrapper, defaulting to -the host's existing `as_llm: judge` model, 60 iterations and a 30-minute advisory -budget, matching dev_meta's default budgets. Configure a separate host model by -overriding `components.agent_wrapper.home_case_study.as_llm`. ReMe owns model +the bounded wrapper, defaulting to the dedicated `as_llm: home_case_study` +component with model `qwen3.7-max`, 60 iterations and a 30-minute advisory +budget, matching dev_meta's default budgets. It uses `LLM_BACKEND`, `LLM_API_KEY` +and `LLM_BASE_URL` without changing the evaluation judge's model. Override +`components.as_llm.home_case_study.model` to choose another model, or +`components.agent_wrapper.home_case_study.as_llm` to select another host component. ReMe owns model startup, token accounting and saved session transcripts. The response answer is -the report; metadata includes the validation, case, evaluated commit, session ID -and `usage`. No case scratch directory or intermediate analysis files are created. +`{"results": [...]}`; each item contains `case_id`, `success`, and either `report` +with its session ID/usage or `error`. Metadata includes `evaluation_path`, the +evaluated commit and total `usage` (null if any case fails or usage is unavailable). +Response success is false if any case fails; completed sibling reports remain +available. No case scratch directory or intermediate analysis files are created. + +`validation_path` remains an alias for `evaluation_path`. Legacy `case_id` calls +still return a text report with per-case metadata. Do not supply both path names +or both case selectors. Other inspection Jobs retain their `validation_path` argument. The toolkit includes validation inspection, compact query-evidence tracing, memory history/comparison, and the three committed-code navigation Jobs. @@ -785,7 +801,7 @@ jobs: With the configured service running, call from another terminal: ```bash -reme home_case_study validation_path=evaluations/candidate/abcdef0/1 case_id=case1 'focus=Trace low-scoring queries' +reme home_case_study evaluation_path=evaluations/candidate/abcdef0/1 'case_ids=["case1","case2"]' 'focus=Trace low-scoring queries' ``` Calls with `_allowed_paths` restrictions fail closed, as other Home artifact diff --git a/plugins/home/src/reme_home/agent/case_study.py b/plugins/home/src/reme_home/agent/case_study.py index cccebd1d..a8256d8b 100644 --- a/plugins/home/src/reme_home/agent/case_study.py +++ b/plugins/home/src/reme_home/agent/case_study.py @@ -1,14 +1,15 @@ -"""One case-local investigation using ReMe Jobs and sandboxed command tools.""" +"""Sequential case-local investigations using ReMe Jobs and sandboxed command tools.""" +import json import sys from pathlib import Path from agentscope.tool import FunctionTool, Toolkit -from pydantic import Field, field_validator +from pydantic import Field, field_validator, model_validator import yaml +from reme.schema.token_usage import TokenUsage from reme.steps.base_step import BaseStep, Ref -from reme.utils.file_sandbox import create_file_sandbox_backend from ..auto_bash import AutoBash from ..code.repository import resolve_revision, validate_repository @@ -19,9 +20,11 @@ from ..evaluation.inputs import entity_id from ..evaluation.memory_inspection_step import _repository from ..preparation.spec import SpecModel from ..workspace import HomeWorkspace +from .sandbox import create_case_study_sandbox from .wrapper import BoundedAgentWrapper PROMPT = Path(__file__).with_suffix(".yaml") +DEFAULT_FOCUS = "Investigate the failed or low-scoring queries and their earliest supported causes." CASE_TOOLS = ( "home_inspect_validation", "home_trace_query_evidence", @@ -29,6 +32,33 @@ CASE_TOOLS = ( "home_compare_memory", ) CODE_TOOLS = ("home_list_code_tree", "home_search_code", "home_read_code_file") +_SANDBOX_PROBE = """\ +import json +import pathlib +import subprocess +import sys + +scope = json.loads(sys.argv[1]) +with open(scope['source_case'], 'rb') as stream: + stream.read(1) +with open(pathlib.Path(scope['validation_path']) / 'manifest.json', 'rb') as stream: + stream.read(1) +subprocess.run([ + '/bin/bash', '--noprofile', '--norc', '-c', + 'git -C "$1" --no-pager -c core.fsmonitor=false -c log.showSignature=false ' + 'show --no-ext-diff --no-textconv --no-patch --format=%H "$2"', + 'probe', scope['code_repository'], scope['candidate_commit'], +], check=True) +""" + + +async def probe_sandbox(backend, scope): + """Verify the same interpreter, shell, Git and evidence access used by the investigation.""" + return await backend.exec_shell( + [sys.executable, "-I", "-B", "-c", _SANDBOX_PROBE, json.dumps(scope)], + cwd=scope["case_path"], + timeout=10, + ) class CaseStudyInput(SpecModel): @@ -37,7 +67,7 @@ class CaseStudyInput(SpecModel): validation_path: str = Field(min_length=1) case_id: str = Field(min_length=1) focus: str = Field( - default="Investigate the failed or low-scoring queries and their earliest supported causes.", + default=DEFAULT_FOCUS, min_length=1, ) @@ -48,6 +78,39 @@ class CaseStudyInput(SpecModel): return entity_id(value) +class CaseStudyBatchInput(SpecModel): + """Share one evaluation and focus across an ordered, bounded case selection.""" + + evaluation_path: str = Field(min_length=1) + case_ids: list[str] = Field(min_length=1, max_length=8) + focus: str = Field(default=DEFAULT_FOCUS, min_length=1) + + @model_validator(mode="before") + @classmethod + def normalize_legacy(cls, value): + """Accept old single-case arguments without accepting ambiguous combinations.""" + if not isinstance(value, dict): + return value + value = dict(value) + for old, new in (("validation_path", "evaluation_path"), ("case_id", "case_ids")): + if old in value: + if new in value: + raise ValueError(f"Provide {new} or {old}, not both") + item = value.pop(old) + value[new] = [item] if old == "case_id" else item + return value + + @field_validator("case_ids") + @classmethod + def validate_cases(cls, values): + """Reject unsafe or duplicate selectors before any agent runs.""" + for value in values: + entity_id(value) + if len(set(values)) != len(values): + raise ValueError("case_ids must not contain duplicates") + return values + + def prepare_scope(home, spec): """Resolve evidence and the validation's immutable candidate revision before model calls.""" run, manifest, _memory = _repository(home, spec.validation_path, spec.case_id) @@ -91,6 +154,11 @@ def python_tool(backend, cwd): ) -> dict: """Execute code or file, exclusively; print results to stdout. No host path is writable. + Read case JSON, memory files and documentation here, not with home_read_code_file. + For example code="import json; print(json.load(open('/allowed/result.json'))['queries'][0])". + In code mode do not pass args; in file mode pass file and args, without code. + Check returncode and stderr: a denied or failed read is not evidence that content is absent. + Args: code: Python source, mutually exclusive with file. timeout: Execution timeout in seconds, greater than zero and at most 600. @@ -135,10 +203,64 @@ def create_toolkit(wrapper, scope, backend): ) if name == "home_compare_memory": injected["target_validation_path"] = None - tools.append(wrapper._make_tool(job, injected_job_kwargs=injected)) # pylint: disable=protected-access + tool = wrapper._make_tool( # pylint: disable=protected-access + job, + injected_job_kwargs=injected, + ) + if name in CODE_TOOLS: + tool.description += ( + " Reads candidate source at the host-pinned evaluated commit, not artifact or documentation files." + " Paths are repository-relative, for example plugins/beam/src/reme_beam/agentic_answer.yaml;" + " omit path to search/list the root, never use '.' or an absolute path." + " Read memory, logs, JSON and home/docs with AutoBash or python_execute instead." + ) + if "path" in tool.input_schema.get("properties", {}): + description = ( + "Required candidate-repository-relative file path, never absolute." + if name == "home_read_code_file" + else "Literal repository-relative path, never absolute; omit for the whole repository, not '' or '.'." + ) + tool.input_schema["properties"]["path"] = { + **tool.input_schema["properties"]["path"], + "description": description, + } + if name == "home_trace_query_evidence": + tool.description += ( + " For a selected query, start here. locations are JSON Pointers into paths.source_case/query_result;" + " session source_pointer locates messages in source_case. criteria previews saved rubric judgments" + " when available. Hit counts and memory coordinates do not prove what reached the answerer." + ) + if name == "home_inspect_memory_history": + tool.description += " path filters files within the memory repository; omit it to list all checkpoints." + if name == "home_compare_memory": + tool.input_schema["required"] = list(dict.fromkeys([*tool.input_schema.get("required", []), "base_ref"])) + tool.input_schema["properties"]["base_ref"] = { + "type": "string", + "minLength": 1, + "description": ( + "Required earlier memory commit from home_inspect_memory_history;" + " compare with a different target_ref." + ), + } + tool.description += ( + " Same-case comparison only: supply base_ref and target_ref from home_inspect_memory_history." + ' Example: {"base_ref": "", "target_ref": "", "path": "daily"}.' + " Identical refs have no changes; omit path for the whole memory tree." + ) + tools.append(tool) + bash = AutoBash(cwd=scope["case_path"], backend=backend, is_read_only=False) + bash.description = ( + "Read case artifacts, memory, logs and home/docs with shell commands inside the OS file sandbox." + " No command whitelist; host paths are read-only and network is denied." + " Use absolute allowed paths, e.g. cat '/queries//answer.log', or" + " git -C '' show ':daily/note.md'." + " Prefer pinned code tools for evaluated candidate source. Keep stderr visible; for pipelines use" + " set -o pipefail so a failed read is not hidden by head/grep. A failed read does not prove absence." + " timeout is in milliseconds." + ) tools.extend( [ - AutoBash(cwd=scope["case_path"], backend=backend, is_read_only=False), + bash, python_tool(backend, scope["case_path"]), ], ) @@ -148,36 +270,30 @@ def create_toolkit(wrapper, scope, backend): class HomeCaseStudyStep(BaseStep): - """Run one fresh agent through the application's bounded wrapper and model lifecycle.""" + """Run ordered cases with fresh agents through the normal ReMe lifecycle.""" workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace") agent_wrapper: BoundedAgentWrapper = Ref(BoundedAgentWrapper, "agent_wrapper") async def execute(self): - """Prepare scope, require isolation, then return the case report and framework token usage.""" + """Validate the whole batch, then isolate each sequential investigation and failure.""" if self.context.get("_allowed_paths") is not None: raise ValueError("Home case study requires unrestricted access to its managed artifacts") if not self.workspace.is_started or not self.agent_wrapper.is_started: raise RuntimeError("Start the Home workspace and case-study agent wrapper first") - spec = CaseStudyInput.model_validate( - {key: self.context[key] for key in CaseStudyInput.model_fields if key in self.context}, - ) - scope = await blocking(prepare_scope, self.workspace.home_path, spec) - backend = create_file_sandbox_backend( - cwd=scope["case_path"], - read_only_paths=scope["read_only_paths"], - writable_paths=[], - memory_limit_mib=self.kwargs.get("memory_limit_mib"), - ) - # Fail before spending model tokens if the OS sandbox or its runtime cannot start. - probe = await backend.exec_shell( - [sys.executable, "-I", "-B", "-c", "pass"], - cwd=scope["case_path"], - timeout=10, - ) - if not probe.ok(): - raise RuntimeError("Case-study sandbox unavailable: " + probe.stderr.decode(errors="replace")) - toolkit = create_toolkit(self.agent_wrapper, scope, backend) + fields = (*CaseStudyBatchInput.model_fields, "validation_path", "case_id") + spec = CaseStudyBatchInput.model_validate({key: self.context[key] for key in fields if key in self.context}) + scopes = [] + for case_id in spec.case_ids: + scopes.append( + await blocking( + prepare_scope, + self.workspace.home_path, + CaseStudyInput(validation_path=spec.evaluation_path, case_id=case_id, focus=spec.focus), + ), + ) + if len({scope["candidate_commit"] for scope in scopes}) != 1: + raise ValueError("Evaluation candidate commit changed while preparing case scopes") output_limit = self.kwargs.get("output_token_limit", 2500) if isinstance(output_limit, bool) or not isinstance(output_limit, int) or output_limit < 1: raise ValueError("output_token_limit must be a positive integer") @@ -187,9 +303,58 @@ class HomeCaseStudyStep(BaseStep): for key in ("system_prompt", "task_prompt") ): raise ValueError("Case-study YAML requires nonempty system_prompt and task_prompt strings") + response = self.context.response + response.metadata.update( + evaluation_path=scopes[0]["validation_path"], + candidate_commit=scopes[0]["candidate_commit"], + ) + # Preserve the original text response and error handling for singular callers. + if "case_id" in self.context: + result = await self._investigate(scopes[0], spec.focus, prompts, output_limit) + response.answer = result["result"] + response.metadata.update(validation_path=scopes[0]["validation_path"], case_id=spec.case_ids[0]) + response.metadata.update({key: result[key] for key in ("session_id", "usage") if key in result}) + return response + results = [] + response.answer = {"results": results} + for scope in scopes: + try: + result = await self._investigate(scope, spec.focus, prompts, output_limit) + results.append( + { + "case_id": scope["case_id"], + "success": True, + "report": result["result"], + **{key: result[key] for key in ("session_id", "usage") if key in result}, + }, + ) + except Exception as exc: # A failed investigator must not discard or cancel sibling reports. + self.logger.exception(f"Case-study investigation failed: case_id={scope['case_id']}") + results.append({"case_id": scope["case_id"], "success": False, "error": f"{type(exc).__name__}: {exc}"}) + response.success = all(item["success"] for item in results) + # Failed or unreported calls have unknown usage; never present a partial sum as a full total. + response.metadata["usage"] = ( + TokenUsage.combine([TokenUsage.model_validate(item["usage"]) for item in results]).model_dump() + if all(item.get("usage") is not None for item in results) + else None + ) + return response + + async def _investigate(self, scope, focus, prompts, output_limit): + """Build a fresh case-scoped toolkit and SDK agent for exactly one case.""" + backend = create_case_study_sandbox( + cwd=scope["case_path"], + read_only_paths=scope["read_only_paths"], + writable_paths=[], + memory_limit_mib=self.kwargs.get("memory_limit_mib"), + ) + probe = await probe_sandbox(backend, scope) + if not probe.ok(): + raise RuntimeError("Case-study sandbox unavailable: " + probe.stderr.decode(errors="replace")) + toolkit = create_toolkit(self.agent_wrapper, scope, backend) prompt = prompts["system_prompt"].format(**scope, output_token_limit=output_limit) - result = await self.agent_wrapper.reply( - prompts["task_prompt"].format(focus=spec.focus), + return await self.agent_wrapper.reply( + prompts["task_prompt"].format(focus=focus), system_prompt=prompt, toolkit=toolkit, builtin_tools=[], @@ -201,9 +366,3 @@ class HomeCaseStudyStep(BaseStep): output_schema=None, permission_mode="bypass", ) - self.context.response.answer = result["result"] - self.context.response.metadata.update( - {key: scope[key] for key in ("validation_path", "case_id", "candidate_commit")}, - ) - self.context.response.metadata.update({key: result[key] for key in ("session_id", "usage") if key in result}) - return self.context.response diff --git a/plugins/home/src/reme_home/agent/case_study.yaml b/plugins/home/src/reme_home/agent/case_study.yaml index 0750393f..a9e561cd 100644 --- a/plugins/home/src/reme_home/agent/case_study.yaml +++ b/plugins/home/src/reme_home/agent/case_study.yaml @@ -3,7 +3,12 @@ system_prompt: |- retrieval/ranking, final memory, construction checkpoints, original sessions, and the responsible implementation. At each layer explain what should have happened, what happened, and the supporting or contradicting evidence. Continue to the earliest evidence-supported mechanism; "not found", "ranked too low" and "wrong answer" are symptoms. - Compare plausible explanations and state the smallest missing observation when no cause is established. + Test the strongest alternative before attributing a cause; code/configuration consistent with a failure is only + a hypothesis without runtime evidence linking it to this outcome. Memory existence and search hit counts do not + prove which content reached the answerer. Your own analysis or printed conclusions are not independent evidence. + Distinguish factual recall from applying a user instruction; missing examples alone do not invalidate a rubric. + Leave causes unresolved when evidence is missing: name the remaining explanations and smallest distinguishing + observation. Do not exclude an explanation elsewhere described as unknown. Assigned scope: - Validation: {validation_path} @@ -16,14 +21,18 @@ system_prompt: |- Treat construction partial_success as requiring repair even when answers succeed. Distinguish infrastructure, candidate execution, answer-quality and judging failures without replacing their saved runtime statuses. - Use home_inspect_validation to select anomalous queries and locate result fields and logs. Start query evidence - tracing with home_trace_query_evidence. Respect evidence_source and evidence_is_certain: inferred or uncertain - session annotations are leads, not established facts. Original transcripts and derived memory may both be searchable; + For a selected query start with home_trace_query_evidence; use home_inspect_validation for status or selection. + Respect evidence_source and evidence_is_certain: inferred or uncertain session annotations are leads, not facts. Original transcripts and derived memory may both be searchable; absence from a summary does not prove that the answerer could not retrieve the source session. Use home_inspect_memory_history and home_compare_memory for construction checkpoints and focused diffs. + For a diff, supply distinct base_ref and target_ref from the returned history; path is memory-repository-relative. Use home_list_code_tree, home_search_code and home_read_code_file to examine the evaluated commit; their revision and case-tool scopes are fixed by the host. Follow returned paths for full source details and omitted evidence. + Code-tool paths are candidate-repository-relative; omit path to search/list the root, never use ".". + Read documentation and memory/artifact files with AutoBash or python_execute, not code tools. + A failed/denied read or truncated output cannot establish that information is missing. Keep stderr visible, + use set -o pipefail for shell pipelines, and retry with a suitable reader before drawing a conclusion. AutoBash and python_execute share an OS file sandbox. The case artifacts, source case, validation manifest, home/docs and all of home/code are readable; no host paths are writable. System tools and interpreter libraries @@ -36,7 +45,8 @@ system_prompt: |- Return a concise report, conclusion and strongest evidence first, within approximately {output_token_limit} tokens. Name affected queries and lifecycle stages, explain the causal chain, include contrary evidence and confidence, and separate established causes, hypotheses and unknowns. Cite every error and causal claim with an exact - path:line location or path#key.path for structured JSON. Keep citations tight. Make no unsupported cross-case claims. + path:line location or path#JSON-Pointer for structured JSON; cite source locations, not tool-response fields. + Keep citations tight. Make no unsupported cross-case claims. task_prompt: |- Pay special attention to: {focus} diff --git a/plugins/home/src/reme_home/agent/sandbox.py b/plugins/home/src/reme_home/agent/sandbox.py new file mode 100644 index 00000000..b0defac3 --- /dev/null +++ b/plugins/home/src/reme_home/agent/sandbox.py @@ -0,0 +1,76 @@ +"""Home-specific Git runtime setup around ReMe's unchanged OS file sandboxes.""" + +import os +import subprocess +import sys +from pathlib import Path + +from reme.utils.file_sandbox import create_file_sandbox_backend + + +def _developer_directory(): + """Locate Apple's actual Git runtime without granting its parent directories.""" + if sys.platform != "darwin": + return None + try: + result = subprocess.run( + ["/usr/bin/xcode-select", "-p"], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + if result.returncode == 0 and result.stdout.strip(): + path = Path(result.stdout.strip()).resolve(strict=True) + if (path / "usr/bin/git").is_file() and path != Path("/"): + return str(path) + except (OSError, subprocess.TimeoutExpired, ValueError, RuntimeError): + pass + return None + + +class CaseStudySandbox: + """Delegate execution and cleanup to the OS sandbox, adding only child environment values.""" + + os_name = "posix" + + def __init__(self, backend, environment): + self.backend = backend + self.environment = dict(environment) + + async def getcwd(self): + """Preserve the backend's contained working directory.""" + return await self.backend.getcwd() + + async def exec_shell(self, command, *, cwd=None, timeout=None): + """Set the environment inside isolation using argv, never shell interpolation.""" + if not command: + raise ValueError("command must not be empty") + return await self.backend.exec_shell( + ["/usr/bin/env", *[f"{name}={value}" for name, value in self.environment.items()], *command], + cwd=cwd, + timeout=timeout, + ) + + +def create_case_study_sandbox(*, read_only_paths, **kwargs): + """Explicitly grant Home's Git runtime while preserving core isolation and resource limits.""" + roots = list(read_only_paths) + developer = _developer_directory() + if developer and not any(Path(developer).is_relative_to(root) for root in roots): + roots.append(developer) + binary_paths = [str(Path(sys.prefix) / "bin"), str(Path(sys.base_prefix) / "bin")] + if developer: + binary_paths.append(str(Path(developer) / "usr/bin")) + binary_paths.extend(["/usr/local/bin", "/usr/bin", "/bin"]) + environment = { + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_TERMINAL_PROMPT": "0", + "GIT_OPTIONAL_LOCKS": "0", + "PATH": os.pathsep.join(dict.fromkeys(binary_paths)), + } + if developer: + environment["DEVELOPER_DIR"] = developer + backend = create_file_sandbox_backend(read_only_paths=roots, **kwargs) + return CaseStudySandbox(backend, environment) diff --git a/plugins/home/src/reme_home/agent/wrapper.py b/plugins/home/src/reme_home/agent/wrapper.py index 7586fe80..efb814c2 100644 --- a/plugins/home/src/reme_home/agent/wrapper.py +++ b/plugins/home/src/reme_home/agent/wrapper.py @@ -1,7 +1,11 @@ """ReMe agent wrapper with Home's bounded investigation runtime.""" +import json from typing import Any +from agentscope.message import TextBlock, ToolResultState +from agentscope.tool import FunctionTool, ToolChunk + from reme.components.agent_wrapper import AsAgentWrapper from reme.components.as_llm import BaseAsLLM @@ -31,7 +35,7 @@ class CaseStudyAgentWrapper(BoundedAgentWrapper): rejects an absent investigation model after the judge is validated. """ - def __init__(self, as_llm: str = "judge", **kwargs): + def __init__(self, as_llm: str = "home_case_study", **kwargs): super().__init__(as_llm="", **kwargs) self.home_judge = self.bind("default", HomeJudge, optional=False) self.as_llm = self.bind(as_llm, BaseAsLLM, optional=True) @@ -40,3 +44,22 @@ class CaseStudyAgentWrapper(BoundedAgentWrapper): await super()._start() if self.as_llm is None: raise ValueError("Case-study agent requires a configured as_llm component") + + @classmethod + def _make_tool(cls, job, tool_context_id=None, injected_job_kwargs=None): + """Expose structured investigation evidence as JSON without changing core Job tools.""" + injected = cls._resolve_injected_job_kwargs( + {"tool_context_id": tool_context_id, "injected_job_kwargs": injected_job_kwargs}, + ) + + async def run_job(**kwargs): + response = await job(**cls._merge_injected_job_kwargs(kwargs, injected)) + answer = response.answer + text = json.dumps(answer, ensure_ascii=False) if isinstance(answer, (dict, list)) else str(answer) + state = ToolResultState.SUCCESS if response.success else ToolResultState.ERROR + return ToolChunk(content=[TextBlock(text=text)], state=state) + + tool = FunctionTool(func=run_job, name=job.name, description=job.description, is_concurrency_safe=False) + if parameters := cls._strip_injected_parameters(job.parameters, injected): + tool.input_schema = parameters + return tool diff --git a/plugins/home/src/reme_home/evaluation/evidence_step.py b/plugins/home/src/reme_home/evaluation/evidence_step.py index 54c77fd7..3aee3ab3 100644 --- a/plugins/home/src/reme_home/evaluation/evidence_step.py +++ b/plugins/home/src/reme_home/evaluation/evidence_step.py @@ -65,9 +65,13 @@ def trace_query_evidence(home, spec): if spec.query_id not in manifest.get("query_ids", {}).get(spec.case_id, []): raise ValueError("query_id is not selected by this evaluation manifest") source_path, case = _source_case(home, manifest, spec.case_id) - query = next((item for item in case.get("queries", []) if item.get("query_id") == spec.query_id), None) - if query is None: + query_index = next( + (i for i, item in enumerate(case.get("queries", [])) if item.get("query_id") == spec.query_id), + None, + ) + if query_index is None: raise ValueError(f"Normalized case has no query: {spec.query_id}") + query = case["queries"][query_index] metadata = query.get("metadata") or {} annotated = metadata.get("evidence_session_ids") selected = TypeAdapter(SessionIds).validate_python( @@ -78,8 +82,8 @@ def trace_query_evidence(home, spec): source = "independent_annotation" if annotated is not None else "diagnostician_inference" if annotated is None and spec.evidence_session_ids is None: source = "unavailable" - known = {session["session_id"] for session in case.get("sessions", [])} - if unknown := set(selected) - known: + known = {session["session_id"]: i for i, session in enumerate(case.get("sessions", []))} + if unknown := set(selected) - known.keys(): raise ValueError(f"Evidence sessions are not present in case: {sorted(unknown)}") status, head = _head(repository) @@ -95,26 +99,40 @@ def trace_query_evidence(home, spec): else [] ) files = [name for name in files if name] - row = {"id": session_id, "checkpoint": checkpoint, "changed_files": files[:MAX_CHANGED_FILES]} + row = { + "id": session_id, + "source_pointer": f"/sessions/{known[session_id]}", + "checkpoint": checkpoint, + "changed_files": files[:MAX_CHANGED_FILES], + } if len(files) > MAX_CHANGED_FILES: row["omitted_changed_file_count"] = len(files) - MAX_CHANGED_FILES sessions.append(row) results_path = contained(run, f"cases/{spec.case_id}/queries/result.json") payload = _object(run, str(results_path.relative_to(run))) - result = next( - (item for item in (payload or {}).get("queries", []) if item.get("query_id") == spec.query_id), + result_index = next( + (i for i, item in enumerate((payload or {}).get("queries", [])) if item.get("query_id") == spec.query_id), None, ) - outcome = result or {} + outcome = payload["queries"][result_index] if result_index is not None else {} + rubric = metadata.get("rubric") + expected_is_rubric = manifest.get("dataset") == "beam" and isinstance(query.get("golden_answer"), list) + if rubric is None and expected_is_rubric: + rubric = query["golden_answer"] response = { "query": { "question": _preview(outcome.get("question", query.get("question")), 2000), "expected": _preview(outcome.get("golden_answer", query.get("golden_answer")), 2000), + "expected_kind": "rubric" if expected_is_rubric else "reference_answer", "actual": _preview(outcome.get("answer"), 2000), "score": outcome.get("score"), "error": _preview(outcome.get("error")), - "result_missing": result is None, + "result_missing": result_index is None, + }, + "locations": { + "source_query": f"/queries/{query_index}", + "saved_query": f"/queries/{result_index}" if result_index is not None else None, }, "evidence_source": source, "evidence_is_certain": metadata.get("evidence_is_certain"), @@ -127,6 +145,23 @@ def trace_query_evidence(home, spec): "query_artifacts": str(contained(run, f"cases/{spec.case_id}/queries/{spec.query_id}", directory=True)), }, } + judged = ((outcome.get("judge_result") or {}).get("metadata") or {}).get("llm_judge_responses") + if isinstance(rubric, list): + field = "metadata/rubric" if metadata.get("rubric") is not None else "golden_answer" + response["locations"]["rubric"] = f"/queries/{query_index}/{field}" + if isinstance(rubric, list) and isinstance(judged, list) and len(rubric) == len(judged): + response["criteria"] = [ + { + "requirement": _preview(requirement), + "score": item.get("score"), + "reason": _preview(item.get("reason")), + "pointer": f"/queries/{result_index}/judge_result/metadata/llm_judge_responses/{i}", + } + for i, (requirement, item) in enumerate(zip(rubric[:20], judged[:20])) + if isinstance(item, dict) + ] + if len(rubric) > 20: + response["omitted_criterion_count"] = len(rubric) - 20 if len(selected) > MAX_SESSIONS: response["omitted_session_count"] = len(selected) - MAX_SESSIONS return response diff --git a/plugins/home/src/reme_home/plugin.yaml b/plugins/home/src/reme_home/plugin.yaml index 3d0dff8f..30ea4f95 100644 --- a/plugins/home/src/reme_home/plugin.yaml +++ b/plugins/home/src/reme_home/plugin.yaml @@ -19,10 +19,23 @@ backends: application_defaults: components: + as_llm: + home_case_study: + backend: ${LLM_BACKEND:-openai} + model: qwen3.7-max + stream: false + context_size: 200000 + max_retries: 3 + credential: + api_key: ${LLM_API_KEY:-} + base_url: ${LLM_BASE_URL:-} + parameters: + max_tokens: 65536 + thinking_enable: false agent_wrapper: home_case_study: backend: home_case_study_agent - as_llm: judge + as_llm: home_case_study time_budget_minutes: 30 react_config: max_iters: 60 @@ -53,15 +66,29 @@ application_defaults: jobs: home_case_study: backend: base - description: "Investigate one validation case with read-only sandboxed Bash/Python and commit-pinned evidence tools; return an evidence-supported diagnostic report" + description: "Investigate up to eight ordered evaluation cases sequentially with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error" parameters: type: object additionalProperties: false - required: [validation_path, case_id] + allOf: + - oneOf: + - required: [evaluation_path] + - required: [validation_path] + - oneOf: + - required: [case_ids] + - required: [case_id] properties: - validation_path: {type: string, minLength: 1} - case_id: {type: string, minLength: 1} - focus: {type: string, minLength: 1, description: "Questions or failure mechanisms to prioritize"} + evaluation_path: {type: string, minLength: 1, description: "Shared evaluation result directory"} + case_ids: + type: array + minItems: 1 + maxItems: 8 + uniqueItems: true + items: {type: string, minLength: 1} + description: "Ordered case IDs; results preserve this order, with one isolated agent per case" + focus: {type: string, minLength: 1, description: "Shared diagnostic request applied unchanged to every selected case"} + validation_path: {type: string, minLength: 1, description: "Deprecated alias for evaluation_path; do not supply both"} + case_id: {type: string, minLength: 1, description: "Legacy single-case selector returning a text report; do not combine with case_ids"} steps: - backend: home_case_study_step agent_wrapper: home_case_study diff --git a/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md b/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md index ff6ae516..ce1ac525 100644 --- a/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md +++ b/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md @@ -199,7 +199,12 @@ Start a failed-query investigation with `home_trace_query_evidence(validation_path, case_id, query_id)`. It returns compact question/answer/score/error previews, source evidence session IDs, matching Git checkpoints, changed files and paths to complete evidence. It -does not call a model or include transcripts, diffs or judge-result bodies. +does not call a model or include transcripts, diffs or full judge-result bodies. +`locations` contains JSON Pointers into the source case and saved query result; +each session's `source_pointer` locates it in the source case. `query.expected_kind` +distinguishes reference answers from BEAM rubric lists. Optional `criteria` pairs +saved rubric judgments only when counts match (up to 20, with 600-character +requirement/reason previews); missing judgments are never inferred. Dataset `evidence_session_ids` annotations take precedence; only when absent may you supply inferred `evidence_session_ids`. Respect `evidence_source` and `evidence_is_certain`: inferred or uncertain sessions are leads, not established @@ -238,10 +243,15 @@ inspection paths. ## Single-case diagnostic agent -Call `home_case_study(validation_path, case_id, focus=...)` to investigate exactly -one case. It returns a text report with evidence citations and uncertainty, using -fresh agent state, bounded iterations and advisory time reminders. There is no -parallel case dispatcher. Scope and evaluated code revision are host-injected +Call `home_case_study(evaluation_path, case_ids, focus=...)` for up to eight +unique cases sharing one focus and evaluation. Cases run sequentially and results +preserve input order. Each gets fresh agent state, bounded iterations and advisory +time reminders. The answer contains `results`, with a report/session/usage or error +for each case; a failed case does not cancel the rest. Response success is false +if any case fails. Metadata includes `evaluation_path`, evaluated commit and total +usage (null if any call fails or usage is unavailable). `validation_path` remains +a path alias; legacy `case_id` calls retain a text report. Do not combine aliases. +Other inspection tools still use `validation_path`. There is no parallel case dispatcher. Scope and evaluated code revision are host-injected into the structured tools. AutoBash and Python share a mandatory read-only file sandbox covering the selected case artifacts, source JSON, validation manifest, home/docs and all home/code, plus required system runtimes. Neither executor has @@ -250,5 +260,16 @@ and Python rather than a static command whitelist. The directly readable code worktree can differ from the evaluated commit; use pinned code tools to verify version-sensitive claims. No write/edit, validation or code-mutation tools are exposed. Sandbox startup failure prevents model execution; it never falls back -to an unrestricted process. See the Home README for model, budget and platform +to an unrestricted process. Startup checks Python, source/manifest reads and +`git show` at the evaluated commit before spending model tokens. macOS grants +read-only access to the selected Xcode/Command Line Tools runtime and sets +`DEVELOPER_DIR`; Linux keeps Bubblewrap isolation and searches the virtual +environment and base interpreter's `bin` directories for tools. + +Code tools accept candidate-repository-relative paths, not absolute artifact or +documentation paths. Omit `path` to list/search the root. Read logs, source JSON, +memory and home/docs with AutoBash or Python. Memory diffs require `base_ref` +and a different `target_ref` from the memory history. Keep stderr visible and use +`set -o pipefail` for shell pipelines: a failed/denied read is not evidence of +missing content. See the Home README for model, budget and platform sandbox configuration. diff --git a/plugins/home/tests/test_case_study.py b/plugins/home/tests/test_case_study.py index 05d12c4a..466e63a7 100644 --- a/plugins/home/tests/test_case_study.py +++ b/plugins/home/tests/test_case_study.py @@ -5,14 +5,18 @@ import asyncio import json +import shutil import sys from types import SimpleNamespace import pytest +from jsonschema import Draft202012Validator +from pydantic import ValidationError from agentscope.message import TextBlock, ToolCallBlock from agentscope.model import ChatResponse, ChatUsage from agentscope.tool import ExecResult from reme_home.agent import case_study +from reme_home.agent.sandbox import create_case_study_sandbox from reme_home.evaluation.artifacts import write_json from reme_home.preparation.code import git @@ -26,7 +30,6 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import from test_evaluation_inspection import write_run, snapshot from reme.utils.local_backend import LocalProcessBackend -from reme.utils.file_sandbox import create_file_sandbox_backend class RecordingSandbox: @@ -52,7 +55,7 @@ def sandbox(monkeypatch): instances.append(backend) return backend - monkeypatch.setattr(case_study, "create_file_sandbox_backend", create) + monkeypatch.setattr(case_study, "create_case_study_sandbox", create) return instances @@ -95,6 +98,16 @@ class InvestigationModel(RecordingModel): return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0)) +@pytest.mark.asyncio +async def test_case_study_uses_dedicated_model_without_changing_judge(app): + models = app.context.components["as_llm"] + wrapper = app.context.components["agent_wrapper"]["home_case_study"] + assert wrapper.as_llm is models["home_case_study"] + assert wrapper.as_llm.kwargs["model"] == "qwen3.7-max" + assert app.context.components["agent_wrapper"]["judge"].as_llm is models["judge"] + assert models["home_case_study"] is not models["judge"] + + @pytest.mark.asyncio async def test_single_run_real_agent_tools_scope_and_no_scratch(app, prepared, case_run, sandbox): run, case_id, query_id, commit, source = case_run @@ -113,6 +126,9 @@ async def test_single_run_real_agent_tools_scope_and_no_scratch(app, prepared, c assert "Evidence-supported report" in response.answer assert response.metadata["candidate_commit"] == commit and response.metadata["usage"]["total_tokens"] == 25 assert response.metadata["session_id"] + results = [b for m in model.calls[1]["messages"] for b in m.content if b.type == "tool_result"] + evidence_result = json.loads(results[0].output[0].text) + assert evidence_result["locations"]["saved_query"] == "/queries/0" assert "value = 1" in str(model.calls[2]["messages"]) assert "value = 999" not in str(model.calls[2]["messages"]) schemas = {item["function"]["name"]: item["function"]["parameters"] for item in model.calls[0]["tools"]} @@ -121,6 +137,13 @@ async def test_single_run_real_agent_tools_scope_and_no_scratch(app, prepared, c assert "validation_path" not in schemas["home_inspect_validation"]["properties"] assert "revision" not in schemas["home_read_code_file"]["properties"] assert "target_validation_path" not in schemas["home_compare_memory"]["properties"] + assert "base_ref" in schemas["home_compare_memory"]["required"] + assert schemas["home_compare_memory"]["properties"]["base_ref"]["type"] == "string" + descriptions = {item["function"]["name"]: item["function"]["description"] for item in model.calls[0]["tools"]} + assert "repository-relative" in descriptions["home_read_code_file"] + assert "python_execute" in descriptions["home_read_code_file"] + assert "pipefail" in descriptions["AutoBash"] + assert "do not pass args" in descriptions["python_execute"] policy = sandbox[0].policy assert policy["writable_paths"] == [] and policy["cwd"] == str(run / f"cases/{case_id}") assert str(prepared / "code") in policy["read_only_paths"] and str(source) in policy["read_only_paths"] @@ -140,7 +163,9 @@ async def test_scope_override_rejected_by_real_job_tool(app, prepared, case_run, run, case_id, _query_id, _commit, _source = case_run wrapper = app.context.components["agent_wrapper"]["home_case_study"] scope = case_study.prepare_scope(prepared, case_study.CaseStudyInput(validation_path=str(run), case_id=case_id)) + job_parameters = json.dumps(app.context.jobs["home_inspect_memory_history"].parameters, sort_keys=True) toolkit = case_study.create_toolkit(wrapper, scope, RecordingSandbox()) + assert json.dumps(app.context.jobs["home_inspect_memory_history"].parameters, sort_keys=True) == job_parameters schemas = await toolkit.get_tool_schemas() assert len(schemas) == 9 tool = wrapper._make_tool( @@ -181,7 +206,7 @@ async def test_invalid_scope_and_sandbox_failure_precede_llm(app, prepared, case def unavailable(**kwargs): raise RuntimeError("No sandbox backend") - monkeypatch.setattr(case_study, "create_file_sandbox_backend", unavailable) + monkeypatch.setattr(case_study, "create_case_study_sandbox", unavailable) response = await app.run_job("home_case_study", validation_path=str(run), case_id=case_id) assert not response.success and "No sandbox backend" in str(response.answer) and not model.calls @@ -207,9 +232,13 @@ async def test_os_sandbox_read_scope_and_no_writes(tmp_path): case.mkdir() code.mkdir() (code / "source.py").write_text("visible code") + git(code, "init") + git(code, "add", "source.py") + git(code, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "initial") outside = tmp_path / "secret.txt" outside.write_text("outside scope") - backend = create_file_sandbox_backend( + (case / "manifest.json").write_text("{}") + backend = create_case_study_sandbox( cwd=str(case), read_only_paths=[str(case), str(code)], writable_paths=[], @@ -221,9 +250,24 @@ async def test_os_sandbox_read_scope_and_no_writes(tmp_path): pytest.skip(f"OS sandbox unavailable on this host: {error}") if not probe.ok(): pytest.skip("OS sandbox cannot start on this host: " + probe.stderr.decode(errors="replace")) + ready = await case_study.probe_sandbox( + backend, + { + "case_path": str(case), + "source_case": str(code / "source.py"), + "validation_path": str(case), + "code_repository": str(code), + "candidate_commit": git(code, "rev-parse", "HEAD"), + }, + ) + assert ready.ok(), ready.stderr.decode(errors="replace") tool = case_study.python_tool(backend, str(case)) result = await tool._func(code=f"from pathlib import Path; print(Path({str(code / 'source.py')!r}).read_text())") assert result["returncode"] == 0 and "visible code" in result["stdout"] + result = await tool._func( + code=f"import subprocess; subprocess.run(['git', '-C', {str(code)!r}, 'show', 'HEAD:source.py'], check=True)", + ) + assert result["returncode"] == 0 and "visible code" in result["stdout"], result for script in ( f"open({str(outside)!r}).read()", "open('written.txt', 'w').write('denied')", @@ -237,6 +281,17 @@ async def test_os_sandbox_read_scope_and_no_writes(tmp_path): chunks = [chunk async for chunk in bash.call("echo changed > written.txt")] assert "Command failed" in str(chunks) assert not (case / "written.txt").exists() and (code / "source.py").read_text() == "visible code" + chunks = [chunk async for chunk in bash.call(f"git -C '{code}' show HEAD:source.py")] + assert "visible code" in str(chunks), chunks + git_config = (code / ".git/config").read_bytes() + result = await tool._func( + code=( + f"import subprocess; subprocess.run(['git', '-C', {str(code)!r}," + " 'config', 'test.blocked', 'yes'], check=True)" + ), + ) + assert result["returncode"] != 0 + assert (code / ".git/config").read_bytes() == git_config @pytest.mark.asyncio @@ -265,3 +320,154 @@ async def test_python_code_file_timeout_and_cancellation(tmp_path): ): with pytest.raises(ValueError): await tool._func(**arguments) + + +@pytest.fixture +def batch_run(case_run): + run, case_id, query_id, commit, source = case_run + case = json.loads(source.read_text()) + case["case_id"] = "case2" + write_json(source.parent / "batch-second.json", case) + shutil.copytree(run / f"cases/{case_id}", run / "cases/case2") + manifest = json.loads((run / "manifest.json").read_text()) + manifest["case_ids"].append("case2") + manifest["query_ids"]["case2"] = [query_id] + write_json(run / "manifest.json", manifest) + return run, case_id, query_id, commit + + +class BatchModel(RecordingModel): + """Read each fresh agent's case-local evidence before returning its report.""" + + def __init__(self, query_id): + super().__init__() + self.query_id = query_id + + async def __call__(self, **kwargs): + self.calls.append(kwargs) + results = [b for m in kwargs["messages"] for b in m.content if b.type == "tool_result"] + if not results: + content = [ + ToolCallBlock( + id="batch-trace", + name="home_trace_query_evidence", + input=json.dumps({"query_id": self.query_id}), + ), + ] + else: + content = [TextBlock(text="case-local report")] + return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0)) + + +@pytest.mark.asyncio +async def test_batch_shares_focus_but_isolates_agents_tools_and_sandboxes(app, prepared, batch_run, sandbox): + run, first, query_id, commit = batch_run + model = BatchModel(query_id) + app.context.components["agent_wrapper"]["home_case_study"].as_llm = SimpleNamespace(model=model) + before = snapshot(prepared) + response = await app.run_job( + "home_case_study", + evaluation_path=str(run), + case_ids=["case2", first], + focus="Shared hypothesis", + ) + assert response.success, response.answer + results = response.answer["results"] + assert [item["case_id"] for item in results] == ["case2", first] + assert all(item["report"] == "case-local report" for item in results) + assert len({item["session_id"] for item in results}) == 2 + assert response.metadata["evaluation_path"] == str(run) and response.metadata["candidate_commit"] == commit + assert response.metadata["usage"]["total_tokens"] == 20 + assert len(model.calls) == 4 + for i, case_id in enumerate(["case2", first]): + calls = model.calls[i * 2 : i * 2 + 2] + assert "Shared hypothesis" in str(calls[0]["messages"]) + outputs = [b for m in calls[1]["messages"] for b in m.content if b.type == "tool_result"] + evidence = json.loads(outputs[0].output[0].text) + assert evidence["paths"]["query_result"] == str(run / f"cases/{case_id}/queries/result.json") + assert sandbox[i].policy["cwd"] == str(run / f"cases/{case_id}") + other = first if case_id == "case2" else "case2" + assert str(run / f"cases/{other}") not in sandbox[i].policy["read_only_paths"] + assert snapshot(prepared) == before + + +@pytest.mark.asyncio +async def test_batch_continues_after_case_failure_and_keeps_usage_unknown(app, batch_run, sandbox, monkeypatch): + run, first, query_id, _commit = batch_run + model = BatchModel(query_id) + app.context.components["agent_wrapper"]["home_case_study"].as_llm = SimpleNamespace(model=model) + original = RecordingSandbox.exec_shell + + async def fail_second_case(self, command, *, cwd, timeout): + if str(cwd).endswith("/case2"): + return ExecResult(exit_code=1, stdout=b"", stderr=b"sandbox denied") + return await original(self, command, cwd=cwd, timeout=timeout) + + monkeypatch.setattr(RecordingSandbox, "exec_shell", fail_second_case) + response = await app.run_job("home_case_study", validation_path=str(run), case_ids=["case2", first]) + assert not response.success + failed, succeeded = response.answer["results"] + assert failed["case_id"] == "case2" and "sandbox unavailable" in failed["error"] + assert succeeded["case_id"] == first and succeeded["success"] and succeeded["session_id"] + assert response.metadata["usage"] is None + + +@pytest.mark.asyncio +async def test_entire_batch_is_validated_before_spending_model_tokens(app, batch_run, sandbox): + run, first, _query_id, _commit = batch_run + response = await app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "missing"]) + assert not response.success and "not selected" in response.answer and not sandbox + + +@pytest.mark.asyncio +async def test_batch_cancellation_propagates_without_starting_next_case(app, batch_run, sandbox, monkeypatch): + run, first, _query_id, _commit = batch_run + + async def cancel(*args, **kwargs): + raise asyncio.CancelledError + + monkeypatch.setattr(app.context.components["agent_wrapper"]["home_case_study"], "reply", cancel) + with pytest.raises(asyncio.CancelledError): + await app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "case2"]) + assert len(sandbox) == 1 + + +@pytest.mark.parametrize( + "updates", + [ + {"case_ids": []}, + {"case_ids": ["case1", "case1"]}, + {"case_ids": ["../escape"]}, + {"case_ids": [str(i) for i in range(9)]}, + {"case_ids": "case1"}, + {"case_ids": [1]}, + {"validation_path": "another"}, + {"case_id": "case1"}, + ], +) +def test_batch_request_rejects_invalid_or_ambiguous_inputs(updates): + with pytest.raises(ValidationError): + case_study.CaseStudyBatchInput.model_validate( + {"evaluation_path": "evaluations/a/b/1", "case_ids": ["case1"], **updates}, + ) + + +@pytest.mark.parametrize("path_key", ["evaluation_path", "validation_path"]) +@pytest.mark.parametrize("case_key", ["case_ids", "case_id"]) +@pytest.mark.asyncio +async def test_new_and_legacy_selectors_match_job_schema(app, batch_run, sandbox, path_key, case_key): + run, first, query_id, _commit = batch_run + app.context.components["agent_wrapper"]["home_case_study"].as_llm = SimpleNamespace(model=BatchModel(query_id)) + arguments = {path_key: str(run), case_key: [first] if case_key == "case_ids" else first} + schema = app.context.jobs["home_case_study"].parameters + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema) + assert validator.is_valid(arguments) + assert not validator.is_valid({**arguments, "evaluation_path": str(run), "validation_path": str(run)}) + assert not validator.is_valid({**arguments, "case_ids": [first], "case_id": first}) + response = await app.run_job("home_case_study", **arguments) + assert response.success, response.answer + if case_key == "case_id": + assert response.answer == "case-local report" and response.metadata["case_id"] == first + else: + assert response.answer["results"][0]["case_id"] == first diff --git a/plugins/home/tests/test_case_study_sandbox.py b/plugins/home/tests/test_case_study_sandbox.py new file mode 100644 index 00000000..a1fe57ad --- /dev/null +++ b/plugins/home/tests/test_case_study_sandbox.py @@ -0,0 +1,96 @@ +"""Home runtime adaptation preserves the core sandbox's path and process contracts.""" + +# pylint: disable=missing-function-docstring,protected-access + +import asyncio +import os +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from reme_home.agent import sandbox +from reme.utils.file_sandbox import PathBubbleBackend, PathSeatbeltBackend + + +def test_macos_grants_only_selected_developer_runtime(tmp_path, monkeypatch): + developer = tmp_path / "developer tools" + (developer / "usr/bin").mkdir(parents=True) + (developer / "usr/bin/git").touch() + workspace = tmp_path / "case" + workspace.mkdir() + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr( + sandbox.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=str(developer)), + ) + monkeypatch.setattr(sandbox, "create_file_sandbox_backend", PathSeatbeltBackend) + roots = [str(workspace)] + adapted = sandbox.create_case_study_sandbox( + cwd=workspace, + read_only_paths=roots, + writable_paths=[], + memory_limit_mib=0, + ) + assert roots == [str(workspace)] + profile = adapted.backend._profile(str(workspace / "temporary")) + assert f'(allow file-read* (subpath "{developer}"))' in profile + assert f'(allow file-write* (subpath "{developer}"))' not in profile + assert f'(allow file-read* (subpath "{tmp_path}"))' not in profile + assert adapted.environment["DEVELOPER_DIR"] == str(developer) + paths = adapted.environment["PATH"].split(os.pathsep) + assert paths[0] == str(sandbox.Path(sys.prefix) / "bin") + assert paths.index(str(developer / "usr/bin")) < paths.index("/usr/bin") + + +@pytest.mark.parametrize("result", [(1, ""), (0, ""), (0, "/nonexistent/reme-toolchain")]) +def test_failed_macos_lookup_adds_no_grant(tmp_path, monkeypatch, result): + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr( + sandbox.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=result[0], stdout=result[1]), + ) + + def factory(**kwargs): + return SimpleNamespace(policy=kwargs) + + monkeypatch.setattr(sandbox, "create_file_sandbox_backend", factory) + adapted = sandbox.create_case_study_sandbox(cwd=tmp_path, read_only_paths=[str(tmp_path)], writable_paths=[]) + assert adapted.backend.policy["read_only_paths"] == [str(tmp_path)] + assert "DEVELOPER_DIR" not in adapted.environment + + +@pytest.mark.asyncio +async def test_linux_environment_is_inside_bubblewrap(tmp_path, monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(sandbox, "create_file_sandbox_backend", PathBubbleBackend) + adapted = sandbox.create_case_study_sandbox( + cwd=tmp_path, + read_only_paths=[str(tmp_path)], + writable_paths=[], + memory_limit_mib=0, + ) + backend = adapted.backend + backend.exec_shell = AsyncMock(return_value="result") + command = [sys.executable, "-I", "-B", "-c", "print('literal $HOME; `pwd`')"] + assert await adapted.exec_shell(command, cwd=str(tmp_path), timeout=7) == "result" + args, kwargs = backend.exec_shell.call_args + assert kwargs == {"cwd": str(tmp_path), "timeout": 7} + assert args[0][-len(command) :] == command + argv = backend._bwrap_argv(args[0], str(tmp_path)) + assert "--clearenv" in argv and "--unshare-all" in argv + assert argv.index("--") < argv.index("/usr/bin/env") + assert "GIT_CONFIG_NOSYSTEM=1" in argv and "GIT_CONFIG_GLOBAL=/dev/null" in argv + assert adapted.environment["PATH"].split(":")[:2] == list( + dict.fromkeys([str(sandbox.Path(sys.prefix) / "bin"), str(sandbox.Path(sys.base_prefix) / "bin")]), + ) + assert backend._writable_paths == [] + assert await adapted.getcwd() == str(tmp_path) + backend.exec_shell.side_effect = asyncio.CancelledError + with pytest.raises(asyncio.CancelledError): + await adapted.exec_shell(command) + with pytest.raises(ValueError, match="empty"): + await adapted.exec_shell([]) diff --git a/plugins/home/tests/test_query_evidence.py b/plugins/home/tests/test_query_evidence.py index e77e6929..8076f813 100644 --- a/plugins/home/tests/test_query_evidence.py +++ b/plugins/home/tests/test_query_evidence.py @@ -63,6 +63,7 @@ def test_compact_annotation_root_checkpoint_and_read_only(query_evidence): assert result["query"] == { "question": "What is the fact?", "expected": "old fact", + "expected_kind": "reference_answer", "actual": "new fact", "score": 0, "error": None, @@ -71,10 +72,17 @@ def test_compact_annotation_root_checkpoint_and_read_only(query_evidence): assert result["evidence_source"] == "independent_annotation" assert result["evidence_is_certain"] is False and result["memory_status"] == "available" assert result["sessions"] == [ - {"id": "session1", "checkpoint": first, "changed_files": ["memory/note.md", "removed.txt"]}, + { + "id": "session1", + "source_pointer": "/sessions/0", + "checkpoint": first, + "changed_files": ["memory/note.md", "removed.txt"], + }, ] assert result["paths"]["source_case"] == str(source) assert result["paths"]["memory_repository"] == str(repo) + assert result["locations"] == {"source_query": "/queries/0", "saved_query": "/queries/0"} + assert "criteria" not in result assert "source transcript" not in json.dumps(result) and "judge_result" not in json.dumps(result) assert snapshot(home) == before @@ -94,7 +102,12 @@ def test_inference_missing_checkpoint_and_debug_dataset(query_evidence): assert result["paths"]["source_case"] == str(debug_source) assert result["sessions"][0]["checkpoint"] == second assert result["sessions"][0]["changed_files"] == ["added.bin", "memory/note.md", "removed.txt"] - assert result["sessions"][1] == {"id": "not-constructed", "checkpoint": None, "changed_files": []} + assert result["sessions"][1] == { + "id": "not-constructed", + "source_pointer": "/sessions/2", + "checkpoint": None, + "changed_files": [], + } assert trace(home, run)["evidence_source"] == "unavailable" with pytest.raises(ValueError, match="not present"): trace(home, run, evidence_session_ids=["unknown"]) @@ -122,6 +135,7 @@ def test_missing_history_and_results(query_evidence, state): (run / "cases/case1/queries/result.json").unlink() result = trace(home, run) assert result["memory_status"] == state and result["sessions"][0]["checkpoint"] is None + assert result["locations"]["saved_query"] is None and "criteria" not in result assert result["query"]["result_missing"] and result["query"]["score"] is None assert result["query"]["actual"] is None and result["query"]["expected"] == "old fact" @@ -219,3 +233,59 @@ async def test_registered_job_and_restricted_scope(app, prepared, query_evidence ) assert not response.success and "unrestricted" in str(response.answer) assert snapshot(prepared) == before + + +@pytest.mark.parametrize("count", [2, 1, 25]) +def test_rubric_judgments_and_independent_source_pointers(query_evidence, count): + home, run, _repo, _first, _second, source, case = query_evidence + case["queries"].insert(0, {"query_id": "other"}) + case["sessions"].insert(0, {"session_id": "other-session"}) + case["queries"][1]["metadata"]["rubric"] = ["required" + str(i) for i in range(25 if count == 25 else 2)] + write_json(source, case) + result_path = run / "cases/case1/queries/result.json" + judged = [{"score": 0.5, "reason": "r" * 1000} for _ in range(count)] + write_json( + result_path, + { + "queries": [ + { + "query_id": "q1", + "judge_result": { + "metadata": {"llm_judge_responses": judged}, + }, + }, + ], + }, + ) + result = trace(home, run) + assert result["locations"] == { + "source_query": "/queries/1", + "saved_query": "/queries/0", + "rubric": "/queries/1/metadata/rubric", + } + assert result["sessions"][0]["source_pointer"] == "/sessions/1" + if count == 1: + assert "criteria" not in result # Do not silently pair incomplete judgments. + else: + assert len(result["criteria"]) == min(count, 20) + item = result["criteria"][0] + assert item["requirement"] == "required0" and len(item["reason"]) < 700 + value = json.loads(result_path.read_text()) + for segment in item["pointer"].strip("/").split("/"): + value = value[int(segment)] if isinstance(value, list) else value[segment] + assert value == judged[0] + if count == 25: + assert result["omitted_criterion_count"] == 5 + + +def test_beam_rubric_fallback_uses_explicit_dataset_contract(query_evidence): + home, run, _repo, _first, _second, source, case = query_evidence + case["queries"][0]["golden_answer"] = ["required fact"] + write_json(source, case) + manifest = json.loads((run / "manifest.json").read_text()) + manifest["dataset"] = "beam" + write_json(run / "manifest.json", manifest) + result = trace(home, run) + assert result["query"]["expected_kind"] == "rubric" + assert result["locations"]["rubric"] == "/queries/0/golden_answer" + assert "criteria" not in result # A rubric is not a saved judgment.