From 91b7c14df44ebf9c83cfbc0dcce3373589d8f187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E7=89=9B=E8=82=89?= Date: Thu, 29 Oct 2026 16:57:58 +0800 Subject: [PATCH] feat(home): migrate agent, sampling and evidence inspection tooling --- benchmark/beam/README.md | 6 + benchmark/beam/README_ZH.md | 5 + benchmark/beam/config_home.yaml | 8 +- benchmark/home_runner.py | 15 +- benchmark/longmemeval/README.md | 6 + benchmark/longmemeval/README_ZH.md | 5 + benchmark/longmemeval/config_home.yaml | 8 +- deploy/benchmark_docker/docker_sandbox.py | 6 + plugins/home/README.md | 391 +++++++++++++++++ plugins/home/src/reme_home/agent/__init__.py | 6 + plugins/home/src/reme_home/agent/bounded.py | 308 +++++++++++++ plugins/home/src/reme_home/agent/wrapper.py | 21 + plugins/home/src/reme_home/code/__init__.py | 1 + plugins/home/src/reme_home/code/component.py | 72 +++ plugins/home/src/reme_home/code/models.py | 68 +++ .../home/src/reme_home/code/regex_worker.py | 84 ++++ plugins/home/src/reme_home/code/repository.py | 213 +++++++++ plugins/home/src/reme_home/code/snapshot.py | 143 ++++++ plugins/home/src/reme_home/code/step.py | 22 + .../src/reme_home/evaluation/component.py | 96 +++- .../home/src/reme_home/evaluation/history.py | 345 +++++++++++++++ .../home/src/reme_home/evaluation/inputs.py | 55 ++- .../reme_home/evaluation/inspection_step.py | 358 +++++++++++++++ .../evaluation/memory_inspection_step.py | 284 ++++++++++++ plugins/home/src/reme_home/evaluation/step.py | 4 +- plugins/home/src/reme_home/plugin.yaml | 304 ++++++++++++- .../home/src/reme_home/preparation/code.py | 23 +- .../runtime_docs/architecture-guide.md | 16 + .../runtime_docs/validation-artifact-guide.md | 81 ++++ plugins/home/src/reme_home/sampling.py | 169 +++++++ plugins/home/tests/test_benchmark_runner.py | 38 ++ plugins/home/tests/test_bounded_agent.py | 351 +++++++++++++++ plugins/home/tests/test_bounded_wrapper.py | 210 +++++++++ plugins/home/tests/test_code.py | 316 ++++++++++++++ plugins/home/tests/test_evaluation.py | 4 + plugins/home/tests/test_evaluation_history.py | 360 +++++++++++++++ .../home/tests/test_evaluation_inspection.py | 312 +++++++++++++ plugins/home/tests/test_memory_inspection.py | 230 ++++++++++ plugins/home/tests/test_sampling.py | 412 ++++++++++++++++++ .../agent_wrapper/as_agent_wrapper.py | 8 +- .../test_benchmark_docker_start_callback.py | 84 ++++ 41 files changed, 5428 insertions(+), 20 deletions(-) create mode 100644 plugins/home/src/reme_home/agent/__init__.py create mode 100644 plugins/home/src/reme_home/agent/bounded.py create mode 100644 plugins/home/src/reme_home/agent/wrapper.py create mode 100644 plugins/home/src/reme_home/code/__init__.py create mode 100644 plugins/home/src/reme_home/code/component.py create mode 100644 plugins/home/src/reme_home/code/models.py create mode 100644 plugins/home/src/reme_home/code/regex_worker.py create mode 100644 plugins/home/src/reme_home/code/repository.py create mode 100644 plugins/home/src/reme_home/code/snapshot.py create mode 100644 plugins/home/src/reme_home/code/step.py create mode 100644 plugins/home/src/reme_home/evaluation/history.py create mode 100644 plugins/home/src/reme_home/evaluation/inspection_step.py create mode 100644 plugins/home/src/reme_home/evaluation/memory_inspection_step.py create mode 100644 plugins/home/src/reme_home/sampling.py create mode 100644 plugins/home/tests/test_bounded_agent.py create mode 100644 plugins/home/tests/test_bounded_wrapper.py create mode 100644 plugins/home/tests/test_code.py create mode 100644 plugins/home/tests/test_evaluation_history.py create mode 100644 plugins/home/tests/test_evaluation_inspection.py create mode 100644 plugins/home/tests/test_memory_inspection.py create mode 100644 plugins/home/tests/test_sampling.py create mode 100644 tests/unit/test_benchmark_docker_start_callback.py diff --git a/benchmark/beam/README.md b/benchmark/beam/README.md index 3c8a2620..8834c85f 100644 --- a/benchmark/beam/README.md +++ b/benchmark/beam/README.md @@ -158,6 +158,12 @@ the host application loads `home`. `source_ref: HEAD` exports committed source only, excluding uncommitted modifications. Preparation failure rolls back its outputs; after success, another call fails instead of resetting the experiment. +The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`, +`max_queries` (null means unlimited), `temperature`, `ewma_alpha` and +`uniform_exploration`. The runner validates these and passes them to the host +`home.evaluation` Component. It still only prepares; it does not run evaluations. +Each evaluation controls `sampling` and `record_history` separately. + Python callers can use the same lifecycle: ```python diff --git a/benchmark/beam/README_ZH.md b/benchmark/beam/README_ZH.md index f0f8eeb7..44cd7a93 100644 --- a/benchmark/beam/README_ZH.md +++ b/benchmark/beam/README_ZH.md @@ -149,4 +149,9 @@ python benchmark/beam/run_home.py 准备失败会回滚;成功后禁止再次准备同一工作区。 Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。 +顶层 `sampling_policy` 配置采样默认值:`max_cases`、`max_queries`(null 表示不限)、 +`temperature`、`ewma_alpha` 和 `uniform_exploration`。入口校验后将其传给宿主 +`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling` +和 `record_history`。 + 默认选择 BEAM **100K 全部 case**。 diff --git a/benchmark/beam/config_home.yaml b/benchmark/beam/config_home.yaml index 3ce562bc..5f00d4cc 100644 --- a/benchmark/beam/config_home.yaml +++ b/benchmark/beam/config_home.yaml @@ -1,7 +1,13 @@ # Preparation only. Paths are relative to the ReMe repository root, not the cwd. workspace_dir: benchmark/beam/workspaces/home-100K +# Policy defaults for Home evaluation; each call still controls sampling=true/false. +sampling_policy: + max_cases: 5 + max_queries: null + temperature: 1.0 + ewma_alpha: 0.3 + uniform_exploration: 0.2 preparation: - schema_version: 1 benchmark: name: beam source: benchmark/beam/dataset/BEAM diff --git a/benchmark/home_runner.py b/benchmark/home_runner.py index 85ded77b..e4c2a23e 100644 --- a/benchmark/home_runner.py +++ b/benchmark/home_runner.py @@ -9,6 +9,7 @@ from pydantic import BaseModel, ConfigDict, Field import yaml from reme_home.preparation.spec import PreparationSpec from reme_home.judge import JUDGE_PLUGINS +from reme_home.sampling import SamplingPolicy from reme.application import Application @@ -21,6 +22,7 @@ class RunHomeConfig(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) workspace_dir: str = Field(min_length=1) preparation: dict + sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy) def project_path(value: str, project_root: Path) -> Path: @@ -30,7 +32,7 @@ def project_path(value: str, project_root: Path) -> Path: return (project_root / value).resolve() -def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): +def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): """Resolve source paths and validate explicit Home preparation parameters.""" raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) config = RunHomeConfig.model_validate(raw) @@ -40,15 +42,22 @@ def load_config(config_path: str | Path, *, workspace_dir: str | None = None, pr spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root)) spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root)) workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root) - return workspace, spec.model_dump(mode="json") + return workspace, spec.model_dump(mode="json"), config.sampling_policy + + +def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): + """Return resolved preparation inputs, preserving the preparation-only caller contract.""" + workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root) + return workspace, spec async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict: """Start Home, prepare once, and close all resources without evaluating cases.""" - workspace, spec = load_config(config_path, workspace_dir=workspace_dir) + workspace, spec, policy = _load_config(config_path, workspace_dir=workspace_dir) app = Application( workspace_dir=str(workspace), plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]], + components={"home.evaluation": {"default": {"sampling_policy": policy.model_dump(mode="json")}}}, service={"backend": "cli"}, enable_logo=False, log_to_file=False, diff --git a/benchmark/longmemeval/README.md b/benchmark/longmemeval/README.md index 9232c3dc..61e02b6a 100644 --- a/benchmark/longmemeval/README.md +++ b/benchmark/longmemeval/README.md @@ -131,6 +131,12 @@ the host application loads `home`. `source_ref: HEAD` exports committed source only, excluding uncommitted modifications. Preparation failure rolls back its outputs; after success, another call fails instead of resetting the experiment. +The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`, +`max_queries` (null means unlimited), `temperature`, `ewma_alpha` and +`uniform_exploration`. The runner validates these and passes them to the host +`home.evaluation` Component. It still only prepares; it does not run evaluations. +Each evaluation controls `sampling` and `record_history` separately. + Python callers can use the same lifecycle: ```python diff --git a/benchmark/longmemeval/README_ZH.md b/benchmark/longmemeval/README_ZH.md index 57146af1..85e38bf4 100644 --- a/benchmark/longmemeval/README_ZH.md +++ b/benchmark/longmemeval/README_ZH.md @@ -120,6 +120,11 @@ python benchmark/longmemeval/run_home.py 准备失败会回滚;成功后禁止再次准备同一工作区。 Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=...)`。 +顶层 `sampling_policy` 配置采样默认值:`max_cases`、`max_queries`(null 表示不限)、 +`temperature`、`ewma_alpha` 和 `uniform_exploration`。入口校验后将其传给宿主 +`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling` +和 `record_history`。 + 默认使用 `dev_meta` 已保存的 **group_1–group_5,共 250 个 case**, ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchmark.case_ids`, 运行时无需读取分组文件,group_6–group_10 保持为留出集。 diff --git a/benchmark/longmemeval/config_home.yaml b/benchmark/longmemeval/config_home.yaml index 95a66f05..aa887faf 100644 --- a/benchmark/longmemeval/config_home.yaml +++ b/benchmark/longmemeval/config_home.yaml @@ -1,8 +1,14 @@ # Preparation only. Paths are relative to the ReMe repository root, not the cwd. # Preserve dev_meta's fixed training split; groups 6-10 remain held out. workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5 +# Policy defaults for Home evaluation; each call still controls sampling=true/false. +sampling_policy: + max_cases: 5 + max_queries: null + temperature: 1.0 + ewma_alpha: 0.3 + uniform_exploration: 0.2 preparation: - schema_version: 1 benchmark: name: longmemeval source: benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json diff --git a/deploy/benchmark_docker/docker_sandbox.py b/deploy/benchmark_docker/docker_sandbox.py index a601b70a..5a015d5d 100644 --- a/deploy/benchmark_docker/docker_sandbox.py +++ b/deploy/benchmark_docker/docker_sandbox.py @@ -272,8 +272,11 @@ class DockerReMeSandboxFactory: plugins: tuple[str, ...] | list[str] = (), command_timeout: float = 1800.0, workspace_builder: WorkspaceBuilder | None = None, + on_sandbox_started: Callable[[], None] | None = None, ) -> None: self.candidate = candidate + self._on_sandbox_started = on_sandbox_started + self._sandbox_started_reported = False self.env = dict(env or {}) self.config = config self.plugins = tuple(plugins) @@ -309,6 +312,9 @@ class DockerReMeSandboxFactory: ) try: await case.initialize() + if self._on_sandbox_started is not None and not self._sandbox_started_reported: + self._on_sandbox_started() + self._sandbox_started_reported = True except BaseException as exc: await cleanup_on_error([(f"workspace {container_id!r}", workspace.close())], exc) raise diff --git a/plugins/home/README.md b/plugins/home/README.md index 462bd574..bdf55e25 100644 --- a/plugins/home/README.md +++ b/plugins/home/README.md @@ -315,3 +315,394 @@ processes must not write there. Recovery requires a writable, functioning filesystem; if rollback fails, its journal remains for the next attempt and the Job reports the error. Calls with `_allowed_paths` restrictions are rejected because preparation owns multiple workspace output locations. + +## Bounded agent foundation + +`reme_home.agent.BoundedAgentWrapper` inherits ReMe's `AsAgentWrapper` and is +registered as the `home_bounded` agent-wrapper backend when Home is enabled. +It reuses ReMe's Job tools, hidden `injected_job_kwargs`, `tool_context_id`, +built-in tools, permissions, model configuration, skills, token accounting and +persisted sessions. No agent component or optimization run is enabled automatically. +Configure a component explicitly, alongside the required model and Home plugins: + +```yaml +components: + agent_wrapper: + investigator: + backend: home_bounded + as_llm: default + react_config: + max_iters: 40 + time_budget_minutes: 30 # Optional advisory budget; null disables it. + job_tools: [home_evaluate] + injected_job_kwargs: + case_ids: ["1"] # Hidden from the model and enforced by the host. +``` + +Call `reply()` or `reply_stream()` through the normal ReMe wrapper interface; +call-time options override component defaults. Each built agent owns its own +timer and request state. The internal `BoundedReActAgent` still subclasses the AS +runtime to implement loop hooks; callers use `BoundedAgentWrapper` for ReMe wiring. +The lower-level `BoundedReActAgent` and `RunTimeBudgetTimer` exports remain available. + +Each reply invocation starts and stops its advisory timer, including on errors and +cancellation. Iteration reminders occur at 50%, 75%, 90% and 95%; coincident or +skipped thresholds produce one reminder. Time reminders occur at 60%, 75%, 90%, +100%, then each additional 10%, alternating temporary and retained messages. Time +budgets do not cancel model calls or tools. Completed context replacements are +counted in time reminders. Reminders use native AS hint blocks, preserving the +reply's cumulative token usage across reasoning and tool calls. Use separate +agent instances for concurrent work. + +The AgentScope loop owns the iteration cap and its single finalization call. +For text reports, Home removes all tools from that call, retains the original +task, and trims oldest messages if needed to fit the context budget. The final +report is still attempted if its preceding context compression fails; ordinary +compression errors outside finalization and cancellation continue to propagate. +The final response is buffered until complete, including for streaming models; unsolicited +tool calls are discarded. If finalization fails or returns no text, the reply +explicitly identifies any retained prior model output as incomplete. Cancellation +and provider interruption keep AgentScope's interruption behavior. Native +structured-output requests retain the SDK's schema and grace-iteration behavior. +Events and final-message token usage retain AgentScope's existing semantics. + +## Candidate versions and code inspection + +Home registers the `home.code` Component and ten ordinary Step-backed Jobs. They +operate only on the prepared `home/code/repo`, sharing the preparation/evaluation +process lock. Enable Home and prepare the workspace before calling them. + +| Job | Inputs and behavior | +| --- | --- | +| `home_code_status` | Current branch, full commit SHA, exact changed paths and pending merge state. | +| `home_list_candidate_branches` | Local branches and candidates, excluding protected `init` from the candidate count. | +| `home_create_candidate_branch` | `branch_name`: fork current HEAD and switch to a new candidate. | +| `home_switch_candidate_branch` | `branch_name`: switch to an existing local branch, including `init`. | +| `home_merge_candidate_branch` | `branch_name`: merge another candidate without committing; conflicts are aborted. | +| `home_commit_candidate` | `message`: stage and commit all visible changes within prepared `candidate_paths`. | +| `home_code_diff` | Omit inputs for tracked changes against HEAD; supply `base_revision` and `revision` to compare commits. With only `base_revision`, compare the working tree against that commit. | +| `home_list_code_tree` | Optional `revision`, literal `path`, and `max_results` (default 500, maximum 2000). | +| `home_read_code_file` | Required `path`; optional `revision`, one-based `start_line` and inclusive `end_line`. | +| `home_search_code` | Required Python regex `pattern`; optional `revision`, literal `path`, and `max_results` (maximum 30). | + +Start the configured service in one terminal, then use a second terminal: + +```bash +reme start config=/absolute/path/to/home.yaml +``` + +```bash +reme home_code_status +reme home_create_candidate_branch branch_name=retrieval-routing +# Edit files under the prepared candidate_paths, then inspect and commit. +reme home_code_diff +reme home_commit_candidate message='Improve evidence routing' +reme home_evaluate debug=true +reme home_evaluate +``` + +Branch names must be single path-safe names without slashes. Creation, switching +and merging require a clean checkout, including untracked files and pending +merges. `init` must continue to point to the preparation's initial commit; commits +and merges into it are rejected. Detached candidates, symlinks, submodules and +changes outside `candidate_paths` are rejected by the managed mutation tools. +Merges require an explicit later commit. No remote operations, branch deletion, +amend, reset or rebase are exposed. Empty ordinary commits fail through Git. +These guarantees apply to cooperating Home Jobs; do not grant an optimizer +unrestricted shell writes or direct access to `.git` if it must obey this boundary. +Cancellation waits for already-started Git work to finish before returning. + +Code-navigation Jobs resolve `revision` (default `HEAD`) to a full SHA once per +call, read raw committed objects and never execute candidate code. Dirty working +files and Git archive attributes do not affect these reads. Pin the same SHA +across a diagnostic session through ReMe's hidden injection: + +```yaml +job_tools: [home_list_code_tree, home_read_code_file, home_search_code] +injected_job_kwargs: + revision: "" +``` + +The wrapper hides injected fields from the model and rejects attempts to supply +them. For comparisons, configure two agent invocations or two named Job aliases +with separately pinned revisions. `HomeCodeStep` takes a fixed `operation` in +Step configuration; repository location and operation are never model-selected. +The Python entry point is `await home_code.run(operation, **arguments)`. + +Reads are limited to 300 lines and 100,000 content characters, with explicit +truncation flags. Search accepts Python multiline regex, merges adjacent match +contexts, and displays at most five lines per result block and 100,000 content +characters. Files above 5 MB and searches above 100 MB are rejected; regex +matching runs in a separate process with a ten-second timeout. Use narrower +paths or line ranges to continue inspecting. These tool-operation timeouts are +independent of the bounded agent's advisory time budget. + +Evaluation also checks the standalone repository identity and protected baseline +before taking its existing clean-HEAD snapshot. Formal evaluation deduplication, +automatic ancestor/donor selection and optimization scheduling remain separate +policies; these Jobs do not enable an optimizer or change evaluation selection. + +## Evaluation sampling and history + +`home_evaluate` is the single evaluation Job. Set `sampling=true` for adaptive +case/query selection, or leave it false for a full baseline or an explicit +host-provided `case_ids`/`query_ids` selection. `sampling=true` conflicts with +explicit selections in scored mode. `debug=true` ignores the sampling flag, +uses debug cases and never reads, writes or registers evaluation history. +The former `home_validate` Job is removed; use `home_evaluate sampling=true`. + +Selection and optimization-history recording are independent. `record_history` +defaults to `true` for compatibility: a first full optimization baseline contributes +observations even with `sampling=false`. Set `record_history=false` to score code +without feeding optimization history. This still saves the manifest, scores and +result artifacts, but never registers a history entry, increments either counter +or updates sampling observations. Rebuilding history preserves that exclusion. +Debug always skips history regardless of either flag. + +Evaluation parameters have distinct names and lifetimes: + +| Parameter | Where it belongs | Meaning | +| --- | --- | --- | +| `sampling` | Job input, default `false` | Enable adaptive selection for this invocation. | +| `record_history` | Job input, default `true` | Record started rounds and successful results for optimization. | +| `sampling_policy` | Component defaults or host Job input | Case/query limits and weighting settings. | + +Home registers these Component defaults automatically; override the policy in +application configuration when needed: + +```yaml +components: + home.evaluation: + default: + backend: home_evaluation + sampling_policy: + max_cases: 5 + max_queries: null + temperature: 1.0 + ewma_alpha: 0.3 + uniform_exploration: 0.2 +``` + +The Component policy is immutable. Each invocation validates its own policy, +merging a partial `sampling_policy` over Component defaults and persisting the +effective values with that run. Explicit null `max_queries` removes the query +budget. The previous Component configuration key `sampling` is now +`sampling_policy`; passing a policy object as the Job's boolean `sampling` fails. +Normal ReMe Job defaults also work: set `sampling: true` and optionally +`sampling_policy` directly under `jobs.home_evaluate`. Call-time values take +precedence over same-named Job defaults; a call-time policy object replaces the +Job-default policy object, then merges with Component defaults. + +With the configured service running, use a second terminal: + +```bash +reme home_evaluate sampling=false # Full baseline. +reme home_evaluate sampling=false record_history=false # Standalone full evaluation. +reme home_evaluate sampling=true # Adaptive formal evaluation. +reme home_evaluate debug=true sampling=true # Debug only; no history feedback. +``` + +Host Python callers can override the policy for one invocation without changing +the next invocation's defaults: + +```python +result = await app.run_job( + "home_evaluate", + sampling=True, + sampling_policy={"max_cases": 3, "max_queries": 12}, +) + +# Score specified prepared cases without affecting the optimizer's history. +standalone = await app.run_job( + "home_evaluate", + sampling=False, + record_history=False, + case_ids=["case-a", "case-b"], # Omit to run the full prepared dataset. +) +``` + +`sampling_policy` and explicit case/query IDs stay out of the default agent tool +schema. An optimizer that must obey a fixed budget should receive hidden, +enforced values through the normal wrapper injection: + +```yaml +job_tools: [home_evaluate] +injected_job_kwargs: + sampling: true + record_history: true + sampling_policy: + max_cases: 3 + max_queries: 12 +``` + +The wrapper hides injected parameters and rejects model attempts to provide +those keys. Use ordinary call arguments when the host should decide the mode +per invocation; use injection when that decision must remain fixed for an agent. + +Case/query selection retains dev_meta b53048e4's EWMA loss, staleness, observation +uncertainty, un-rechecked error priority, normalized adaptive/uniform mixture, +weighted drawing without replacement and fair-share query budget allocation. +Policy defaults are unchanged. Positive integer case/query limits, positive +finite temperature, alpha in `(0, 1]` and uniform exploration in `[0, 1]` are +validated. Seeds, draw weights, eligibility and exact selections are saved. +An explicit construction donor uses the existing dev_meta eligibility rules and +checksum checks. A later full rebuild considers the full training set again. + +### Successful evaluation history + +`reme_home.evaluation.history.HomeEvaluationHistory` is the dedicated +`home.evaluation_history` Component, automatically bound by `HomeEvaluation`. +It owns successful-result registration, lookup, deduplication, cache rebuilding +and the observations used for the next draw. Steps remain invocation adapters; +they do not own cross-evaluation state. + +A successful history entry requires `record_history=true` and a started, completed, scored training-set +run with a full summary and no infrastructure errors, following the existing +`Response.success` contract. Low or zero quality scores remain valid observations. +Classified candidate failures retain the existing response semantics; only their +actual numeric scores train EWMA. A run with any infrastructure error is entirely +excluded, including scores from its healthy cases. Debug, cancelled, fail-fast +and pre-start failures never enter successful history. A full summary is the durable +completion boundary: cancelling the caller during subsequent history bookkeeping +does not relabel that already-completed evaluation. Failed registration discards +only derived caches so they can rebuild from the final source run status. + +```python +history = app.context.components["home.evaluation_history"]["default"] +results = history.list_results() # Typed EvaluationRecord objects, chronological. +result = history.get_result(results[-1].run_key) +history.rebuild() # Rebuild derived indexes from saved artifacts. +``` + +The Component owns the workspace lock for these synchronous filesystem methods; +async host code can run them in a worker thread. Its compact catalog lives at +`home/evaluations/history.json`. Each record points to its run via `run_key` and +contains commit/branch identity, selected IDs, completion time and summary metrics; +questions, answers and memory files stay in their original result directory. + +The separate startup ledger advances only after the first successfully initialized +sandbox, including when a later worker or execution fails. Extra/replacement +workers do not count twice. `home/sampling/state.json` distinguishes started +`round` from `successful_round`. Only successful history advances the probability +clock or changes EWMA; failed runs cannot age cases or alter the next draw's +weights. Started plans live under +`home/sampling/plans///.json`. Their `round` is the +startup ledger round; `selection_round` records the successful-history snapshot +used for an adaptive draw. + +For `record_history=false`, full or explicit evaluation does not access the history +manager. Adaptive selection can still read existing observations, while leaving +them unchanged; its draw is saved locally as `manifest.selection_plan`, without +a startup-ledger round or shared sampling plan. `manifest.options.record_history` +persists the requested setting, and the response's `record_history` indicates +whether the run participates in history (always false for debug). Legacy manifests +without the setting retain the previous history behavior. + +Formal runs remain serialized by `home/metadata/validation.lock`; debug bypasses +that lock. Missing caches reconstruct from started plans and successful scored +artifacts. Older sampling caches are rebuilt under the success-only rule. +`history.rebuild()` rewrites only the catalog and sampling cache. Preserve source +manifests, plans and result artifacts when rebuilding; candidate code, datasets +and memory files are never modified. + +## Evaluation result inspection + +Two read-only Jobs navigate saved evaluations independently of successful +optimization history. They include debug, failed, cancelled, running and standalone +(`record_history=false`) runs. Reads never register results, update sampling, +rebuild summaries or write artifacts. + +`home_list_validation_results` accepts optional `branch_name` and `commit_sha` +filters plus `offset=0`, `limit=20` (1–100). Without filters it lists all runs, +newest first, with a stable path tie-breaker. It returns `row_count`, `rows` and +`next_offset`, compact identity/options, saved summary counts and scores, failure +previews and artifact paths. A malformed run is reported as a `read_error` row +without stopping the unfiltered catalog. + +`commit_sha` accepts a full SHA or hexadecimal prefix. A prefix must match exactly +one of **all commits reachable from the selected local branch**, including commits +that have never been evaluated. Omitting `branch_name` with a commit filter uses +the current candidate branch. Missing or ambiguous matches fail explicitly; the +response includes the resolved branch and full SHA. A branch-only filter can also +browse historical runs from a deleted branch. + +`home_inspect_validation` requires `validation_path` (absolute or relative to +`home/`) and accepts optional `case_id`, `query_id`, `offset=0` and `limit=20`: + +| Selection | Returned rows | +| --- | --- | +| Run only | Case summaries, missing scores first, then low scores. | +| Run and case | Query summaries in manifest order, plus independently paged failed construction Jobs. | +| Run, case and query | One query's question, answer, rubric, score and judge/error previews. | + +`query_id` requires `case_id`; both must be selected by the run manifest. Cases or +queries without published evidence are explicitly marked `missing`, with null +scores. The tool uses a saved full summary, or an existing partial summary when +the full summary is absent. Missing summaries are reported without reconstruction. +Errors are previewed to 600 characters, and selected query fields to 2,000 each; +truncated fields carry a marker. `rows_source`, JSON `pointer` values, and log and +memory paths identify the original evidence. Failed construction Jobs have their +own `row_count`, `rows`, `next_offset` under `construction.failed_jobs`; their +pagination is independent of the query rows, using the supplied offset/limit. + +With the configured service running, use a second terminal: + +```bash +reme home_list_validation_results branch_name=candidate commit_sha=abc123 limit=10 +reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1 +reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1 case_id=case-a query_id=query-a +``` + +Host callers use the same Jobs through `app.run_job()`. For a case-scoped agent, +inject `validation_path` and `case_id` with `injected_job_kwargs`; the wrapper hides +and enforces these values. Both Jobs reject path escapes and symlinks, and fail +closed for restricted `_allowed_paths` contexts, like the other Home Jobs. + +### Memory checkpoint inspection + +Two read-only Jobs investigate a case's exported memory Git repository. Locate +an evaluation and case with `home_list_validation_results` and +`home_inspect_validation` first. Neither Job changes artifacts, Git refs, the +index, evaluation history or sampling state. + +- `home_inspect_memory_history(validation_path, case_id, path=null, offset=0, + limit=20)` returns `status`, `workspace_exists`, `has_git_history`, `head`, + `repository`, and newest-first checkpoint `rows` with `row_count` and + `next_offset`. Limits are 1–100. Each row includes the full commit SHA, parents, + timestamp, subject (up to 2,000 characters) and a `session_id` when the subject + follows `session: `. `path` follows a literal workspace-relative file or + directory. Missing exports, absent Git history and empty repositories have + distinct statuses; corrupt repositories fail explicitly. +- `home_compare_memory(validation_path, case_id, base_ref=null, + target_ref="HEAD", target_validation_path=null, path=null)` compares two memory + commits. Within one evaluation, `base_ref` is required. For another evaluation, + supply `target_validation_path`; both sides then default to `HEAD`. The case + must be selected in both manifests. Returns resolved commits, candidate commit + identities, manifest fingerprints, added/removed/modified paths, content-tree + hashes, a unified `diff`, and `truncated`. + +References accept HEAD-relative expressions and hexadecimal SHA prefixes unique +among commits reachable from that memory repository's HEAD. Resolved expressions +must also be HEAD-reachable. This is independent of candidate-code branch history. +Comparisons read committed regular files from Git objects, ignoring dirty or +untracked checkout contents. File mode changes alone are not content differences. +Binary changes are identified without printing their bytes. Each snapshot is +limited to 2,000 files, 20 MB total and 5 MB per file; diff output is limited to +100,000 characters. Narrow `path` if a limit is reached. Tree hashes describe the +selected path scope. Linked Git storage, alternates, shallow/partial histories and +symbolic links in Git metadata are rejected. + +After starting the service with the Home configuration, call from another terminal: + +```bash +reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1 +reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 case_id=case1 base_ref=HEAD~1 +reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 target_validation_path=evaluations/candidate/1234567/1 case_id=case1 +``` + +`validation_path` is absolute or relative to `home/`; for example the relative +form is `evaluations/candidate/abcdef0/1`. For agent tools, host `injected_job_kwargs` can pin +`validation_path` and `case_id`; the wrapper hides injected fields and rejects +model overrides. Like the other Home inspection Jobs, these Jobs reject +`_allowed_paths` restrictions rather than bypassing them. Query-to-session evidence +tracing and the case study agent are separate functionality. diff --git a/plugins/home/src/reme_home/agent/__init__.py b/plugins/home/src/reme_home/agent/__init__.py new file mode 100644 index 00000000..3a6a4a09 --- /dev/null +++ b/plugins/home/src/reme_home/agent/__init__.py @@ -0,0 +1,6 @@ +"""Agent foundations for Home's investigation and optimization workflows.""" + +from .bounded import BoundedReActAgent, RunTimeBudgetTimer +from .wrapper import BoundedAgentWrapper + +__all__ = ["BoundedAgentWrapper", "BoundedReActAgent", "RunTimeBudgetTimer"] diff --git a/plugins/home/src/reme_home/agent/bounded.py b/plugins/home/src/reme_home/agent/bounded.py new file mode 100644 index 00000000..d89df5a9 --- /dev/null +++ b/plugins/home/src/reme_home/agent/bounded.py @@ -0,0 +1,308 @@ +"""AgentScope agent that turns an exhausted ReAct budget into a final answer.""" + +from __future__ import annotations + +import inspect +import logging +import math +import time +from dataclasses import dataclass +from typing import Any, AsyncGenerator, Callable + +from agentscope.agent import Agent +from agentscope.message import HintBlock, Msg, TextBlock, UserMsg +from agentscope.model import ChatResponse, FinishedReason +from agentscope.tool import ToolChoice + +logger = logging.getLogger(__name__) + +_REMINDER_RATIOS = (0.50, 0.75, 0.90, 0.95) + + +@dataclass(frozen=True) +class TimeBudgetReminder: + """One reminder plus whether it should survive the current reasoning call.""" + + content: str + persistent: bool + + +class RunTimeBudgetTimer: + """Track one agent run and emit advisory reminders without stopping it.""" + + def __init__( + self, + time_budget_minutes: float, + *, + agent_label: str, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if ( + not isinstance(time_budget_minutes, (int, float)) + or isinstance(time_budget_minutes, bool) + or not math.isfinite(time_budget_minutes) + or time_budget_minutes <= 0 + ): + raise ValueError("time_budget_minutes must be a positive number") + if not isinstance(agent_label, str) or not agent_label.strip(): + raise ValueError("agent_label must be a non-empty string") + self.time_budget_minutes = float(time_budget_minutes) + self.agent_label = agent_label.strip() + self._clock = clock + self._started_at: float | None = None + self._last_reminded_ratio = 0.0 + self._next_reminder_is_persistent = False + self.context_compression_count = 0 + + @property + def active(self) -> bool: + """Return whether a run is currently being timed.""" + + return self._started_at is not None + + def start(self) -> None: + """Start a fresh run timer.""" + + self._started_at = self._clock() + self._last_reminded_ratio = 0.0 + self._next_reminder_is_persistent = False + self.context_compression_count = 0 + + def stop(self) -> None: + """Stop timing the current run.""" + + self._started_at = None + + def record_context_compression(self) -> None: + """Record one completed context compression in the current run.""" + + self.context_compression_count += 1 + + def next_reminder(self) -> TimeBudgetReminder | None: + """Return the latest newly reached soft-budget reminder.""" + + if self._started_at is None: + return None + elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0) + elapsed_ratio = elapsed_minutes / self.time_budget_minutes + reached_ratio: float | None = None + if elapsed_ratio >= 1.0: + reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10 + else: + for threshold in (0.90, 0.75, 0.60): + if elapsed_ratio >= threshold: + reached_ratio = threshold + break + if reached_ratio is None or reached_ratio <= self._last_reminded_ratio: + return None + self._last_reminded_ratio = reached_ratio + persistent = self._next_reminder_is_persistent + self._next_reminder_is_persistent = not persistent + elapsed = _format_minutes(elapsed_minutes) + percentage = round(reached_ratio * 100) + return TimeBudgetReminder( + content=( + f"The {self.agent_label} run has used {elapsed} ({percentage}% of its " + f"{_format_minutes(self.time_budget_minutes)} soft time budget). " + f"The run has completed {self.context_compression_count} context compressions. " + "This reminder does not terminate the run. Prioritize the highest-value remaining investigation and " + "produce the required final report as soon as the evidence is sufficient." + ), + persistent=persistent, + ) + + +def _format_minutes(minutes: float) -> str: + """Format a timer duration compactly.""" + + if minutes < 1: + return f"{max(0, round(minutes * 60))} seconds" + rounded = round(minutes, 1) + value = int(rounded) if rounded.is_integer() else rounded + unit = "minute" if value == 1 else "minutes" + return f"{value} {unit}" + + +def _remove_hint_by_identity(messages: list[Msg], target: HintBlock) -> None: + """Remove only the temporary hint, retaining the reply message and its usage.""" + for message in messages: + for index, block in enumerate(message.content): + if block is target: + del message.content[index] + return + + +class BoundedReActAgent(Agent): + """Add advisory budgets and safe finalization to AgentScope's bounded loop. + + AgentScope owns iteration accounting, tool execution, events and cancellation. + Its one final reasoning call after ``max_iters`` receives no tools. Native + structured-output requests retain AgentScope's schema and grace-iteration rules. + Instances, like AgentScope agents, must not serve concurrent replies. + """ + + def __init__(self, *args: Any, run_time_budget_timer: RunTimeBudgetTimer | None = None, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.run_time_budget_timer = run_time_budget_timer + self._sent_iteration_reminders: set[int] = set() + self._original_task_prompt = "" + + async def _reply_impl(self, inputs: Any = None, **kwargs: Any) -> AsyncGenerator[Any, None]: + """Own the advisory timer for this invocation, including error cleanup.""" + self._sent_iteration_reminders.clear() + if isinstance(inputs, Msg): + self._original_task_prompt = inputs.get_text_content() or "" + elif isinstance(inputs, list): + self._original_task_prompt = "\n\n".join(message.get_text_content() or "" for message in inputs) + elif inputs is None: + # ReMe observes inputs before reply(); retain that task for finalization. + self._original_task_prompt = next( + ( + message.get_text_content() or "" + for message in reversed(self.state.context) + if message.role == "user" + ), + "", + ) + timer = self.run_time_budget_timer + if timer is not None: + timer.start() + try: + async for item in super()._reply_impl(inputs=inputs, **kwargs): + yield item + finally: + if timer is not None: + timer.stop() + + async def compress_context(self, *args: Any, **kwargs: Any) -> None: + """Count successful compressions; allow final reports after compression failure.""" + summary_before, context_before = self.state.summary, self.state.context + try: + await super().compress_context(*args, **kwargs) + except Exception as exc: + if not self._is_finalizing(): + raise + logger.warning( + "Compression before the final report failed for Home agent %s; attempting finalization: %s", + self.name, + type(exc).__name__, + ) + return + if self.run_time_budget_timer is not None and ( + self.state.summary is not summary_before or self.state.context is not context_before + ): + self.run_time_budget_timer.record_context_compression() + + async def _reasoning(self, *args: Any, **kwargs: Any) -> AsyncGenerator[Any, None]: + """Deliver each iteration threshold once and alternate time reminders.""" + timer = self.run_time_budget_timer + reminder = timer.next_reminder() if timer is not None else None + reminder_hint = None + if reminder is not None: + reminder_hint = HintBlock(source="home_run_time_guard", hint=reminder.content) + self.state.append_context(self.name, [reminder_hint]) + completed = self.state.cur_iter + thresholds = {math.ceil(self.react_config.max_iters * ratio) for ratio in _REMINDER_RATIOS} + reached = thresholds - self._sent_iteration_reminders + reached = {threshold for threshold in reached if threshold <= completed} + if reached: + remaining = max(0, self.react_config.max_iters - completed) + self.state.append_context( + self.name, + [ + HintBlock( + source="home_iteration_guard", + hint=( + f"You have completed {completed} reasoning-acting iterations; " + f"{remaining} iteration(s) remain." + ), + ), + ], + ) + self._sent_iteration_reminders.update(reached) + try: + async for item in super()._reasoning(*args, **kwargs): + yield item + finally: + if reminder is not None and not reminder.persistent and reminder_hint is not None: + _remove_hint_by_identity(self.state.context, reminder_hint) + + def _is_finalizing(self) -> bool: + """Only the iteration cap triggers finalization; elapsed time is advisory.""" + return self.state.cur_iter >= self.react_config.max_iters and self.state.reply_context.structured_schema is None + + async def _call_model(self, messages: list[Msg], tools: list[dict], tool_choice: ToolChoice | None = None): + """Use the native finalization slot; never start a second synthesis call.""" + if not self._is_finalizing(): + return await super()._call_model(messages=messages, tools=tools, tool_choice=tool_choice) + + last_text = next( + ( + message.get_text_content() + for message in reversed(self.state.context) + if message.role == "assistant" and message.name == self.name and message.get_text_content() + ), + "(no previous displayable model response)", + ) + messages = list(messages) + messages.append( + UserMsg( + name="home_iteration_guard", + content=( + "The iteration budget is exhausted. Return the best final report from collected evidence. " + "Do not call tools or propose more investigation. Distinguish findings from uncertainty.\n" + f"Original task: {self._original_task_prompt or '(see the conversation above)'}" + ), + ), + ) + response_usage = None + try: + limit = int(self.model.context_size * self.context_config.trigger_ratio) + while await self.model.count_tokens(messages=messages, tools=[]) >= limit: + if len(messages) <= 2: + raise ValueError("Final instructions exceed the model context budget") + del messages[1] + response = await super()._call_model(messages=messages, tools=[], tool_choice=ToolChoice(mode="none")) + response = await self._consume_final_response(response) + response_usage = response.usage + if response.finished_reason == FinishedReason.INTERRUPTED: + return response + content = [block for block in response.content if isinstance(block, TextBlock) and block.text.strip()] + if not content: + raise ValueError("Final model response contains no displayable answer") + # Ignore unsolicited tool calls, even when accompanied by valid text. + response.content = content + return response + except Exception as exc: + logger.exception("Final report failed for Home agent %s", self.name) + return ChatResponse( + content=[ + TextBlock( + text=( + f"Reached the maximum of {self.react_config.max_iters} reasoning-acting iterations. " + f"Final report generation failed ({type(exc).__name__}); " + f"the following is incomplete prior model output:\n\n{last_text}" + ), + ), + ], + is_last=True, + usage=response_usage, + ) + + @staticmethod + async def _consume_final_response(response: ChatResponse | AsyncGenerator[ChatResponse, None]) -> ChatResponse: + """Require a completed response for streaming and non-streaming models.""" + if isinstance(response, ChatResponse): + return response + if not inspect.isasyncgen(response): + raise TypeError(f"Unexpected final response: {type(response)}") + completed = None + try: + async for chunk in response: + if chunk.is_last: + completed = chunk + finally: + await response.aclose() + if completed is None: + raise ValueError("Final response stream ended without a completed response") + return completed diff --git a/plugins/home/src/reme_home/agent/wrapper.py b/plugins/home/src/reme_home/agent/wrapper.py new file mode 100644 index 00000000..60425c3c --- /dev/null +++ b/plugins/home/src/reme_home/agent/wrapper.py @@ -0,0 +1,21 @@ +"""ReMe agent wrapper with Home's bounded investigation runtime.""" + +from typing import Any + +from reme.components.agent_wrapper import AsAgentWrapper + +from .bounded import BoundedReActAgent, RunTimeBudgetTimer + + +class BoundedAgentWrapper(AsAgentWrapper): + """Reuse ReMe Job tools, injected arguments, permissions and persisted sessions. + + ``react_config.max_iters`` configures the iteration cap. Optional + ``time_budget_minutes`` adds advisory reminders; each built SDK agent owns + its own timer so requests never share budget state. + """ + + def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> BoundedReActAgent: + budget = options.get("time_budget_minutes") + timer = None if budget is None else RunTimeBudgetTimer(budget, agent_label=self.name) + return BoundedReActAgent(run_time_budget_timer=timer, **kwargs) diff --git a/plugins/home/src/reme_home/code/__init__.py b/plugins/home/src/reme_home/code/__init__.py new file mode 100644 index 00000000..5772d9b5 --- /dev/null +++ b/plugins/home/src/reme_home/code/__init__.py @@ -0,0 +1 @@ +"""Home candidate version management and immutable code inspection.""" diff --git a/plugins/home/src/reme_home/code/component.py b/plugins/home/src/reme_home/code/component.py new file mode 100644 index 00000000..eaaecb03 --- /dev/null +++ b/plugins/home/src/reme_home/code/component.py @@ -0,0 +1,72 @@ +"""Application-owned Git work serialized with Home preparation and snapshots.""" + +import asyncio + +from reme.components.base_component import BaseComponent + +from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory +from . import repository, snapshot +from .models import ARGUMENTS + +OPERATIONS = { + "status": repository.status, + "list_branches": lambda repo, _record: repository.branches(repo), + "create_branch": repository.create_branch, + "switch_branch": repository.switch_branch, + "merge_branch": repository.merge_branch, + "commit": repository.commit, + "diff": repository.diff, + "list_code_tree": snapshot.list_code_tree, + "read_code_file": snapshot.read_code_file, + "search_code": snapshot.search_code, +} + + +class HomeCode(BaseComponent): + """Manage only the prepared Home candidate; no remotes or history rewrites.""" + + component_type = "home.code" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.workspace = self.bind("default", HomeWorkspace, optional=False) + self._pending: set[asyncio.Task] = set() + + async def _close(self): + if self._pending: + await asyncio.gather(*self._pending, return_exceptions=True) + + async def run(self, operation: str, **kwargs) -> dict: + """Validate before dispatch, and finish owned Git work before propagating cancellation.""" + if not self.is_started: + raise RuntimeError("Start the Home application before managing candidate code") + if operation not in OPERATIONS: + raise ValueError(f"Unknown code operation: {operation}") + arguments = ARGUMENTS[operation].model_validate(kwargs).model_dump() + task = asyncio.create_task(asyncio.to_thread(self._run, operation, arguments)) + self._pending.add(task) + cancelled = False + try: + while True: + try: + result = await asyncio.shield(task) + if cancelled: + raise asyncio.CancelledError + return result + except asyncio.CancelledError: + if task.done(): + raise + cancelled = True + except Exception: + if cancelled: + raise asyncio.CancelledError from None + raise + finally: + self._pending.discard(task) + + def _run(self, operation: str, arguments: dict) -> dict: + """Hold the same cross-process lock used when evaluations snapshot HEAD.""" + _validate_directory(self.workspace_path.resolve(), self.workspace.state_path) + with _initialization_lock(self.workspace.state_path / "workspace.lock"): + repo, record = repository.validate_repository(self.workspace.home_path) + return OPERATIONS[operation](repo, record, **arguments) diff --git a/plugins/home/src/reme_home/code/models.py b/plugins/home/src/reme_home/code/models.py new file mode 100644 index 00000000..b64ce460 --- /dev/null +++ b/plugins/home/src/reme_home/code/models.py @@ -0,0 +1,68 @@ +"""Strict input contracts for candidate version and code-navigation Jobs.""" + +from pydantic import Field + +from ..preparation.spec import SpecModel + + +class NoArguments(SpecModel): + """An operation with no public inputs.""" + + +class BranchArguments(SpecModel): + """One existing or new local candidate branch.""" + + branch_name: str = Field(min_length=1, max_length=100) + + +class CommitArguments(SpecModel): + """Commit the complete candidate change with an explicit message.""" + + message: str = Field(min_length=1, max_length=10_000, pattern=r"\S") + + +class DiffArguments(SpecModel): + """Omit revisions for tracked working changes against HEAD.""" + + base_revision: str | None = None + revision: str | None = None + + +class TreeArguments(SpecModel): + """Resolve HEAD by default; hosts can pin revision with injected_job_kwargs.""" + + revision: str = Field(default="HEAD", min_length=1) + path: str | None = None + max_results: int = Field(default=500, ge=1, le=2000) + + +class ReadArguments(SpecModel): + """A bounded line range from a committed regular text file.""" + + path: str = Field(min_length=1) + revision: str = Field(default="HEAD", min_length=1) + start_line: int = Field(default=1, ge=1) + end_line: int | None = Field(default=None, ge=1) + + +class SearchArguments(SpecModel): + """Python regex search with multiline matching and bounded result blocks.""" + + pattern: str = Field(min_length=1, max_length=500) + revision: str = Field(default="HEAD", min_length=1) + path: str | None = None + max_results: int = Field(default=30, ge=1, le=30) + + +ARGUMENTS = { + "status": NoArguments, + "list_branches": NoArguments, + "create_branch": BranchArguments, + "switch_branch": BranchArguments, + "merge_branch": BranchArguments, + "commit": CommitArguments, + "diff": DiffArguments, + "list_code_tree": TreeArguments, + "read_code_file": ReadArguments, + "search_code": SearchArguments, +} diff --git a/plugins/home/src/reme_home/code/regex_worker.py b/plugins/home/src/reme_home/code/regex_worker.py new file mode 100644 index 00000000..588dae59 --- /dev/null +++ b/plugins/home/src/reme_home/code/regex_worker.py @@ -0,0 +1,84 @@ +"""Stdlib-only, subprocess-isolated Python regex matching for committed text.""" + +from bisect import bisect_right +from itertools import accumulate +import json +import re +import sys + + +def search(files: dict[str, str], pattern: str, max_results: int) -> dict: + """Merge adjacent contexts, retaining at most five lines per displayed block.""" + compiled = re.compile(pattern, re.MULTILINE) + blocks = [] + occurrences = 0 + for path, text in files.items(): + lines = text.splitlines() + starts = [0, *accumulate(map(len, text.splitlines(keepends=True)))] + for match in compiled.finditer(text): + if match.start() == match.end(): + raise ValueError("pattern must consume at least one character") + occurrences += 1 + if occurrences > 100_000: + raise ValueError("Too many regex matches; narrow pattern or path") + first = bisect_right(starts, match.start()) + last = bisect_right(starts, match.end() - 1) + start, end = max(1, first - 1), min(len(lines), last + 1) + if ( + blocks + and blocks[-1]["path"] == path + and start <= blocks[-1]["end"] + 1 + and (max(end, blocks[-1]["end"]) - blocks[-1]["start"] < 5) + ): + block = blocks[-1] + block["end"] = max(end, block["end"]) + block["ranges"].append((first, last)) + else: + blocks.append({"path": path, "start": start, "end": end, "ranges": [(first, last)]}) + matches = [] + characters = 0 + for block in blocks[:max_results]: + lines = files[block["path"]].splitlines() + displayed_end = min(block["end"], block["start"] + 4) + content = "\n".join( + f"{'>' if any(first <= index <= last for first, last in block['ranges']) else ' '} " + f"{index} | {lines[index - 1]}" + for index in range(block["start"], displayed_end + 1) + ) + remaining = 100_000 - characters + if remaining <= 0: + break + content = content[:remaining] + characters += len(content) + matches.append( + { + "path": block["path"], + "start_line": block["start"], + "end_line": block["end"], + "displayed_end_line": displayed_end, + "occurrence_count": len(block["ranges"]), + "content": content, + "content_truncated": displayed_end < block["end"] or characters >= 100_000, + }, + ) + return { + "occurrence_count": occurrences, + "merged_match_count": len(blocks), + "returned_match_count": len(matches), + "matches": matches, + "truncated": len(blocks) > len(matches) or any(match["content_truncated"] for match in matches), + } + + +def main(): + """Emit structured errors without candidate text or traceback noise.""" + arguments = json.load(sys.stdin) + try: + result = search(**arguments) + except (ValueError, re.error) as exc: + result = {"error": str(exc)} + json.dump(result, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/plugins/home/src/reme_home/code/repository.py b/plugins/home/src/reme_home/code/repository.py new file mode 100644 index 00000000..a02d24b7 --- /dev/null +++ b/plugins/home/src/reme_home/code/repository.py @@ -0,0 +1,213 @@ +"""Local candidate Git operations shared by Home Jobs and evaluation snapshots.""" + +from pathlib import Path + +from ..evaluation.artifacts import contained, read_json +from ..preparation.code import git +from ..preparation.spec import relative_path + +INITIAL_BRANCH = "init" +MAX_OUTPUT = 100_000 + + +def resolve_revision(repository: Path, revision: str) -> str: + """Resolve a revision once, without accepting command-line options.""" + if not isinstance(revision, str) or not revision.strip() or revision.startswith("-"): + raise ValueError("revision must be a nonempty Git revision without a leading '-'") + return git(repository, "rev-parse", "--verify", "--end-of-options", revision + "^{commit}") + + +def validate_repository(home: Path, record: dict | None = None) -> tuple[Path, dict]: + """Require the prepared standalone repository and its unchanged baseline ref.""" + record = record if record is not None else read_json(contained(home, "metadata/preparation.json")) + repository = contained(home, "code/repo", directory=True) + metadata = contained(repository, ".git", directory=True) + if not metadata.is_dir() or git(repository, "rev-parse", "--show-toplevel") != str(repository): + raise ValueError("Candidate must be a standalone Git repository") + if resolve_revision(repository, "refs/heads/init") != record["code"]["initial_commit"]: + raise ValueError("The protected init branch has changed since preparation") + git(repository, "merge-base", "--is-ancestor", record["code"]["initial_commit"], "HEAD") + return repository, record + + +def current_branch(repository: Path) -> str: + """Reject detached HEAD so every candidate commit remains branch-reachable.""" + branch = git(repository, "branch", "--show-current") + validate_branch(repository, branch) + return branch + + +def validate_branch(repository: Path, branch: str) -> str: + """Branch names also serve as single evaluation-directory names.""" + if not branch or branch.startswith("-") or any(char in branch for char in "/\\\0\n\r"): + raise ValueError("branch_name must be one path-safe local branch name") + if git(repository, "check-ref-format", "--branch", branch) != branch: + raise ValueError("Use a literal local branch name") + return branch + + +def require_clean(repository: Path) -> None: + """Do not carry uncommitted or untracked files across branch operations.""" + if git(repository, "status", "--porcelain=v2", "--untracked-files=all", "--ignore-submodules=none"): + raise ValueError("Candidate repository must be clean before changing branches") + if (repository / ".git/MERGE_HEAD").exists(): + raise ValueError("Finish the pending merge first") + + +def branches(repository: Path) -> dict: + """Describe local refs and the current immutable commit.""" + local = sorted(git(repository, "for-each-ref", "--format=%(refname:short)", "refs/heads").splitlines()) + candidates = [branch for branch in local if branch != INITIAL_BRANCH] + return { + "current_branch": current_branch(repository), + "commit_sha": resolve_revision(repository, "HEAD"), + "local_branches": local, + "candidate_branches": candidates, + "candidate_branch_count": len(candidates), + } + + +def status(repository: Path, _record: dict) -> dict: + """Report exact paths, including names containing whitespace.""" + result = branches(repository) + output = git(repository, "status", "--porcelain=v1", "-z", "--untracked-files=all", strip=False) + entries = iter(output.split("\0") if output else []) + changes = [] + for entry in entries: + if not entry: + continue + change = {"status": entry[:2], "path": entry[3:]} + if "R" in entry[:2] or "C" in entry[:2]: + change["original_path"] = next(entries) + changes.append(change) + result.update(changes=changes, clean=not changes, merge_pending=(repository / ".git/MERGE_HEAD").exists()) + return result + + +def create_branch(repository: Path, record: dict, *, branch_name: str) -> dict: + """Fork from the current candidate or baseline and switch to the new branch.""" + validate_branch(repository, branch_name) + if branch_name in branches(repository)["local_branches"]: + raise ValueError("Local branch already exists") + require_clean(repository) + validate_tree(repository, record, "HEAD") + git(repository, "switch", "--no-track", "--create", branch_name) + return {"status": "created", **branches(repository)} + + +def switch_branch(repository: Path, record: dict, *, branch_name: str) -> dict: + """Switch only to an existing local ref, including the baseline.""" + validate_branch(repository, branch_name) + if branch_name not in branches(repository)["local_branches"]: + raise ValueError("Unknown local branch") + require_clean(repository) + validate_tree(repository, record, resolve_revision(repository, "refs/heads/" + branch_name)) + git(repository, "switch", "--no-guess", branch_name) + return {"status": "switched", **branches(repository)} + + +def validate_tree(repository: Path, record: dict, revision: str) -> None: + """Only declared editable roots may differ from the prepared baseline.""" + roots = [Path(relative_path(path)) for path in record["code"]["candidate_paths"]] + changed = git( + repository, + "diff", + "--no-ext-diff", + "--no-textconv", + "--name-only", + "--no-renames", + "-z", + record["code"]["initial_commit"], + revision, + "--", + strip=False, + ) + for name in filter(None, changed.split("\0")): + path = Path(relative_path(name)) + if ".git" in path.parts or not any(path.is_relative_to(root) for root in roots): + raise ValueError(f"Change outside candidate_paths: {name}") + for entry in filter(None, git(repository, "ls-tree", "-r", "-z", revision, strip=False).split("\0")): + header, name = entry.split("\t", 1) + mode, kind, _oid = header.split() + if mode not in {"100644", "100755"} or kind != "blob" or ".git" in Path(relative_path(name)).parts: + raise ValueError(f"Candidate must contain regular files only: {name}") + + +def merge_branch(repository: Path, record: dict, *, branch_name: str) -> dict: + """Stage compatible candidates; automatically abort conflicts.""" + validate_branch(repository, branch_name) + state = branches(repository) + if state["current_branch"] == INITIAL_BRANCH: + raise ValueError("Cannot merge into init") + if branch_name in (INITIAL_BRANCH, state["current_branch"]): + raise ValueError("Select another candidate branch") + if branch_name not in state["candidate_branches"]: + raise ValueError("Unknown candidate branch") + require_clean(repository) + validate_tree(repository, record, "HEAD") + validate_tree(repository, record, resolve_revision(repository, "refs/heads/" + branch_name)) + try: + git( + repository, + "-c", + "user.name=Home", + "-c", + "user.email=home@localhost", + "merge", + "--no-commit", + "--no-ff", + branch_name, + ) + except Exception: + if (repository / ".git/MERGE_HEAD").exists(): + git(repository, "merge", "--abort") + raise + merge_status = "merged_not_committed" if (repository / ".git/MERGE_HEAD").exists() else "already_up_to_date" + return {"status": merge_status, "source_branch": branch_name, **branches(repository)} + + +def commit(repository: Path, record: dict, *, message: str) -> dict: + """Commit all candidate changes without rewriting existing history.""" + if current_branch(repository) == INITIAL_BRANCH: + raise ValueError("Create a candidate branch before committing; init is protected") + # Validate existing staged changes before staging anything else, then the complete index. + validate_tree(repository, record, git(repository, "write-tree")) + changes = status(repository, record)["changes"] + roots = [Path(value) for value in record["code"]["candidate_paths"]] + for change in changes: + for name in (change["path"], change.get("original_path", change["path"])): + path = Path(relative_path(name)) + if ".git" in path.parts or not any(path.is_relative_to(root) for root in roots): + raise ValueError(f"Change outside candidate_paths: {name}") + target = repository / path + if target.is_symlink(): + raise ValueError(f"Candidate must contain regular files only: {name}") + git(repository, "add", "--all", "--", ".") + validate_tree(repository, record, git(repository, "write-tree")) + git(repository, "-c", "user.name=Home", "-c", "user.email=home@localhost", "commit", "--message", message) + return {"status": "committed", **branches(repository)} + + +def diff(repository: Path, _record: dict, *, base_revision: str | None = None, revision: str | None = None) -> dict: + """Compare commits or all tracked changes against HEAD; never run external diff helpers.""" + if revision is not None and base_revision is None: + raise ValueError("revision requires base_revision") + base = resolve_revision(repository, base_revision or "HEAD") + target = resolve_revision(repository, revision) if revision is not None else None + output = git( + repository, + "diff", + "--no-ext-diff", + "--no-textconv", + "--no-renames", + "--ignore-submodules=all", + base, + *([target] if target else []), + "--", + ) + return { + "base_revision": base, + "revision": target, + "content": output[:MAX_OUTPUT], + "truncated": len(output) > MAX_OUTPUT, + } diff --git a/plugins/home/src/reme_home/code/snapshot.py b/plugins/home/src/reme_home/code/snapshot.py new file mode 100644 index 00000000..843bb966 --- /dev/null +++ b/plugins/home/src/reme_home/code/snapshot.py @@ -0,0 +1,143 @@ +"""Bounded navigation of immutable Git objects, independent of the checkout.""" + +import json +from pathlib import Path +import subprocess +import sys + +from ..preparation.code import git, git_bytes +from ..preparation.spec import relative_path +from .repository import MAX_OUTPUT, resolve_revision + +MAX_FILE_BYTES = 5_000_000 +MAX_SEARCH_BYTES = 100_000_000 +MAX_READ_LINES = 300 + + +def code_path(path: str) -> str: + """Accept literal repository-relative paths, never Git pathspec expressions.""" + value = relative_path(path) + if ".git" in Path(value).parts or "\0" in value: + raise ValueError("Invalid code path") + return value + + +def tree(repository: Path, revision: str, path: str | None = None) -> list[tuple[str, str]]: + """Read regular-file object IDs with NUL-delimited names and no archive filters.""" + # ls-tree does not support pathspec magic. Filter exact names/prefixes ourselves. + entries = git(repository, "ls-tree", "-r", "-z", "--full-tree", revision, strip=False) + prefix = code_path(path) if path is not None else None + result = [] + for entry in filter(None, entries.split("\0")): + header, name = entry.split("\t", 1) + if prefix is not None and name != prefix and not name.startswith(prefix + "/"): + continue + mode, kind, oid = header.split() + code_path(name) + if mode not in {"100644", "100755"} or kind != "blob": + raise ValueError(f"Code snapshot must contain regular files only: {name}") + result.append((name, oid)) + return result + + +def blob(repository: Path, oid: str) -> bytes: + """Check blob size before loading its bytes; never follow tracked symlinks.""" + if int(git(repository, "cat-file", "-s", oid)) > MAX_FILE_BYTES: + raise ValueError("Code file exceeds 5 MB; narrow the path") + return git_bytes(repository, "cat-file", "blob", oid) + + +def list_code_tree( + repository: Path, + _record: dict, + *, + revision: str = "HEAD", + path: str | None = None, + max_results: int = 500, +) -> dict: + """List tracked paths at a resolved commit.""" + revision = resolve_revision(repository, revision) + entries = tree(repository, revision, path) + return { + "revision": revision, + "path": path, + "paths": [name for name, _ in entries[:max_results]], + "truncated": len(entries) > max_results, + } + + +def read_code_file( + repository: Path, + _record: dict, + *, + path: str, + revision: str = "HEAD", + start_line: int = 1, + end_line: int | None = None, +) -> dict: + """Return at most 300 lines and 100,000 characters from one committed text file.""" + path = code_path(path) + revision = resolve_revision(repository, revision) + entries = dict(tree(repository, revision, path)) + if path not in entries: + raise ValueError(f"No tracked regular file at {path}") + data = blob(repository, entries[path]) + if b"\0" in data: + raise ValueError("Code file is binary") + lines = data.decode("utf-8", errors="replace").splitlines() + requested_end = end_line if end_line is not None else start_line + MAX_READ_LINES - 1 + if requested_end < start_line: + raise ValueError("end_line must be greater than or equal to start_line") + displayed_end = min(requested_end, start_line + MAX_READ_LINES - 1, len(lines)) + content = "\n".join(lines[start_line - 1 : displayed_end]) + if content: + content += "\n" + truncated = len(content) > MAX_OUTPUT or (bool(content) and (start_line > 1 or displayed_end < len(lines))) + content = content[:MAX_OUTPUT] + if truncated: + content += "\n" + return { + "revision": revision, + "path": path, + "start_line": start_line, + "end_line": displayed_end, + "total_lines": len(lines), + "truncated": truncated, + "content": content, + } + + +def search_code( + repository: Path, + _record: dict, + *, + pattern: str, + revision: str = "HEAD", + path: str | None = None, + max_results: int = 30, +) -> dict: + """Search with Python multiline regex; isolate regex execution under a hard subprocess timeout.""" + revision = resolve_revision(repository, revision) + files = {} + total = 0 + for name, oid in tree(repository, revision, path): + size = int(git(repository, "cat-file", "-s", oid)) + total += size + if total > MAX_SEARCH_BYTES: + raise ValueError("Code search exceeds 100 MB; narrow path") + data = blob(repository, oid) + if b"\0" not in data: + files[name] = data.decode("utf-8", errors="replace") + # Arbitrary Python regex can backtrack indefinitely. Run only the text matcher + # in a short-lived stdlib-only process; it never imports or executes candidate code. + result = subprocess.run( + [sys.executable, "-I", str(Path(__file__).with_name("regex_worker.py"))], + input=json.dumps({"files": files, "pattern": pattern, "max_results": max_results}).encode(), + capture_output=True, + check=True, + timeout=10, + ) + payload = json.loads(result.stdout) + if "error" in payload: + raise ValueError(payload["error"]) + return {"revision": revision, "pattern": pattern, "path": path, **payload} diff --git a/plugins/home/src/reme_home/code/step.py b/plugins/home/src/reme_home/code/step.py new file mode 100644 index 00000000..70255005 --- /dev/null +++ b/plugins/home/src/reme_home/code/step.py @@ -0,0 +1,22 @@ +"""Thin Job adapters; operations are fixed by Step configuration, never model input.""" + +from reme.steps.base_step import BaseStep, Ref + +from .component import HomeCode +from .models import ARGUMENTS + + +class HomeCodeStep(BaseStep): + """Expose one configured candidate operation using the normal ReMe Job lifecycle.""" + + code: HomeCode = Ref(HomeCode, "home.code") + + async def execute(self): + """Leave repository location and operation selection under host control.""" + if self.context.get("_allowed_paths") is not None: + raise ValueError("Home code Jobs require unrestricted access to their managed workspace") + operation = self.kwargs["operation"] + fields = ARGUMENTS[operation].model_fields + arguments = {name: self.context.get(name) for name in fields if name in self.context} + self.context.response.answer = await self.code.run(operation, **arguments) + return self.context.response diff --git a/plugins/home/src/reme_home/evaluation/component.py b/plugins/home/src/reme_home/evaluation/component.py index 01cfc0ed..13ecba99 100644 --- a/plugins/home/src/reme_home/evaluation/component.py +++ b/plugins/home/src/reme_home/evaluation/component.py @@ -4,6 +4,7 @@ import asyncio import os import shutil import traceback +from contextlib import nullcontext from deploy.benchmark_docker import BenchmarkDockerPool, CasePlan, DockerReMeSandboxFactory, SourceCandidate @@ -22,10 +23,12 @@ from .artifacts import ( answer_exception, failure_kind, ) -from .inputs import EvaluationSpec, load_inputs, reuse_inputs, snapshot_code +from .inputs import EvaluationSpec, load_inputs, reuse_inputs, snapshot_code, replay_sampling_eligibility from .compaction import compact_construction, compact_query from ..judge import HomeJudge from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory +from ..sampling import SamplingPolicy, create_sampling_plan +from .history import HomeEvaluationHistory async def blocking(function, *args, **kwargs): @@ -43,11 +46,15 @@ class HomeEvaluation(BaseComponent): component_type = "home.evaluation" - def __init__(self, **kwargs): + def __init__(self, sampling_policy: dict | None = None, **kwargs): + if "sampling" in kwargs: + raise ValueError("Use sampling_policy for Component defaults; sampling is a Job boolean") super().__init__(**kwargs) self.workspace = self.bind("default", HomeWorkspace, optional=False) self.judge = self.bind("default", HomeJudge, optional=False) + self.history = self.bind("default", HomeEvaluationHistory, optional=False) self._pending: set[asyncio.Task] = set() + self.sampling_policy = SamplingPolicy.model_validate(sampling_policy if sampling_policy is not None else {}) async def _close(self): if self._pending: @@ -58,7 +65,10 @@ class HomeEvaluation(BaseComponent): if not self.is_started: raise RuntimeError("Start the Home application before evaluation") spec = EvaluationSpec.model_validate(kwargs) - task = asyncio.create_task(self._evaluate(spec)) + override = spec.sampling_policy.model_dump(exclude_unset=True) if spec.sampling_policy is not None else {} + policy = SamplingPolicy.model_validate({**self.sampling_policy.model_dump(), **override}) + spec = spec.model_copy(update={"sampling_policy": policy}) + task = asyncio.create_task(self._serialized_evaluate(spec)) self._pending.add(task) try: return await asyncio.shield(task) @@ -69,11 +79,40 @@ class HomeEvaluation(BaseComponent): finally: self._pending.discard(task) + async def _serialized_evaluate(self, spec): + """Match dev_meta's serialized formal validations; debug never owns sampling state.""" + _validate_directory(self.workspace_path.resolve(), self.workspace.state_path) + lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock") + with lock: + return await self._evaluate(spec) + def _prepare(self, spec, environment): home = self.workspace.home_path + sampling_state = self.history.sampling_state() if spec.sampling and not spec.debug else None _validate_directory(self.workspace_path.resolve(), self.workspace.state_path) with _initialization_lock(self.workspace.state_path / "workspace.lock"): record, dataset, cases = load_inputs(home, spec) + plan = None + if spec.sampling and not spec.debug: + eligibility = replay_sampling_eligibility(home, spec, cases) + plan = create_sampling_plan( + cases, + sampling_state, + spec.sampling_policy, + eligible_query_ids=eligibility, + ) + selected = {case.case_id: case for case in cases} + cases = [] + for key in plan["case_ids"]: + case = selected[key] + queries = {query.query_id: query for query in case.queries} + cases.append( + case.model_copy( + update={ + "queries": [queries[query_id] for query_id in plan["query_ids"][key]], + }, + ), + ) judge_identity = self.judge.identity(dataset.dataset) snapshot, commit, branch = snapshot_code(home, record) fingerprints = { @@ -102,7 +141,22 @@ class HomeEvaluation(BaseComponent): if not isinstance(rubric, list) or not rubric: raise ValueError(f"BEAM query {query.query_id} has no scoring rubric") evaluation_query(dataset.dataset, query) - return record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs + return record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs, plan + + def _sampling_started(self, run, manifest, plan, policy=None, *, record_history=True): + """The factory invokes this only after a real sandbox successfully initializes.""" + if not record_history: + manifest["sandbox_started_at"] = utc_now().isoformat() + if plan is not None: + # Keep draw evidence local without claiming a started-history round. + manifest["selection_plan"] = plan + write_json(run / "manifest.json", manifest) + return + self.history.record_started(run, policy if policy is not None else self.sampling_policy, plan) + manifest.update(read_json(run / "manifest.json")) + + def _finish_sampling(self, run): + self.history.record(run) def _allocate(self, branch, commit, manifest): """Allocate ascending IDs under a collision-safe commit prefix.""" @@ -130,7 +184,7 @@ class HomeEvaluation(BaseComponent): async def _evaluate(self, spec): environment = {**os.environ, **self.app_context.app_config.environment} prepared = await blocking(self._prepare, spec, environment) - record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs = prepared + record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs, sampling_plan = prepared judge = None if spec.debug else self.judge run = None try: @@ -199,6 +253,17 @@ class HomeEvaluation(BaseComponent): plugins=tuple(record["code"]["plugins"]), command_timeout=spec.timeout_seconds, env=environment, + on_sandbox_started=( + None + if spec.debug + else lambda: self._sampling_started( + run, + manifest, + sampling_plan, + spec.sampling_policy, + record_history=spec.record_history, + ) + ), ) semaphore = asyncio.Semaphore(spec.concurrency) by_id = {case.case_id: case for case in cases} @@ -242,20 +307,34 @@ class HomeEvaluation(BaseComponent): previous.update(cleanup_error=result.error, failure_kind="infra_error") write_json(failure, previous) summary = summarize(run, cases, scoring_required=not spec.debug) - manifest.update(status="completed", finished_at=utc_now().isoformat()) + manifest.update( + status="completed", + success=summary["case_status_counts"]["infra_error"] == 0, + finished_at=utc_now().isoformat(), + ) write_json(run / "manifest.json", manifest) write_json(run / "summary.json", summary, compact_leaf_containers=True) + if not spec.debug and spec.record_history: + await blocking(self._finish_sampling, run) return { "result_dir": str(run), "commit_sha": commit, "execution_mode": manifest["execution_mode"], + "record_history": spec.record_history and not spec.debug, "status": "completed", **{key: value for key, value in summary.items() if key != "cases"}, } except BaseException as error: + # A full summary is the durable completion boundary. Cancellation + # during drained history bookkeeping cancels the caller, not the + # already completed evaluation or its successful history entry. + registration_failed = run is not None and manifest.get("status") == "completed" + if isinstance(error, asyncio.CancelledError) and registration_failed: + raise if run is not None: manifest.update( status="cancelled" if isinstance(error, asyncio.CancelledError) else "failed", + success=False, finished_at=utc_now().isoformat(), ) write_json(run / "manifest.json", manifest) @@ -274,6 +353,11 @@ class HomeEvaluation(BaseComponent): scoring_required=not spec.debug, ) write_json(run / "partial_summary.json", partial, compact_leaf_containers=True) + if registration_failed and not spec.debug and spec.record_history: + try: + await blocking(self.history.invalidate) + except Exception as cache_error: + error.add_note(f"History cache invalidation failed: {cache_error}") if isinstance(error, asyncio.CancelledError) or run is None: raise raise RuntimeError(f"Evaluation failed; artifacts: {run}: {error}") from error diff --git a/plugins/home/src/reme_home/evaluation/history.py b/plugins/home/src/reme_home/evaluation/history.py new file mode 100644 index 00000000..85e42e6d --- /dev/null +++ b/plugins/home/src/reme_home/evaluation/history.py @@ -0,0 +1,345 @@ +"""Successful evaluation history and rebuildable sampling evidence owned by Home.""" + +from pathlib import Path +from typing import Mapping, Literal +import math + +from pydantic import Field + +from reme.components.base_component import BaseComponent + +from ..preparation.data.models import utc_now +from ..preparation.spec import SpecModel +from ..sampling import SamplingPolicy, _empty_record, _observe, _query_key +from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory +from .artifacts import contained, read_json, write_json + + +class EvaluationRecord(SpecModel): + """A compact catalog entry backed by an immutable successful result directory.""" + + run_key: str + branch_name: str + commit_sha: str + validation_id: str + round: int + case_ids: list[str] + query_ids: dict[str, list[str]] + execution_mode: str + finished_at: str + case_count: int + candidate_failure_count: int + scored_query_count: int + mean_query_score: float | None + + +class HistoryIndex(SpecModel): + """A disposable index; manifests, summaries and plans remain authoritative.""" + + schema_version: Literal[1] = 1 + results: list[EvaluationRecord] = Field(default_factory=list) + + +def successful_result(home: Path, run: Path) -> tuple[dict, dict] | None: + """Match the non-debug Response.success contract, including valid low scores.""" + manifest = read_json(contained(home, run / "manifest.json")) + if manifest.get("options", {}).get("record_history") is False: + return None + if ( + manifest.get("mode") != "scored" + or manifest.get("dataset_kind") != "training-set" + or manifest.get("status") != "completed" + or manifest.get("success") is False + or not manifest.get("sandbox_started_at") + ): + return None + path = contained(home, run / "summary.json") + if not path.is_file(): + return None + summary = read_json(path) + if summary.get("case_status_counts", {}).get("infra_error") != 0 or not summary.get("case_count"): + return None + return manifest, summary + + +def result_record(home: Path, run: Path) -> EvaluationRecord | None: + """Exclude failed, cancelled, debug and incomplete directories from the catalog.""" + result = successful_result(home, run) + if result is None: + return None + manifest, summary = result + return EvaluationRecord( + run_key=run.relative_to(home / "evaluations").as_posix(), + branch_name=manifest["branch_name"], + commit_sha=manifest["commit_sha"], + validation_id=manifest["validation_id"], + round=manifest["sampling"]["round"], + case_ids=manifest["case_ids"], + query_ids=manifest["query_ids"], + execution_mode=manifest["execution_mode"], + finished_at=manifest["finished_at"], + case_count=summary["case_count"], + candidate_failure_count=summary["case_status_counts"]["candidate_failure"], + scored_query_count=summary["scored_query_count"], + mean_query_score=summary["mean_query_score"], + ) + + +def read_state(home: Path) -> dict: + """Read a derived cache; started plans and scored artifacts retain its source evidence.""" + path = contained(home, "sampling/state.json") + if not path.exists(): + return rebuild_sampling_state(home, persist=False) + payload = read_json(path) + if not isinstance(payload, dict) or payload.get("schema_version") not in {1, 2}: + raise ValueError("Unsupported sampling state schema") + if not isinstance(payload.get("round"), int) or isinstance(payload["round"], bool) or payload["round"] < 0: + raise ValueError("Invalid sampling round") + for name in ("cases", "queries", "started_validations"): + if not isinstance(payload.get(name, {}), dict): + raise ValueError(f"Invalid sampling {name}") + payload.setdefault(name, {}) + if not isinstance(payload.get("processed_validations"), list): + raise ValueError("Invalid processed validations") + if payload["schema_version"] == 1 or "successful_round" not in payload: + return rebuild_sampling_state(home, persist=False) + if ( + not isinstance(payload["successful_round"], int) + or isinstance(payload["successful_round"], bool) + or (payload["successful_round"] < 0) + ): + raise ValueError("Invalid successful-history round") + return payload + + +def start_validation(home: Path, run: Path, policy: SamplingPolicy, plan: dict | None = None) -> dict: + """Record once after the first successfully initialized sandbox, never before.""" + manifest = read_json(contained(home, run / "manifest.json")) + if ( + manifest.get("mode") == "debug" + or manifest.get("dataset_kind") != "training-set" + or manifest.get("options", {}).get("record_history") is False + ): + return {} + key = run.relative_to(home / "evaluations").as_posix() + state = read_state(home) + if key in state["started_validations"]: + return read_json(contained(home, manifest["sampling"]["plan_path"])) + plan = ( + dict(plan) + if plan is not None + else { + "schema_version": 1, + "mode": "explicit", + "case_ids": manifest["case_ids"], + "query_ids": manifest["query_ids"], + "policy": policy.model_dump(), + } + ) + # Preserve the draw's state snapshot separately from the committed start round. + if "round" in plan: + plan["selection_round"] = plan["round"] + plan.update( + round=state["round"] + 1, + validation_id=manifest["validation_id"], + validation_key=key, + sandbox_started_at=utc_now().isoformat(), + ) + relative = f"sampling/plans/{key}.json" + write_json(contained(home, relative), plan) + manifest["sampling"] = {"round": plan["round"], "mode": plan["mode"], "plan_path": relative} + manifest["sandbox_started_at"] = plan["sandbox_started_at"] + write_json(run / "manifest.json", manifest) + state["round"] = plan["round"] + state["started_validations"][key] = plan["round"] + write_json(contained(home, "sampling/state.json"), state) + return plan + + +def update_sampling_state(home: Path, run: Path, *, _state: dict | None = None) -> None: + """Update observations only from successful completed formal evaluations.""" + result = successful_result(home, run) + if result is None: + return + manifest, summary = result + plan = read_json(contained(home, manifest["sampling"]["plan_path"])) + policy = SamplingPolicy.model_validate(plan["policy"]) + state = read_state(home) if _state is None else _state + key = plan["validation_key"] + if key in state["processed_validations"]: + return + round_id = state["successful_round"] + 1 + cases, queries = state["cases"], state["queries"] + summary_cases = summary.get("cases", {}) + items = ( + summary_cases.items() + if isinstance(summary_cases, Mapping) + else ((case.get("case_id"), case) for case in summary_cases) + ) + for case_id, case_summary in items: + if not isinstance(case_id, str) or not isinstance(case_summary, Mapping): + continue + folder = contained(run, f"cases/{case_id}", directory=True) + query_path = contained(folder, "queries/result.json") + query_rows = read_json(query_path).get("queries", []) if query_path.is_file() else [] + failure_path = contained(folder, "failure.json") + failure = read_json(failure_path) if failure_path.is_file() else {} + # Home summaries omit the redundant case status; reconstruct it using + # the same authoritative failure/query evidence as summarize(). + status = case_summary.get("status") or ( + "infra_error" + if failure.get("failure_kind") == "infra_error" + or any(query.get("infrastructure_error") for query in query_rows) + else "candidate_failure" if failure or any(query.get("error") for query in query_rows) else "completed" + ) + case_record = cases.setdefault(case_id, _empty_record()) + case_record["last_selected_round"] = max(case_record.get("last_selected_round") or 0, round_id) + if status == "infra_error": + case_record["infra_error_count"] += 1 + continue + if status == "candidate_failure": + case_record["candidate_failure_count"] += 1 + scores = [] + for query in query_rows: + query_id = query.get("query_id") + if not isinstance(query_id, str): + continue + query_record = queries.setdefault(_query_key(case_id, query_id), _empty_record()) + query_record["last_selected_round"] = max(query_record.get("last_selected_round") or 0, round_id) + if query.get("infrastructure_error"): + query_record["infra_error_count"] += 1 + continue + if query.get("failure_kind") == "candidate_failure": + query_record["candidate_failure_count"] += 1 + score = query.get("score") + if isinstance(score, (int, float)) and not isinstance(score, bool) and math.isfinite(score): + scores.append(float(score)) + _observe(query_record, 1.0 - score, round_id, policy.ewma_alpha) + if scores: + _observe(case_record, 1.0 - sum(scores) / len(scores), round_id, policy.ewma_alpha) + state["successful_round"] = round_id + state["processed_validations"].append(key) + if _state is None: + write_json(contained(home, "sampling/state.json"), state) + + +def rebuild_sampling_state(home: Path, *, persist: bool = True) -> dict: + """Rebuild the disposable cache from immutable started plans and scored artifacts.""" + state = { + "schema_version": 2, + "round": 0, + "successful_round": 0, + "cases": {}, + "queries": {}, + "started_validations": {}, + "processed_validations": [], + } + runs = [] + for path in contained(home, "evaluations", directory=True).glob("*/*/*/manifest.json"): + manifest = read_json(contained(home, path)) + if ( + manifest.get("mode") == "debug" + or manifest.get("dataset_kind") != "training-set" + or manifest.get("options", {}).get("record_history") is False + or not (manifest.get("sandbox_started_at")) + ): + continue + plan = read_json(contained(home, manifest["sampling"]["plan_path"])) + runs.append((plan["round"], path.parent, plan["validation_key"])) + for round_id, run, key in sorted(runs): + state["round"] = max(state["round"], round_id) + state["started_validations"][key] = round_id + update_sampling_state(home, run, _state=state) + if persist: + write_json(contained(home, "sampling/state.json"), state) + return state + + +class HomeEvaluationHistory(BaseComponent): + """Own successful-result lookup, registration and sampling caches for one workspace.""" + + component_type = "home.evaluation_history" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.workspace = self.bind("default", HomeWorkspace, optional=False) + + def _history_lock(self): + if not self.is_started: + raise RuntimeError("Start the Home application before reading evaluation history") + _validate_directory(self.workspace_path.resolve(), self.workspace.state_path) + return _initialization_lock(self.workspace.state_path / "workspace.lock") + + def _run_path(self, run: Path) -> Path: + home = self.workspace.home_path + run = contained(home, run, directory=True) + relative = run.relative_to(home / "evaluations") + if len(relative.parts) != 3: + raise ValueError("Expected one evaluation result directory") + return run + + def _index(self, *, rebuild: bool = False) -> HistoryIndex: + home = self.workspace.home_path + path = contained(home, "evaluations/history.json") + if path.exists() and not rebuild: + return HistoryIndex.model_validate(read_json(path)) + records = [] + for manifest in contained(home, "evaluations", directory=True).glob("*/*/*/manifest.json"): + record = result_record(home, contained(home, manifest.parent, directory=True)) + if record is not None: + records.append(record) + return HistoryIndex(results=sorted(records, key=lambda record: (record.round, record.run_key))) + + def sampling_state(self) -> dict: + """Return owned, reconstructed-on-demand observations for the next draw.""" + with self._history_lock(): + state = read_state(self.workspace.home_path) + return {"round": state["successful_round"], "cases": state["cases"], "queries": state["queries"]} + + def record_started(self, run: Path, policy: SamplingPolicy, plan: dict | None = None) -> dict: + """Record a started round, independently of whether it later enters successful history.""" + with self._history_lock(): + return start_validation(self.workspace.home_path, self._run_path(run), policy, plan) + + def record(self, run: Path) -> EvaluationRecord | None: + """Register a successful result once; unsuccessful runs never train the sampler.""" + with self._history_lock(): + home = self.workspace.home_path + run = self._run_path(run) + record = result_record(home, run) + if record is None: + return None + index = self._index() + by_key = {item.run_key: item for item in index.results} + by_key[record.run_key] = record + update_sampling_state(home, run) + index = HistoryIndex(results=sorted(by_key.values(), key=lambda item: (item.round, item.run_key))) + write_json(contained(home, "evaluations/history.json"), index.model_dump(mode="json")) + return record + + def list_results(self) -> list[EvaluationRecord]: + """Return successful results in chronological order, including low-scoring candidates.""" + with self._history_lock(): + return self._index().results + + def get_result(self, run_key: str) -> EvaluationRecord: + """Look up a successful result by its stable workspace-relative identity.""" + for result in self.list_results(): + if result.run_key == run_key: + return result + raise KeyError(f"No successful evaluation: {run_key}") + + def rebuild(self) -> list[EvaluationRecord]: + """Rebuild only derived indexes and sampling observations; never rewrite source evidence.""" + with self._history_lock(): + home = self.workspace.home_path + index = self._index(rebuild=True) + rebuild_sampling_state(home) + write_json(contained(home, "evaluations/history.json"), index.model_dump(mode="json")) + return index.results + + def invalidate(self) -> None: + """Discard only derived caches after an unsuccessful registration; retain all source evidence.""" + with self._history_lock(): + for name in ("evaluations/history.json", "sampling/state.json"): + contained(self.workspace.home_path, name).unlink(missing_ok=True) diff --git a/plugins/home/src/reme_home/evaluation/inputs.py b/plugins/home/src/reme_home/evaluation/inputs.py index 23d7a4c1..41240474 100644 --- a/plugins/home/src/reme_home/evaluation/inputs.py +++ b/plugins/home/src/reme_home/evaluation/inputs.py @@ -16,6 +16,8 @@ from ..preparation.code import git, tree_fingerprint from ..preparation.data.models import CaseSpec, DatasetManifest, fingerprint from .artifacts import contained, file_sha256, inspect_archive, read_json from ..preparation.spec import SpecModel +from ..code.repository import validate_repository +from ..sampling import SamplingPolicy class EvaluationSpec(SpecModel): @@ -25,6 +27,9 @@ class EvaluationSpec(SpecModel): query_ids: dict[str, list[str]] | None = None reuse_construction_from: str | None = Field(default=None, min_length=1) debug: bool = False + sampling: bool = False + record_history: bool = True + sampling_policy: SamplingPolicy | None = None concurrency: int = Field(default=4, ge=1) timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False) base_image: str = Field(default="reme-benchmark-base:agentscope-2.0.10", min_length=1) @@ -64,6 +69,8 @@ class EvaluationSpec(SpecModel): """Debug stops on failure by default and never reuses scored evidence.""" if self.debug and self.reuse_construction_from: raise ValueError("Construction reuse requires training-set") + if self.sampling and not self.debug and (self.case_ids is not None or self.query_ids is not None): + raise ValueError("sampling=true selects case/query IDs automatically; omit explicit selections") if "fail_fast" not in self.model_fields_set: self.fail_fast = self.debug return self @@ -119,7 +126,7 @@ def load_inputs(home: Path, spec: EvaluationSpec): def snapshot_code(home: Path, record: dict): """Combine fixed runtime with clean current HEAD; reject all uncommitted changes.""" - repository = contained(home, "code/repo", directory=True) + repository, _ = validate_repository(home, record) runtime = contained(home, "code/runtime", directory=True) if tree_fingerprint(runtime) != record["code"]["runtime_fingerprint"]: raise ValueError("Fixed runtime has changed since preparation") @@ -235,3 +242,49 @@ def reuse_inputs(home: Path, spec: EvaluationSpec, cases: list, fingerprints: di if errors: raise ValueError("Invalid construction snapshots: " + "; ".join(errors)) return {"result_dir": str(source), "commit_sha": manifest["commit_sha"]}, archives + + +def replay_sampling_eligibility(home: Path, spec: EvaluationSpec, cases: list[CaseSpec]) -> dict | None: + """Port dev_meta's replay eligibility without restricting future full rebuilds.""" + if spec.reuse_construction_from is None: + return None + source = contained(home, spec.reuse_construction_from, directory=True) + evaluations = contained(home, "evaluations", directory=True) + if not source.is_relative_to(evaluations) or source == evaluations: + raise ValueError("reuse_construction_from must be inside home/evaluations") + manifest = read_json(contained(source, "manifest.json")) + summary = read_json(contained(source, "summary.json")) + if ( + manifest.get("dataset_kind") != "training-set" + or manifest.get("mode") == "debug" + or (manifest.get("status", summary.get("status")) != "completed") + ): + raise ValueError("Replay sampling requires a completed scored training-set run") + case_ids = manifest.get("case_ids") + if not isinstance(case_ids, list) or not case_ids or len(set(case_ids)) != len(case_ids): + raise ValueError("Replay source must select unique nonempty cases") + reusable = set() + for case_id in case_ids: + entity_id(case_id) + folder = contained(source, f"cases/{case_id}", directory=True) + result_path = contained(folder, "memory_construction/result.json") + archive = contained(folder, "memory_construction/reme_workspace.tar.gz") + if not result_path.is_file() or not archive.is_file() or read_json(result_path).get("success") is not True: + continue + reusable.add(case_id) + if (len(case_ids) - len(reusable)) / len(case_ids) > 0.05: + raise ValueError("Replay source construction failure ratio exceeds 5%") + for case_id in reusable: + payload = read_json(contained(source, f"cases/{case_id}/queries/result.json")) + queries = payload.get("queries") + if not queries or not all( + isinstance(query, dict) + and query.get("error") is None + and isinstance(query.get("answer"), str) + and query["answer"].strip() + for query in queries + ): + raise ValueError(f"Replay source has no completed successful answers for {case_id}") + # A construction snapshot can answer new query subsets from the same case; + # dev_meta intentionally considers every installed query of a reusable case. + return {case.case_id: [query.query_id for query in case.queries] for case in cases if case.case_id in reusable} diff --git a/plugins/home/src/reme_home/evaluation/inspection_step.py b/plugins/home/src/reme_home/evaluation/inspection_step.py new file mode 100644 index 00000000..c0ecd488 --- /dev/null +++ b/plugins/home/src/reme_home/evaluation/inspection_step.py @@ -0,0 +1,358 @@ +"""Read-only evaluation navigation Jobs, independent of optimization history.""" + +import asyncio +import json + +from pydantic import Field, field_validator, model_validator + +from reme.steps.base_step import BaseStep, Ref + +from ..code.repository import current_branch, resolve_revision, validate_branch, validate_repository +from ..preparation.code import git +from ..preparation.spec import SpecModel +from ..workspace import HomeWorkspace, _initialization_lock +from .artifacts import contained, read_json +from .inputs import entity_id + +TOTAL_FIELDS = ( + "schema_version", + "scoring_policy", + "case_count", + "case_status_counts", + "construction_status_counts", + "failure_stage_case_counts", + "expected_query_count", + "scored_query_count", + "query_status_counts", + "mean_query_score", + "observed_mean_query_score", +) + + +class ListValidationResultsInput(SpecModel): + """Filters affect catalog navigation, never which evaluations enter history.""" + + branch_name: str | None = None + commit_sha: str | None = Field(default=None, pattern=r"^[0-9a-fA-F]+$") + offset: int = Field(default=0, ge=0) + limit: int = Field(default=20, ge=1, le=100) + + @field_validator("branch_name") + @classmethod + def validate_branch_name(cls, value): + """Keep historical branch filters within one directory segment.""" + return entity_id(value) if value is not None else None + + +class InspectValidationInput(SpecModel): + """A query belongs to a case and every inspection belongs to one run.""" + + validation_path: str = Field(min_length=1) + case_id: str | None = None + query_id: str | None = None + offset: int = Field(default=0, ge=0) + limit: int = Field(default=20, ge=1, le=100) + + @field_validator("case_id", "query_id") + @classmethod + def validate_entity(cls, value): + """Reject path traversal before resolving any artifact.""" + return entity_id(value) if value is not None else None + + @model_validator(mode="after") + def validate_selection(self): + """Require a case when selecting one query.""" + if self.query_id is not None and self.case_id is None: + raise ValueError("query_id requires case_id") + return self + + +def _preview(value, limit=600): + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + return value if len(text) <= limit else text[:limit] + "… [truncated; read source field]" + + +def _pointer(value): + return str(value).replace("~", "~0").replace("/", "~1") + + +def _page(rows, spec): + return { + "row_count": len(rows), + "rows": rows[spec.offset : spec.offset + spec.limit], + "next_offset": spec.offset + spec.limit if spec.offset + spec.limit < len(rows) else None, + } + + +def _object(root, relative, *, required=False): + path = contained(root, relative) + if not path.exists() and not required: + return None + payload = read_json(path) + if not isinstance(payload, dict): + raise ValueError(f"Expected JSON object: {path}") + return payload + + +def _summary(run): + for name in ("summary.json", "partial_summary.json"): + value = _object(run, name) + if value is not None: + return value, str(run / name) + return None, None + + +def _run_path(home, value): + run = contained(home, value, directory=True) + relative = run.relative_to(home / "evaluations") + if len(relative.parts) != 3 or not run.is_dir(): + raise ValueError("validation_path must identify one directory inside home/evaluations") + return run + + +def _commit_filter(home, spec): + if spec.commit_sha is None: + return spec.branch_name, None + # Resolve a local branch first, then match ALL of its reachable commits. + # Git's global abbreviation rules could otherwise reject a branch-unique prefix. + with _initialization_lock(contained(home, "metadata/workspace.lock")): + repo, _ = validate_repository(home) + branch = spec.branch_name if spec.branch_name is not None else current_branch(repo) + validate_branch(repo, branch) + tip = resolve_revision(repo, "refs/heads/" + branch) + prefix = spec.commit_sha.lower() + matches = [sha for sha in git(repo, "rev-list", tip, "--").splitlines() if sha.startswith(prefix)] + if len(matches) != 1: + raise ValueError(f"commit_sha prefix {prefix!r} has {len(matches)} matches on branch {branch!r}; require one") + return branch, matches[0] + + +def _catalog_row(run, manifest): + summary, summary_path = _summary(run) + options = manifest.get("options", {}) + row = { + "validation_path": str(run), + "manifest_path": str(run / "manifest.json"), + **{ + key: manifest.get(key) + for key in ( + "branch_name", + "commit_sha", + "validation_id", + "status", + "success", + "started_at", + "finished_at", + "dataset", + "dataset_kind", + "mode", + "execution_mode", + "construction_source", + "fingerprints", + ) + }, + "sampling": options.get("sampling", False), + "record_history": options.get("record_history", True) and manifest.get("mode") != "debug", + "selected_case_count": len(manifest.get("case_ids", [])), + "selected_query_count": sum(len(ids) for ids in manifest.get("query_ids", {}).values()), + "summary_path": summary_path, + "summary_missing": summary is None, + "totals": {key: summary[key] for key in TOTAL_FIELDS if key in summary} if summary else {}, + } + failure = _object(run, "failure.json") + row["failure"] = _preview(failure) + row["failure_path"] = str(run / "failure.json") if failure is not None else None + return row + + +def _list_results(home, spec): + branch, commit = _commit_filter(home, spec) + evaluations = contained(home, "evaluations", directory=True) + rows = [] + for path in evaluations.glob("*/*/*/manifest.json"): + if branch is not None and path.parent.parent.parent.name != branch: + continue + row = {"validation_path": str(path.parent), "manifest_path": str(path)} + try: + run = _run_path(home, path.parent) + manifest = _object(run, "manifest.json", required=True) + row.update({key: manifest.get(key) for key in ("branch_name", "commit_sha", "started_at")}) + if commit is not None and row["commit_sha"] != commit: + continue + row = _catalog_row(run, manifest) + except (ValueError, OSError, TypeError, KeyError, AttributeError) as error: + row["read_error"] = _preview(str(error)) + if commit is None or row.get("commit_sha") == commit: + rows.append(row) + rows.sort(key=lambda row: (str(row.get("started_at") or ""), row["validation_path"]), reverse=True) + return {"branch_name": branch, "commit_sha": commit, **_page(rows, spec)} + + +def _case_rows(manifest, summary): + cases = (summary or {}).get("cases", {}) + indexed = ( + {key: (value, f"/cases/{_pointer(key)}") for key, value in cases.items()} + if isinstance(cases, dict) + else {value["case_id"]: (value, f"/cases/{index}") for index, value in enumerate(cases)} + ) + rows = [] + for case_id in manifest.get("case_ids", []): + case, pointer = indexed.get(case_id, ({}, None)) + rows.append( + { + "case_id": case_id, + "mean_score": case.get("mean_score"), + "construction": case.get("construction"), + "construction_error_count": case.get("construction_error_count", 0), + "query_error_count": case.get("query_error_count", 0), + "query_tokens": case.get("average_query_total_token_usage"), + "missing": pointer is None, + "pointer": pointer, + }, + ) + rows.sort( + key=lambda row: ( + row["mean_score"] is not None, + row["mean_score"] if row["mean_score"] is not None else -1, + -row["construction_error_count"], + row["case_id"], + ), + ) + return rows + + +def _case_inspection(run, manifest, spec): + folder = contained(run, f"cases/{spec.case_id}", directory=True) + build_path, query_path = "memory_construction/result.json", "queries/result.json" + build, payload = _object(folder, build_path), _object(folder, query_path) + failures = [] + jobs = (build or {}).get("jobs", []) + for index, job in enumerate(jobs): + outcome = job.get("result") or {} + if outcome.get("success") is False: + failures.append( + { + "job": job.get("job"), + "error": _preview(outcome.get("error")), + "pointer": f"/jobs/{index}/result", + }, + ) + queries = (payload or {}).get("queries", []) + by_id = {query["query_id"]: (query, index) for index, query in enumerate(queries)} + selected = manifest["query_ids"][spec.case_id] + if spec.query_id is not None and spec.query_id not in selected: + raise ValueError("query_id is not selected by this evaluation manifest") + rows = [] + for query_id in ([spec.query_id] if spec.query_id is not None else selected): + query, index = by_id.get(query_id, ({}, None)) + row = { + "query_id": query_id, + "score": query.get("score"), + "question_type": query.get("question_type"), + "failure_kind": query.get("failure_kind"), + "error": _preview(query.get("error")), + "answer_success": (query.get("answer_result") or {}).get("success"), + "judge_success": (query.get("judge_result") or {}).get("success"), + "missing": index is None, + "pointer": f"/queries/{index}" if index is not None else None, + } + if spec.query_id is not None: + row["fields"] = { + key: _preview(query[key], 2000) + for key in ( + "question", + "golden_answer", + "answer", + "score", + "question_type", + "answer_result", + "judge_result", + "error", + "exception", + "failure_kind", + "infrastructure_error", + ) + if key in query + } + rows.append(row) + result = { + "case_id": spec.case_id, + "missing_files": [name for name, value in ((build_path, build), (query_path, payload)) if value is None], + "case_failure": _preview(_object(folder, "failure.json")), + "construction": { + "source": str(folder / build_path), + "success": (build or {}).get("success"), + "status": (build or {}).get("status"), + "job_count": len(jobs), + "failed_jobs": _page(failures, spec), + }, + "build_log": str(contained(folder, "memory_construction/build.log")), + "memory_workspace": str(contained(folder, "memory_construction/reme_workspace", directory=True)), + "rows_source": str(folder / query_path), + **_page(rows, spec), + } + if spec.query_id is not None: + query_folder = contained(folder, f"queries/{spec.query_id}", directory=True) + result.update( + query_missing=rows[0]["missing"], + answer_log=str(contained(query_folder, "answer.log")), + query_failure=_preview(_object(query_folder, "failure.json")), + ) + return result + + +def _inspect_result(home, spec): + run = _run_path(home, spec.validation_path) + manifest = _object(run, "manifest.json", required=True) + if spec.case_id is not None and spec.case_id not in manifest.get("case_ids", []): + raise ValueError("case_id is not selected by this evaluation manifest") + result = {"validation_path": str(run), "manifest_path": str(run / "manifest.json")} + if spec.case_id is not None: + return {**result, **_case_inspection(run, manifest, spec)} + summary, summary_path = _summary(run) + return { + **_catalog_row(run, manifest), + "rows_source": summary_path, + **_page(_case_rows(manifest, summary), spec), + } + + +async def _execute(step, model, function): + if step.context.get("_allowed_paths") is not None: + raise ValueError("Home inspection Jobs require unrestricted access to their managed evaluation artifacts") + spec = model.model_validate({name: step.context.get(name) for name in model.model_fields if name in step.context}) + if not step.workspace.is_started: + raise RuntimeError("Start the Home workspace before inspecting evaluations") + task = asyncio.create_task(asyncio.to_thread(function, step.workspace.home_path, spec)) + cancelled = False + while True: + try: + result = await asyncio.shield(task) + if cancelled: + raise asyncio.CancelledError + step.context.response.answer = result + return step.context.response + except asyncio.CancelledError: + if task.done(): + raise + cancelled = True + + +class HomeListValidationResultsStep(BaseStep): + """List all evaluation artifacts, including failures, debug and standalone runs.""" + + workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace") + + async def execute(self): + """Navigate runs without reading or rebuilding optimization history.""" + return await _execute(self, ListValidationResultsInput, _list_results) + + +class HomeInspectValidationStep(BaseStep): + """Inspect a run, a case or one query with bounded result previews.""" + + workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace") + + async def execute(self): + """Read only the requested result scope and return artifact coordinates.""" + return await _execute(self, InspectValidationInput, _inspect_result) diff --git a/plugins/home/src/reme_home/evaluation/memory_inspection_step.py b/plugins/home/src/reme_home/evaluation/memory_inspection_step.py new file mode 100644 index 00000000..abe830b2 --- /dev/null +++ b/plugins/home/src/reme_home/evaluation/memory_inspection_step.py @@ -0,0 +1,284 @@ +"""Read-only navigation and comparison of exported memory Git checkpoints.""" + +import difflib +import hashlib +import re +import subprocess + +from pydantic import Field, field_validator, model_validator + +from reme.steps.base_step import BaseStep, Ref + +from ..code.repository import MAX_OUTPUT, resolve_revision +from ..code.snapshot import blob, code_path, tree +from ..preparation.code import git as safe_git +from ..preparation.spec import SpecModel +from ..workspace import HomeWorkspace +from .inputs import entity_id +from .artifacts import contained +from .inspection_step import _execute, _object, _page, _run_path + +MAX_TREE_FILES = 2000 +MAX_TREE_BYTES = 20_000_000 + + +def git(repository, *arguments): + """Ignore replace refs while reusing the shared noninteractive Git executor.""" + return safe_git(repository, "--no-replace-objects", *arguments) + + +class MemoryScopeInput(SpecModel): + """An existing evaluation and one manifest-selected case.""" + + validation_path: str = Field(min_length=1) + case_id: str = Field(min_length=1) + path: str | None = None + + @field_validator("case_id") + @classmethod + def validate_case(cls, value): + """Accept a single case directory segment.""" + return entity_id(value) + + @field_validator("path") + @classmethod + def validate_path(cls, value): + """Accept literal repository-relative paths only.""" + return code_path(value) if value is not None else None + + +class MemoryHistoryInput(MemoryScopeInput): + """Page through checkpoints, newest first.""" + + offset: int = Field(default=0, ge=0) + limit: int = Field(default=20, ge=1, le=100) + + +class CompareMemoryInput(MemoryScopeInput): + """Compare two commits in one or two exported repositories.""" + + base_ref: str | None = Field(default=None, min_length=1) + target_ref: str = Field(default="HEAD", min_length=1) + target_validation_path: str | None = Field(default=None, min_length=1) + + @model_validator(mode="after") + def validate_base(self): + """Require an explicit earlier checkpoint for a same-repository comparison.""" + if self.target_validation_path is None and self.base_ref is None: + raise ValueError("base_ref is required when target_validation_path is omitted") + return self + + +def _repository(home, validation_path, case_id): + run = _run_path(home, validation_path) + manifest = _object(run, "manifest.json", required=True) + if case_id not in manifest.get("case_ids", []): + raise ValueError("case_id is not selected by this evaluation manifest") + repository = contained(run, f"cases/{case_id}/memory_construction/reme_workspace", directory=True) + return run, manifest, repository + + +def _head(repository): + """Reject indirect Git storage and distinguish absent/empty/corrupt repositories.""" + if not repository.exists(): + return "workspace_missing", None + metadata = contained(repository, ".git") + if not metadata.exists(): + return "history_missing", None + if not metadata.is_dir(): + raise ValueError("Memory history requires a standalone .git directory") + # Exported artifacts must not redirect reads to another repository or host path. + if any(path.is_symlink() for path in metadata.rglob("*")): + raise ValueError("Memory Git metadata must not contain symbolic links") + for name in ("commondir", "objects/info/alternates", "objects/info/http-alternates", "shallow"): + if (metadata / name).exists(): + raise ValueError(f"Unsupported indirect or incomplete memory history: {name}") + if any(metadata.glob("objects/pack/*.promisor")): + raise ValueError("Partial memory repositories are unsupported") + if git(repository, "for-each-ref", "--format=%(refname)", "refs/replace"): + raise ValueError("Memory replace refs are unsupported") + configuration = git(repository, "config", "--local", "--list") + if any(line.lower().startswith(("include.", "includeif.")) for line in configuration.splitlines()): + raise ValueError("Memory Git configuration must not include external files") + if any( + "promisor=" in line.lower() or line.lower().startswith("extensions.partialclone=") + for line in configuration.splitlines() + ): + raise ValueError("Partial memory repositories are unsupported") + if git(repository, "rev-parse", "--show-toplevel") != str(repository): + raise ValueError("Memory history must be a standalone repository") + if git(repository, "rev-parse", "--is-shallow-repository") != "false": + raise ValueError("Memory history must not be shallow") + try: + return "available", resolve_revision(repository, "HEAD") + except subprocess.CalledProcessError: + # Only an unborn symbolic HEAD is an empty repository, not a broken ref. + reference = git(repository, "symbolic-ref", "-q", "HEAD") + if git(repository, "for-each-ref", "--format=%(refname)", reference): + raise ValueError("Memory HEAD cannot be resolved") from None + if not (metadata / "objects").is_dir(): + raise ValueError("Memory object database is missing") from None + return "history_empty", None + + +def _resolve(repository, reference, head): + if re.fullmatch(r"[0-9a-fA-F]+", reference or ""): + matches = [ + sha for sha in git(repository, "rev-list", head, "--").splitlines() if sha.startswith(reference.lower()) + ] + if len(matches) != 1: + raise ValueError(f"Memory commit prefix has {len(matches)} matches reachable from HEAD; require one") + return matches[0] + revision = resolve_revision(repository, reference) + git(repository, "merge-base", "--is-ancestor", revision, head) + return revision + + +def inspect_history(home, spec): + """Return saved Git history without refreshing the worktree or index.""" + run, _manifest, repository = _repository(home, spec.validation_path, spec.case_id) + status, head = _head(repository) + result = { + "validation_path": str(run), + "case_id": spec.case_id, + "repository": str(repository), + "workspace_exists": repository.exists(), + "has_git_history": head is not None, + "status": status, + "head": head, + "path": spec.path, + } + if head is None: + return {**result, **_page([], spec)} + # Literal pathspecs prevent user-controlled wildcard/pathspec expansion. + path_args = ["--", ":(literal)" + spec.path] if spec.path is not None else ["--"] + count = int(git(repository, "rev-list", "--count", head, *path_args)) + raw = git( + repository, + "log", + "--no-show-signature", + f"--skip={spec.offset}", + f"--max-count={spec.limit}", + "--format=%H%x00%P%x00%aI%x00%s", + head, + *path_args, + ) + rows = [] + for line in raw.splitlines(): + sha, parents, timestamp, subject = line.split("\0", 3) + rows.append( + { + "commit_sha": sha, + "parents": parents.split(), + "timestamp": timestamp, + "subject": subject[:2000], + "subject_truncated": len(subject) > 2000, + "session_id": subject[9:2009] if subject.startswith("session: ") else None, + }, + ) + return { + **result, + "row_count": count, + "rows": rows, + "next_offset": spec.offset + len(rows) if spec.offset + len(rows) < count else None, + } + + +def _files(repository, revision, path): + entries = tree(repository, revision, path) + if len(entries) > MAX_TREE_FILES: + raise ValueError("Memory comparison exceeds 2000 files; narrow path") + result = {} + total = 0 + for name, oid in entries: + total += int(git(repository, "cat-file", "-s", oid)) + if total > MAX_TREE_BYTES: + raise ValueError("Memory comparison exceeds 20 MB per snapshot; narrow path") + result[name] = blob(repository, oid) + return result + + +def _hash(files): + digest = hashlib.sha256() + for name, data in sorted(files.items()): + digest.update(name.encode() + b"\0" + hashlib.sha256(data).digest()) + return digest.hexdigest() + + +def compare_memory(home, spec): + """Compare committed regular-file contents, independent of dirty checkout files.""" + base_run, base_manifest, base_repo = _repository(home, spec.validation_path, spec.case_id) + target_run, target_manifest, target_repo = _repository( + home, + spec.target_validation_path or spec.validation_path, + spec.case_id, + ) + base_status, base_head = _head(base_repo) + target_status, target_head = _head(target_repo) + if base_head is None or target_head is None: + raise ValueError(f"Comparison requires memory commits on both sides: {base_status}, {target_status}") + base_sha = _resolve(base_repo, spec.base_ref or "HEAD", base_head) + target_sha = _resolve(target_repo, spec.target_ref, target_head) + before = _files(base_repo, base_sha, spec.path) + after = _files(target_repo, target_sha, spec.path) + added = sorted(after.keys() - before.keys()) + removed = sorted(before.keys() - after.keys()) + modified = sorted(name for name in before.keys() & after.keys() if before[name] != after[name]) + parts = [] + length = 0 + truncated = False + for name in sorted(added + removed + modified): + left, right = before.get(name, b""), after.get(name, b"") + try: + if b"\0" in left or b"\0" in right: + raise UnicodeDecodeError("utf-8", b"\0", 0, 1, "binary") + lines = difflib.unified_diff( + left.decode("utf-8").splitlines(keepends=True), + right.decode("utf-8").splitlines(keepends=True), + fromfile="base/" + name, + tofile="target/" + name, + ) + except UnicodeDecodeError: + lines = iter([f"Binary files differ: {name}\n"]) + for line in lines: + if not line.endswith("\n"): + line += "\n\\ No newline at end of file\n" + available = MAX_OUTPUT - length + parts.append(line[:available]) + length += min(len(line), available) + if len(line) > available: + truncated = True + break + if truncated: + break + return { + "base_validation_path": str(base_run), + "target_validation_path": str(target_run), + "case_id": spec.case_id, + "path": spec.path, + "base_commit": base_sha, + "target_commit": target_sha, + "base_candidate_commit": base_manifest.get("commit_sha"), + "target_candidate_commit": target_manifest.get("commit_sha"), + "base_fingerprints": base_manifest.get("fingerprints"), + "target_fingerprints": target_manifest.get("fingerprints"), + "added": added, + "removed": removed, + "modified": modified, + "base_tree_sha256": _hash(before), + "target_tree_sha256": _hash(after), + "diff": "".join(parts), + "truncated": truncated, + } + + +class HomeMemoryInspectionStep(BaseStep): + """Two read-only Jobs sharing one invocation-scoped Step implementation.""" + + workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace") + + async def execute(self): + """Select the operation from host configuration, never model input.""" + operations = {"history": (MemoryHistoryInput, inspect_history), "compare": (CompareMemoryInput, compare_memory)} + model, function = operations[self.kwargs["operation"]] + return await _execute(self, model, function) diff --git a/plugins/home/src/reme_home/evaluation/step.py b/plugins/home/src/reme_home/evaluation/step.py index ae6781b4..11a2a048 100644 --- a/plugins/home/src/reme_home/evaluation/step.py +++ b/plugins/home/src/reme_home/evaluation/step.py @@ -7,14 +7,14 @@ from .inputs import EvaluationSpec class HomeEvaluateStep(BaseStep): - """Case IDs come from runtime inputs; no selection means all prepared cases.""" + """One evaluation Job with optional host-configured adaptive sampling.""" evaluation: HomeEvaluation = Ref(HomeEvaluation, "home.evaluation") async def execute(self): """Evaluate via the Docker pool and persist host-side scores and evidence.""" if self.context.get("_allowed_paths") is not None: - raise ValueError("home_evaluate requires unrestricted access to its workspace output directories") + raise ValueError("Home evaluation Jobs require unrestricted access to their workspace output directories") arguments = {name: self.context.get(name) for name in EvaluationSpec.model_fields if name in self.context} # Do not silently accept a removed toggle that conflicts with reuse paths. if "eval_only" in self.context: diff --git a/plugins/home/src/reme_home/plugin.yaml b/plugins/home/src/reme_home/plugin.yaml index 47577bc3..b5e2c2be 100644 --- a/plugins/home/src/reme_home/plugin.yaml +++ b/plugins/home/src/reme_home/plugin.yaml @@ -1,20 +1,39 @@ backends: + home_memory_inspection_step: reme_home.evaluation.memory_inspection_step:HomeMemoryInspectionStep + home_code: reme_home.code.component:HomeCode + home_code_step: reme_home.code.step:HomeCodeStep + home_bounded: reme_home.agent:BoundedAgentWrapper home_preparation: reme_home.preparation.component:HomePreparation home_prepare_step: reme_home.preparation.component:HomePrepareStep home_workspace: reme_home.workspace:HomeWorkspace home_judge: reme_home.judge:HomeJudge home_evaluation: reme_home.evaluation.component:HomeEvaluation + home_evaluation_history: reme_home.evaluation.history:HomeEvaluationHistory home_evaluate_step: reme_home.evaluation.step:HomeEvaluateStep + home_list_validation_results_step: reme_home.evaluation.inspection_step:HomeListValidationResultsStep + home_inspect_validation_step: reme_home.evaluation.inspection_step:HomeInspectValidationStep auto_bash_step: reme_home.auto_bash:AutoBashStep application_defaults: components: + home.code: + default: + backend: home_code home.judge: default: backend: home_judge + home.evaluation_history: + default: + backend: home_evaluation_history home.evaluation: default: backend: home_evaluation + sampling_policy: + max_cases: 5 + max_queries: null + temperature: 1.0 + ewma_alpha: 0.3 + uniform_exploration: 0.2 home.preparation: default: backend: home_preparation @@ -22,12 +41,105 @@ application_defaults: default: backend: home_workspace jobs: - home_evaluate: + home_inspect_memory_history: backend: base - description: "Evaluate committed candidate code on prepared cases; optionally restore prior construction tar files" + description: "List a case's exported memory Git checkpoints, newest first; use home_inspect_validation to locate the case" parameters: type: object + additionalProperties: false + required: [validation_path, case_id] properties: + validation_path: {type: string, minLength: 1} + case_id: {type: string, minLength: 1} + path: + type: [string, "null"] + description: "Literal memory-workspace-relative file or directory to follow" + offset: {type: integer, minimum: 0, default: 0} + limit: {type: integer, minimum: 1, maximum: 100, default: 20} + steps: + - backend: home_memory_inspection_step + operation: history + home_compare_memory: + backend: base + description: "Compare a case's memory checkpoints or committed final memory across evaluations; narrow path when output is truncated" + parameters: + type: object + additionalProperties: false + required: [validation_path, case_id] + properties: + validation_path: {type: string, minLength: 1} + case_id: {type: string, minLength: 1} + base_ref: + type: [string, "null"] + description: "Required within one evaluation; defaults to HEAD for cross-evaluation comparisons" + target_ref: + type: string + default: HEAD + description: "Target memory commit or revision; hexadecimal prefixes must be unique among HEAD-reachable commits" + target_validation_path: + type: [string, "null"] + description: "Optional other evaluation containing the same case" + path: + type: [string, "null"] + description: "Literal memory-workspace-relative file or directory" + steps: + - backend: home_memory_inspection_step + operation: compare + home_list_validation_results: + backend: base + description: "List evaluation runs including debug, failed and standalone runs; inspect a returned validation_path with home_inspect_validation" + parameters: + type: object + additionalProperties: false + properties: + branch_name: + type: [string, "null"] + description: "Filter by branch; with commit_sha defaults to the current candidate branch" + commit_sha: + type: [string, "null"] + pattern: "^[0-9a-fA-F]+$" + description: "Full commit SHA or hexadecimal prefix matching exactly one reachable commit on the selected branch" + offset: {type: integer, minimum: 0, default: 0} + limit: {type: integer, minimum: 1, maximum: 100, default: 20} + steps: + - backend: home_list_validation_results_step + home_inspect_validation: + backend: base + description: "Read an evaluation's case summaries, a case's query summaries, or one query's answer/judge previews; follow next_offset and source paths for details" + parameters: + type: object + additionalProperties: false + required: [validation_path] + properties: + validation_path: + type: string + minLength: 1 + description: "Result directory returned by home_evaluate or home_list_validation_results; absolute or relative to home/" + case_id: + type: [string, "null"] + description: "Selected case to inspect; omit to list cases" + query_id: + type: [string, "null"] + description: "Selected query to inspect in detail; requires case_id" + offset: {type: integer, minimum: 0, default: 0} + limit: {type: integer, minimum: 1, maximum: 100, default: 20} + steps: + - backend: home_inspect_validation_step + home_evaluate: + backend: base + description: "Evaluate committed code; optionally sample cases/queries or restore prior construction tar files" + parameters: + type: object + additionalProperties: false + properties: + sampling: + type: boolean + default: false + description: "Adaptively select case/query IDs using the host sampling_policy; debug ignores this flag" + record_history: + type: boolean + default: true + description: "Register successful scored results and started rounds for optimization; false saves results without history or sampling updates; debug always skips history" # case_ids/query_ids are internal runtime inputs, supplied by the host # through injected_job_kwargs or run_job(), never model-selected tools. reuse_construction_from: @@ -101,3 +213,191 @@ application_defaults: - command steps: - backend: auto_bash_step + home_code_status: + backend: base + description: Inspect candidate branch, commit and uncommitted files + parameters: + additionalProperties: false + properties: {} + type: object + steps: + - backend: home_code_step + operation: status + home_list_candidate_branches: + backend: base + description: List the protected init branch and local candidate branches + parameters: + additionalProperties: false + properties: {} + type: object + steps: + - backend: home_code_step + operation: list_branches + home_create_candidate_branch: + backend: base + description: Create and switch to a new candidate branch; requires clean code + parameters: + additionalProperties: false + properties: + branch_name: + maxLength: 100 + minLength: 1 + type: string + required: + - branch_name + type: object + steps: + - backend: home_code_step + operation: create_branch + home_switch_candidate_branch: + backend: base + description: Switch to an existing local branch; requires clean code + parameters: + additionalProperties: false + properties: + branch_name: + maxLength: 100 + minLength: 1 + type: string + required: + - branch_name + type: object + steps: + - backend: home_code_step + operation: switch_branch + home_merge_candidate_branch: + backend: base + description: Stage another candidate branch without committing; conflicts are aborted + parameters: + additionalProperties: false + properties: + branch_name: + maxLength: 100 + minLength: 1 + type: string + required: + - branch_name + type: object + steps: + - backend: home_code_step + operation: merge_branch + home_commit_candidate: + backend: base + description: Commit all changes within candidate_paths; init and existing history are protected + parameters: + additionalProperties: false + properties: + message: + maxLength: 10000 + minLength: 1 + pattern: \S + type: string + required: + - message + type: object + steps: + - backend: home_code_step + operation: commit + home_code_diff: + backend: base + description: Inspect tracked working changes against HEAD or compare immutable commits + parameters: + additionalProperties: false + properties: + base_revision: + anyOf: + - type: string + - type: 'null' + default: null + revision: + anyOf: + - type: string + - type: 'null' + default: null + type: object + steps: + - backend: home_code_step + operation: diff + home_list_code_tree: + backend: base + description: List tracked regular files at a committed revision + parameters: + additionalProperties: false + properties: + revision: + default: HEAD + minLength: 1 + type: string + path: + anyOf: + - type: string + - type: 'null' + default: null + max_results: + default: 500 + maximum: 2000 + minimum: 1 + type: integer + type: object + steps: + - backend: home_code_step + operation: list_code_tree + home_read_code_file: + backend: base + description: Read up to 300 lines from a committed code file + parameters: + additionalProperties: false + properties: + path: + minLength: 1 + type: string + revision: + default: HEAD + minLength: 1 + type: string + start_line: + default: 1 + minimum: 1 + type: integer + end_line: + anyOf: + - minimum: 1 + type: integer + - type: 'null' + default: null + required: + - path + type: object + steps: + - backend: home_code_step + operation: read_code_file + home_search_code: + backend: base + description: Search committed code with a bounded Python multiline regular expression + parameters: + additionalProperties: false + properties: + pattern: + maxLength: 500 + minLength: 1 + type: string + revision: + default: HEAD + minLength: 1 + type: string + path: + anyOf: + - type: string + - type: 'null' + default: null + max_results: + default: 30 + maximum: 30 + minimum: 1 + type: integer + required: + - pattern + type: object + steps: + - backend: home_code_step + operation: search_code diff --git a/plugins/home/src/reme_home/preparation/code.py b/plugins/home/src/reme_home/preparation/code.py index 38f258de..1f9831dd 100644 --- a/plugins/home/src/reme_home/preparation/code.py +++ b/plugins/home/src/reme_home/preparation/code.py @@ -10,18 +10,35 @@ import tomllib from .spec import PreparationSpec, relative_path -def git(repository: Path, *arguments: str) -> str: +def git_bytes(repository: Path, *arguments: str) -> bytes: """Run bounded, noninteractive Git commands without a shell or user hooks.""" environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} environment.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0") result = subprocess.run( - ["git", "-c", "core.hooksPath=" + os.devnull, "-c", "commit.gpgSign=false", "-C", str(repository), *arguments], + [ + "git", + "-c", + "core.hooksPath=" + os.devnull, + "-c", + "commit.gpgSign=false", + "-c", + "core.fsmonitor=false", + "-C", + str(repository), + *arguments, + ], check=True, capture_output=True, env=environment, timeout=120, ) - return result.stdout.decode().strip() + return result.stdout + + +def git(repository: Path, *arguments: str, strip: bool = True) -> str: + """Decode Git output, optionally preserving porcelain whitespace and path bytes.""" + output = git_bytes(repository, *arguments).decode("utf-8") + return output.strip() if strip else output def tree_fingerprint(root: Path) -> str: diff --git a/plugins/home/src/reme_home/runtime_docs/architecture-guide.md b/plugins/home/src/reme_home/runtime_docs/architecture-guide.md index 6e58f903..b81e65bc 100644 --- a/plugins/home/src/reme_home/runtime_docs/architecture-guide.md +++ b/plugins/home/src/reme_home/runtime_docs/architecture-guide.md @@ -76,3 +76,19 @@ flag. Host-installed benchmark judge plugins score answers outside Docker and outside the candidate repository. See `validation-artifact-guide.md` for saved evidence and score interpretation. + +## Candidate versions + +Use the Home code Jobs to list, create, switch, merge and commit candidates in +`home/code/repo`. `init` is the protected preparation baseline. Branch operations +require clean code; merges stay uncommitted for inspection. Commit all intended +changes with `home_commit_candidate` before debug or scored evaluation. Mutations +are limited to prepared `candidate_paths`; keep repository metadata and runtime +outside the optimizer's write permissions. + +`home_list_code_tree`, `home_read_code_file` and `home_search_code` inspect raw +committed files. Their `revision` defaults to current HEAD and returns a resolved +SHA. For diagnosis or version comparisons, the host should inject a fixed SHA +through `injected_job_kwargs` for each agent/Job alias, so subsequent checkout +changes cannot affect the evidence. See the Home plugin README for inputs and +output limits. diff --git a/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md b/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md index b6cd3f06..32749827 100644 --- a/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md +++ b/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md @@ -120,3 +120,84 @@ structured exception details are treated as infrastructure errors. Construction failure-rate tolerance applies only to classified candidate errors. Construction reuse answers and judges again with the current candidate; it does not reuse donor answers or scores. + +## Evaluation history and adaptive sampling evidence + +`home_evaluate sampling=true` enables adaptive selection; false preserves full or +explicit evaluation. Debug always bypasses history. `sampling_policy` is the +host policy input, distinct from the boolean toggle. The effective policy is +captured in `manifest.options` and the started plan. + +`record_history=true` (the default) includes a successful full baseline even when +`sampling=false`. `record_history=false` runs the same scoring and artifact pipeline +without registering results or advancing any optimization/sampling counters. The +choice is stored in `manifest.options.record_history` and respected by history +registration and rebuilding; a missing legacy setting means true. Debug always +skips history. The response reports effective participation as `record_history`. +An untracked adaptive run may read existing observations and saves its draw locally +in `manifest.selection_plan`, without a shared started plan or ledger round. + +`home.evaluation_history` owns the successful-result catalog at +`home/evaluations/history.json`. Entries require completed scored training runs +with `success=true` (or equivalent legacy summary counts) and no infrastructure +errors. Low scores remain useful. Failed/cancelled/debug/partial runs are excluded; +a mixed run with an infrastructure failure contributes no quality observations. +Source manifests, summaries and query results remain authoritative. + +`sandbox_started_at` marks the first successfully initialized sandbox. The +history-enabled manifest's `sampling.plan_path` points to its policy, seed, weights, selection +and started round under `home/sampling/plans/`. Pre-start failures have no marker +or plan, and additional workers do not create extra rounds. The derived sampling +cache separates started `round` and `successful_round`: only successful history +advances staleness/recheck timing and trains EWMA. Adaptive plans preserve their +probability clock in `selection_round`. Actual numeric scores alone count as +quality observations; missing/unjudged answers never become zero scores. + +Missing caches and `HomeEvaluationHistory.rebuild()` reconstruct the catalog and +sampling observations from saved evidence. Rebuilding never modifies datasets, +candidate code or memory files. See the Home README for configuration, host +per-call policy overrides and enforced agent injection examples. + +## Read-only evaluation inspection + +Use `home_list_validation_results` to find runs and pass a returned +`validation_path` to `home_inspect_validation`. The catalog includes failed, +debug, running and untracked runs, not only successful optimization history. +Both Jobs page with `offset` and `limit` (default 20, maximum 100), returning +`row_count`, `rows`, `next_offset`. Catalog filters are `branch_name` and +`commit_sha`; a hexadecimal commit prefix must uniquely match among all commits +reachable from the specified branch, or the current branch when omitted. + +Inspection without `case_id` returns case summaries; add `case_id` for query +summaries and `query_id` for bounded answer/judge fields. Each row's JSON pointer +is relative to `rows_source`. Failed construction Jobs have separate pagination +and pointers relative to `construction.source`. Query fields are limited to +2,000-character previews and errors to 600 characters. Follow returned source +paths for complete evidence. Missing files/results remain explicit, with null +scores, and partial summaries are used when full summaries are absent. Reads do +not rebuild summaries or modify evaluation history, sampling state or artifacts. + +## Memory checkpoint tools + +After locating one case, call `home_inspect_memory_history` with its +`validation_path` and `case_id` to page through exported memory checkpoints. +Optional `path` follows a literal memory-workspace-relative file or directory. +The response distinguishes a missing workspace, absent history and an empty +repository; invalid history fails explicitly. Session IDs are extracted only +from the established `session: ` commit-subject convention. + +Use `home_compare_memory` with `base_ref` and optional `target_ref` (default +`HEAD`) to inspect how memory changed within that evaluation. Add +`target_validation_path` to compare the same case across evaluations; both +references then default to `HEAD`. References must resolve to commits reachable +from each memory HEAD, and hexadecimal prefixes must be unique within that +reachable history. Compare candidate identities and fingerprints before drawing +cross-evaluation conclusions. + +Diffs describe committed tracked regular-file contents rather than dirty checkout +files, untracked files or mode-only changes. Binary changes are identified without +printing bytes. Narrow `path` when the 2,000-file, 20-MB snapshot, 5-MB file or +100,000-character diff limit is reached. Hashes cover the selected path scope. +These tools neither reconstruct missing exports nor modify Git/evaluation/history +artifacts. Raw logs and source JSON remain available through the existing result +inspection paths. diff --git a/plugins/home/src/reme_home/sampling.py b/plugins/home/src/reme_home/sampling.py new file mode 100644 index 00000000..0f329578 --- /dev/null +++ b/plugins/home/src/reme_home/sampling.py @@ -0,0 +1,169 @@ +"""Adaptive formal-validation sampling ported from dev_meta b53048e4. + +Keep the source weighting, weighted ordering and fair-share query allocation. +EvaluationHistory owns persisted observations and started-validation plans. +""" + +import json +import math +import random +import time +from typing import Any, Mapping + +from pydantic import ConfigDict, Field + +from .preparation.data.models import CaseSpec +from .preparation.spec import SpecModel + + +class SamplingPolicy(SpecModel): + """Host-owned limits with the same defaults as dev_meta's validation.sampling.""" + + model_config = ConfigDict(frozen=True) + + max_cases: int = Field(default=5, ge=1) + max_queries: int | None = Field(default=None, ge=1) + temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False) + ewma_alpha: float = Field(default=0.3, gt=0, le=1, allow_inf_nan=False) + uniform_exploration: float = Field(default=0.2, ge=0, le=1, allow_inf_nan=False) + + +def create_sampling_plan( + cases: list[CaseSpec], + state: Mapping[str, Any], + policy: SamplingPolicy, + *, + eligible_query_ids: Mapping[str, list[str]] | None = None, +) -> dict: + """Draw without replacement, without advancing state or writing an attempted plan.""" + if eligible_query_ids is not None: + cases = [case for case in cases if case.case_id in eligible_query_ids] + if not cases: + raise ValueError("training-set contains no cases to sample") + round_id = int(state.get("round", 0)) + 1 + seed = time.time_ns() + rng = random.Random(seed) + case_records = state.get("cases", {}) + case_weights = _mixed_weights( + {case.case_id: case_records.get(case.case_id, {}) for case in cases}, + round_id, + policy, + ) + ordered_cases = _weighted_order(cases, case_weights, rng, lambda case: case.case_id) + query_limited = policy.max_queries is not None + remaining_queries = policy.max_queries + selected_cases: list[str] = [] + selected_queries: dict[str, list[str]] = {} + query_weights: dict[str, dict[str, float]] = {} + query_records = state.get("queries", {}) + for case in ordered_cases: + if len(selected_cases) >= policy.max_cases: + break + if remaining_queries is not None and remaining_queries <= 0: + break + query_ids = [query.query_id for query in case.queries] + if eligible_query_ids is not None: + allowed = set(eligible_query_ids.get(case.case_id, [])) + query_ids = [query_id for query_id in query_ids if query_id in allowed] + if not query_ids: + continue + if query_limited: + weights = _mixed_weights( + {query_id: query_records.get(_query_key(case.case_id, query_id), {}) for query_id in query_ids}, + round_id, + policy, + ) + query_weights[case.case_id] = weights + ordered_queries = _weighted_order(query_ids, weights, rng, lambda value: value) + open_case_slots = max(policy.max_cases - len(selected_cases), 1) + fair_share = max(math.ceil(int(remaining_queries or 0) / open_case_slots), 1) + take = min(len(ordered_queries), fair_share, int(remaining_queries or 0)) + query_ids = ordered_queries[:take] + if not query_ids: + continue + remaining_queries = int(remaining_queries or 0) - len(query_ids) + selected_cases.append(case.case_id) + selected_queries[case.case_id] = query_ids + if not selected_cases: + raise ValueError("sampling budget selected no executable case/query pairs") + return { + "schema_version": 1, + "mode": "sampled", + "round": round_id, + "seed": seed, + "policy": policy.model_dump(), + "case_ids": selected_cases, + "query_ids": selected_queries, + "case_draw_weights": case_weights, + "query_draw_weights": query_weights, + "eligibility": dict(eligible_query_ids) if eligible_query_ids is not None else None, + } + + +def _observe(record: dict[str, Any], loss: float, round_id: int, alpha: float) -> None: + previous = record.get("ewma_loss") + record["ewma_loss"] = loss if previous is None else alpha * loss + (1.0 - alpha) * float(previous) + record["observed_count"] += 1 + record["last_valid_score_round"] = round_id + if loss > 0: + record["last_error_round"] = round_id + record["successful_observations_after_last_error"] = 0 + elif record.get("last_error_round") is not None: + record["successful_observations_after_last_error"] += 1 + + +def _adaptive_weight(record: Mapping[str, Any], round_id: int, policy: SamplingPolicy) -> float: + observed = int(record.get("observed_count", 0)) + raw_risk = record.get("ewma_loss") + risk = float(raw_risk) if isinstance(raw_risk, (int, float)) and not isinstance(raw_risk, bool) else 0.5 + last_selected = record.get("last_selected_round") + staleness = round_id if last_selected is None else max(round_id - int(last_selected), 0) + uncertainty = 1.0 / math.sqrt(observed + 1.0) + last_error = record.get("last_error_round") + successes = int(record.get("successful_observations_after_last_error", 0)) + unrechecked_error = 0.0 + if last_error is not None: + unrechecked_error = math.log1p(max(round_id - int(last_error), 0)) / (successes + 1.0) + priority = risk + 0.2 * math.log1p(staleness) + 0.2 * uncertainty + 0.1 * unrechecked_error + return math.exp(min(priority / policy.temperature, 50.0)) + + +def _mixed_weights( + records: Mapping[str, Mapping[str, Any]], + round_id: int, + policy: SamplingPolicy, +) -> dict[str, float]: + """Mix normalized adaptive weights with a true uniform exploration mass.""" + + adaptive = {key: _adaptive_weight(record, round_id, policy) for key, record in records.items()} + total = sum(adaptive.values()) + uniform = 1.0 / len(adaptive) + return { + key: policy.uniform_exploration * uniform + (1.0 - policy.uniform_exploration) * value / total + for key, value in adaptive.items() + } + + +def _weighted_order(values: list[Any], weights: Mapping[str, float], rng: random.Random, key: Any) -> list[Any]: + return sorted( + values, + key=lambda value: rng.random() ** (1.0 / max(weights[key(value)], 1e-12)), + reverse=True, + ) + + +def _empty_record() -> dict[str, Any]: + return { + "observed_count": 0, + "ewma_loss": None, + "last_selected_round": None, + "last_valid_score_round": None, + "last_error_round": None, + "successful_observations_after_last_error": 0, + "candidate_failure_count": 0, + "infra_error_count": 0, + } + + +def _query_key(case_id: str, query_id: str) -> str: + return json.dumps([case_id, query_id], ensure_ascii=False, separators=(",", ":")) diff --git a/plugins/home/tests/test_benchmark_runner.py b/plugins/home/tests/test_benchmark_runner.py index 1dad8a60..797768c6 100644 --- a/plugins/home/tests/test_benchmark_runner.py +++ b/plugins/home/tests/test_benchmark_runner.py @@ -6,6 +6,7 @@ from pathlib import Path from unittest.mock import AsyncMock import pytest +import yaml from reme.schema import Response @@ -64,6 +65,13 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b assert observed["plugins"] == ["home", "beam-judge" if benchmark == "beam" else "lme-judge"] assert observed["service"] == {"backend": "cli"} assert observed["workspace_dir"] == str(tmp_path / "experiment") + assert observed["components"]["home.evaluation"]["default"]["sampling_policy"] == { + "max_cases": 5, + "max_queries": None, + "temperature": 1.0, + "ewma_alpha": 0.3, + "uniform_exploration": 0.2, + } application.start.assert_awaited_once() application.close.assert_awaited_once() assert application.run_job.await_count == 1 @@ -71,6 +79,36 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b assert result["workspace_dir"] == str(tmp_path / "experiment") +@pytest.mark.asyncio +async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch): + """Partial policies merge typed defaults and never leak into preparation inputs.""" + config = yaml.safe_load((ROOT / "benchmark/beam/config_home.yaml").read_text()) + config["sampling_policy"] = {"max_cases": 2, "max_queries": 7} + path = tmp_path / "home.yaml" + path.write_text(yaml.safe_dump(config)) + application = AsyncMock() + application.run_job.return_value = Response(answer={}) + observed = {} + + def factory(**kwargs): + observed.update(kwargs) + return application + + monkeypatch.setattr(home_runner, "Application", factory) + await home_runner.prepare(path) + policy = observed["components"]["home.evaluation"]["default"]["sampling_policy"] + assert policy["max_cases"] == 2 and policy["max_queries"] == 7 and policy["ewma_alpha"] == 0.3 + assert "sampling_policy" not in application.run_job.call_args.kwargs["spec"] + for invalid in ({"max_cases": 0}, {"unknown": 1}, {"max_queries": "7"}): + config["sampling_policy"] = invalid + path.write_text(yaml.safe_dump(config)) + with pytest.raises(ValueError): + await home_runner.prepare(path) + assert application.start.await_count == 1 + assert application.run_job.await_count == 1 + assert application.run_job.call_args.args == ("home_prepare",) + + @pytest.mark.asyncio @pytest.mark.parametrize("failure", ["startup", "preparation"]) async def test_runner_propagates_failure_and_closes(monkeypatch, failure): diff --git a/plugins/home/tests/test_bounded_agent.py b/plugins/home/tests/test_bounded_agent.py new file mode 100644 index 00000000..c042e490 --- /dev/null +++ b/plugins/home/tests/test_bounded_agent.py @@ -0,0 +1,351 @@ +"""Home budgets exercised through AgentScope's real reply and event loop.""" + +# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access,unused-argument + +import asyncio +from types import SimpleNamespace + +import pytest +from agentscope.agent import Agent, ReActConfig +from agentscope.agent._config import ModelConfig +from agentscope.event import ModelCallEndEvent, ReplyEndEvent, ReplyEndReason +from agentscope.message import AssistantMsg, HintBlock, TextBlock, ThinkingBlock, ToolCallBlock, UserMsg +from agentscope.model import ChatResponse, ChatUsage, FinishedReason +from agentscope.tool import FunctionTool, Toolkit + +from reme_home.agent import BoundedReActAgent, RunTimeBudgetTimer + + +class RecordingModel: + model = "isolated-test" + context_size = 10000 + formatter = SimpleNamespace(supported_input_media_types=[]) + + def __init__(self, *, final=None, stream=False): + self.calls = [] + self.final = final + self.stream = stream + + async def count_tokens(self, messages, tools): + return 10 + + async def __call__(self, **kwargs): + self.calls.append(kwargs) + if kwargs["tool_choice"] is not None and kwargs["tool_choice"].mode == "none": + if isinstance(self.final, BaseException): + raise self.final + response = self.final or ChatResponse(content=[TextBlock(text="final report")], is_last=True) + if self.stream: + + async def chunks(): + yield ChatResponse(content=[TextBlock(text="final ")], is_last=False) + yield response + + return chunks() + return response + return ChatResponse( + content=[ThinkingBlock(thinking="investigating")], + is_last=True, + usage=ChatUsage(input_tokens=2, output_tokens=3, time=0), + ) + + +def hints(agent, source): + return [ + block + for message in agent.state.context + for block in message.content + if isinstance(block, HintBlock) and block.source == source + ] + + +def make_agent(model=None, timer=None, max_iters=2): + return BoundedReActAgent( + name="investigator", + system_prompt="Investigate the evidence.", + model=model or RecordingModel(), + react_config=ReActConfig(max_iters=max_iters, interruption_raise_cancelled_error=True), + model_config=ModelConfig(max_retries=0), + run_time_budget_timer=timer, + ) + + +@pytest.mark.parametrize("budget", [True, 0, -1, float("nan"), float("inf"), "10"]) +def test_timer_rejects_invalid_budget(budget): + with pytest.raises(ValueError, match="positive"): + RunTimeBudgetTimer(budget, agent_label="diagnostic") + + +def test_timer_thresholds_restart_and_skipped_thresholds(): + now = [0.0] + timer = RunTimeBudgetTimer(100, agent_label="diagnostic", clock=lambda: now[0]) + assert timer.next_reminder() is None + timer.start() + for percentage, persistent in [(59, None), (80, False), (90, True), (100, False), (110, True)]: + now[0] = percentage * 60 + reminder = timer.next_reminder() + if persistent is None: + assert reminder is None + else: + assert reminder.persistent is persistent + assert "does not terminate" in reminder.content + assert timer.next_reminder() is None + timer.record_context_compression() + timer.stop() + assert timer.next_reminder() is None + timer.start() + assert timer.context_compression_count == 0 + assert timer.next_reminder() is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_native_loop_has_exactly_one_tool_free_finalization(stream): + usage = ChatUsage(input_tokens=5, output_tokens=7, time=0, cache_input_tokens=2) + model = RecordingModel( + final=ChatResponse(content=[TextBlock(text="report")], is_last=True, usage=usage), + stream=stream, + ) + agent = make_agent(model) + timer = RunTimeBudgetTimer(1, agent_label="diagnostic") + agent.run_time_budget_timer = timer + items = [ + item + async for item in agent.reply_stream(UserMsg(name="caller", content="Find the cause"), yield_final_msg=True) + ] + assert len(model.calls) == 3 + assert agent.state.cur_iter == 3 + request = model.calls[-1] + assert request["tools"] == [] + assert request["tool_choice"].mode == "none" + assert "Find the cause" in request["messages"][-1].get_text_content() + assert items[-1].get_text_content() == "report" + # Reminders must preserve cumulative reply usage as well as per-call events. + assert items[-1].usage.input_tokens == 9 + assert items[-1].usage.output_tokens == 13 + assert items[-1].usage.cache_input_tokens == 2 + calls = [item for item in items if isinstance(item, ModelCallEndEvent)] + assert len(calls) == 3 + assert sum(item.input_tokens for item in calls) == 9 + assert sum(item.output_tokens for item in calls) == 13 + ends = [item for item in items if isinstance(item, ReplyEndEvent)] + assert len(ends) == 1 and ends[0].finished_reason == ReplyEndReason.EXCEED_MAX_ITERS + assert not timer.active + + +@pytest.mark.asyncio +async def test_finalization_discards_unsolicited_tools(): + executed = [] + + def forbidden(): + executed.append(True) + + model = RecordingModel( + final=ChatResponse( + content=[TextBlock(text="report"), ToolCallBlock(id="call", name="forbidden", input="{}")], + is_last=True, + ), + ) + agent = make_agent(model) + agent.toolkit = Toolkit(tools=[FunctionTool(forbidden)]) + result = await agent.reply(UserMsg(name="caller", content="Investigate")) + assert result.get_text_content() == "report" + assert not executed + assert not agent.state.get_unfinished_tool_calls(agent.name) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failure", + [ + RuntimeError("provider unavailable"), + ChatResponse(content=[], is_last=True), + ChatResponse(content=[], is_last=True, usage=ChatUsage(input_tokens=5, output_tokens=7, time=0)), + ], +) +async def test_finalization_failure_returns_explicit_incomplete_output(failure): + model = RecordingModel(final=failure) + agent = make_agent(model) + result = await agent.reply(UserMsg(name="caller", content="Investigate")) + assert "incomplete prior model output" in result.get_text_content() + assert len(model.calls) == 3 + final_usage = failure.usage if isinstance(failure, ChatResponse) else None + assert result.usage.input_tokens == 4 + (final_usage.input_tokens if final_usage else 0) + assert result.usage.output_tokens == 6 + (final_usage.output_tokens if final_usage else 0) + + +@pytest.mark.asyncio +async def test_cancellation_stops_timer_without_fallback(): + timer = RunTimeBudgetTimer(1, agent_label="diagnostic") + agent = make_agent(RecordingModel(final=asyncio.CancelledError()), timer) + with pytest.raises(asyncio.CancelledError): + await agent.reply(UserMsg(name="caller", content="Investigate")) + assert not timer.active + assert not any("incomplete prior model output" in (msg.get_text_content() or "") for msg in agent.state.context) + + +@pytest.mark.asyncio +async def test_interrupted_response_preserves_native_interruption(): + model = RecordingModel(final=ChatResponse(content=[], is_last=True, finished_reason=FinishedReason.INTERRUPTED)) + with pytest.raises(asyncio.CancelledError): + await make_agent(model).reply(UserMsg(name="caller", content="Investigate")) + + +@pytest.mark.asyncio +async def test_new_reply_resets_iteration_reminders(): + model = RecordingModel() + agent = make_agent(model) + for task in ("first", "second"): + result = await agent.reply(UserMsg(name="caller", content=task)) + assert result.get_text_content() == "final report" + assert agent._original_task_prompt == task + assert len(model.calls) == 6 + reminders = hints(agent, "home_iteration_guard") + assert len(reminders) == 4 + + +@pytest.mark.asyncio +async def test_normal_reply_does_not_force_finalization(): + class ImmediateModel(RecordingModel): + async def __call__(self, **kwargs): + self.calls.append(kwargs) + return ChatResponse(content=[TextBlock(text="already sufficient")], is_last=True) + + model = ImmediateModel() + result = await make_agent(model).reply(UserMsg(name="caller", content="Investigate")) + assert result.get_text_content() == "already sufficient" + assert len(model.calls) == 1 + + +@pytest.mark.asyncio +async def test_final_context_trims_old_evidence_but_preserves_system_and_task(): + class SmallContextModel(RecordingModel): + async def count_tokens(self, messages, tools): + return 10000 if len(messages) > 2 else 10 + + model = SmallContextModel() + agent = make_agent(model) + agent.state.cur_iter = 2 + agent._original_task_prompt = "original task" + messages = [UserMsg(name="system", content="system prompt"), UserMsg(name="caller", content="old evidence")] + await agent._call_model(messages, tools=[{"name": "unused"}]) + assert len(model.calls[-1]["messages"]) == 2 + assert model.calls[-1]["messages"][0] is messages[0] + assert "original task" in model.calls[-1]["messages"][-1].get_text_content() + assert len(messages) == 2 + + +@pytest.mark.asyncio +async def test_missing_stream_completion_returns_previous_text_without_retry(): + class IncompleteStreamModel(RecordingModel): + async def __call__(self, **kwargs): + self.calls.append(kwargs) + + async def chunks(): + yield ChatResponse(content=[TextBlock(text="partial")], is_last=False) + + return chunks() + + model = IncompleteStreamModel() + agent = make_agent(model) + agent.state.cur_iter = 2 + agent.state.context.append(AssistantMsg(name=agent.name, content="previous finding")) + response = await agent._call_model([], tools=[]) + assert "incomplete prior model output" in response.content[0].text + assert "previous finding" in response.content[0].text + assert len(model.calls) == 1 + + +@pytest.mark.asyncio +async def test_time_reminders_alternate_and_do_not_stop_investigation(): + now = [0.0] + timer = RunTimeBudgetTimer(100, agent_label="diagnostic", clock=lambda: now[0]) + model = RecordingModel() + agent = make_agent(model, timer, max_iters=100) + timer.start() + for percentage in (60, 75, 90, 100, 110): + now[0] = percentage * 60 + _ = [item async for item in agent._reasoning()] + persistent = hints(agent, "home_run_time_guard") + assert len(persistent) == 2 + assert "(75%" in persistent[0].hint + assert "(100%" in persistent[1].hint + assert len(model.calls) == 5 + assert timer.active + + +@pytest.mark.asyncio +async def test_ephemeral_reminder_removed_when_reasoning_fails(monkeypatch): + now = [0.0] + timer = RunTimeBudgetTimer(1, agent_label="diagnostic", clock=lambda: now[0]) + agent = make_agent(timer=timer) + timer.start() + now[0] = 40 + + async def fail(*args, **kwargs): + raise RuntimeError("failed") + yield # pylint: disable=unreachable + + monkeypatch.setattr(Agent, "_reasoning", fail) + with pytest.raises(RuntimeError): + _ = [item async for item in agent._reasoning()] + assert not hints(agent, "home_run_time_guard") + + +@pytest.mark.asyncio +async def test_only_successful_context_replacement_counts_as_compression(monkeypatch): + timer = RunTimeBudgetTimer(1, agent_label="diagnostic") + agent = make_agent(timer=timer) + + async def no_change(self): + pass + + monkeypatch.setattr(Agent, "compress_context", no_change) + await agent.compress_context() + assert timer.context_compression_count == 0 + + async def change(self): + self.state.context = [] + + monkeypatch.setattr(Agent, "compress_context", change) + await agent.compress_context() + assert timer.context_compression_count == 1 + + +class CompressionFailureModel(RecordingModel): + def __init__(self, failure, fail_after=2): + super().__init__() + self.failure = failure + self.fail_after = fail_after + + async def count_tokens(self, messages, tools): + return 9000 if len(self.calls) >= self.fail_after and len(messages) > 2 else 10 + + async def generate_structured_output(self, **kwargs): + raise self.failure + + +@pytest.mark.asyncio +async def test_final_report_survives_real_compression_failure(): + model = CompressionFailureModel(RuntimeError("compression provider unavailable")) + timer = RunTimeBudgetTimer(1, agent_label="diagnostic") + agent = make_agent(model, timer) + agent.context_config.compression_fallback_to_truncation = False + result = await agent.reply(UserMsg(name="caller", content="Investigate")) + assert result.get_text_content() == "final report" + assert len(model.calls) == 3 + assert model.calls[-1]["tools"] == [] + assert len(model.calls[-1]["messages"]) == 2 + assert timer.context_compression_count == 0 + assert not timer.active + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [RuntimeError("compression failed"), asyncio.CancelledError()]) +async def test_nonfinal_compression_errors_and_cancellation_propagate(failure): + model = CompressionFailureModel(failure, fail_after=1 if isinstance(failure, RuntimeError) else 2) + agent = make_agent(model) + agent.context_config.compression_fallback_to_truncation = False + with pytest.raises(type(failure)): + await agent.reply(UserMsg(name="caller", content="Investigate")) + assert len(model.calls) == model.fail_after diff --git a/plugins/home/tests/test_bounded_wrapper.py b/plugins/home/tests/test_bounded_wrapper.py new file mode 100644 index 00000000..2b261947 --- /dev/null +++ b/plugins/home/tests/test_bounded_wrapper.py @@ -0,0 +1,210 @@ +"""Bounded ReMe agents use real Job tools, injection and persisted sessions.""" + +# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access + +import json +from importlib.metadata import EntryPoint +from types import SimpleNamespace + +import pytest +from agentscope.message import HintBlock, TextBlock, ToolCallBlock +from agentscope.model import ChatResponse, ChatUsage +from agentscope.tool import FunctionTool, Toolkit +from reme_home.agent import BoundedAgentWrapper, BoundedReActAgent, RunTimeBudgetTimer +from test_bounded_agent import RecordingModel + +from reme.components.agent_wrapper import AsAgentWrapper +from reme.components.application_context import ApplicationContext +from reme.components.job import BaseJob +from reme.enumeration import ComponentEnum +from reme.plugin import resolve_plugin_runtime +from reme.steps.base_step import BaseStep +from reme.utils import global_counter_get + + +class CaptureStep(BaseStep): + async def execute(self): + self.app_context.metadata.setdefault("test.calls", []).append(dict(self.context.data)) + self.context.response.answer = "captured" + + +class JobModel(RecordingModel): + def __init__(self, arguments): + super().__init__() + self.arguments = arguments + + async def __call__(self, **kwargs): + self.calls.append(kwargs) + content = ( + [ToolCallBlock(id="capture-call", name="capture", input=json.dumps(self.arguments))] + if len(self.calls) == 1 + else [TextBlock(text="finished")] + ) + return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0)) + + +@pytest.fixture(name="wrapper") +def bounded_wrapper(tmp_path): + context = ApplicationContext(workspace_dir=str(tmp_path)) + result = BoundedAgentWrapper( + app_context=context, + name="investigator", + as_llm="", + time_budget_minutes=30, + react_config={"max_iters": 2}, + ) + result.as_llm = SimpleNamespace(model=RecordingModel()) + return result + + +def test_plugin_registers_bounded_agent_only_when_enabled(monkeypatch): + entry = EntryPoint(name="home", value="reme_home", group="reme.plugins") + monkeypatch.setattr("reme.plugin.find_entry_points", lambda group, name: [entry]) + enabled = resolve_plugin_runtime({"plugins": ["home"]}) + assert enabled.registry.get(ComponentEnum.AGENT_WRAPPER, "home_bounded") is BoundedAgentWrapper + assert resolve_plugin_runtime({}).registry.get(ComponentEnum.AGENT_WRAPPER, "home_bounded") is None + assert issubclass(BoundedAgentWrapper, AsAgentWrapper) + + +@pytest.mark.asyncio +async def test_each_build_owns_timer_and_honors_call_overrides(wrapper): + first, _ = await wrapper._build_agent("first", **wrapper._merged_kwargs({})) + second, _ = await wrapper._build_agent("second", **wrapper._merged_kwargs({"time_budget_minutes": 5})) + assert isinstance(first, BoundedReActAgent) + assert first.run_time_budget_timer.time_budget_minutes == 30 + assert second.run_time_budget_timer.time_budget_minutes == 5 + assert first.run_time_budget_timer is not second.run_time_budget_timer + assert first.react_config.max_iters == 2 + no_timer, _ = await wrapper._build_agent("third", **wrapper._merged_kwargs({"time_budget_minutes": None})) + assert no_timer.run_time_budget_timer is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("conflict", [False, True]) +async def test_real_job_tools_enforce_hidden_injected_arguments(wrapper, conflict): + context = wrapper.app_context + context.registry.register(CaptureStep, "test_capture") + job = BaseJob( + name="capture", + app_context=context, + parameters={ + "type": "object", + "properties": {"visible": {"type": "string"}, "case_ids": {"type": "array"}}, + "required": ["visible", "case_ids"], + }, + steps=[{"backend": "test_capture"}], + ) + context.jobs["capture"] = job + arguments = {"visible": "model value"} + if conflict: + arguments["case_ids"] = ["model-chosen"] + model = JobModel(arguments) + wrapper.as_llm.model = model + await job.start() + await wrapper.start() + try: + result = await wrapper.reply( + "Run the capture job", + job_tools=["capture"], + injected_job_kwargs={"case_ids": ["host-chosen"]}, + tool_context_id="investigation-1", + ) + schemas = json.dumps(model.calls[0]["tools"]) + assert "case_ids" not in schemas + assert "tool_context_id" not in schemas + if conflict: + assert "test.calls" not in context.metadata + else: + assert context.metadata["test.calls"] == [ + { + "visible": "model value", + "case_ids": ["host-chosen"], + "tool_context_id": "investigation-1", + }, + ] + assert result["result"] == "finished" + assert (wrapper.session_path / f"{result['session_id']}.jsonl").is_file() + resumed = await wrapper.reply("Continue", resume=result["session_id"]) + assert resumed["session_id"] == result["session_id"] + finally: + await wrapper.close() + await job.close() + + +@pytest.mark.asyncio +async def test_wrapper_preserves_observed_task_in_forced_finalization(wrapper): + result = await wrapper.reply("Identify the original failure") + assert result["result"] == "final report" + messages = wrapper.as_llm.model.calls[-1]["messages"] + assert "Original task: Identify the original failure" in messages[-1].get_text_content() + assert result["session_id"] + + +@pytest.mark.asyncio +async def test_wrapper_streams_reme_chunks_and_persists_session(wrapper): + chunks = [chunk async for chunk in wrapper.reply_stream("Investigate")] + assert any(chunk.chunk == "final report" for chunk in chunks) + session_id = chunks[-1].session_id + assert session_id + assert (wrapper.session_path / f"{session_id}.jsonl").is_file() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("timed", [False, True]) +async def test_multiround_usage_and_tools_continue_past_soft_time_limit(wrapper, monkeypatch, timed): + now = [0.0] + timer = RunTimeBudgetTimer(1, agent_label="investigator", clock=lambda: now[0]) + monkeypatch.setattr("reme_home.agent.wrapper.RunTimeBudgetTimer", lambda *args, **kwargs: timer) + executions = [] + + async def inspect_evidence(): + executions.append(now[0]) + return "evidence" + + class MultiRoundModel(RecordingModel): + def __init__(self): + super().__init__() + self.time_reminders = [] + + async def __call__(self, **kwargs): + self.calls.append(kwargs) + self.time_reminders.extend( + block.hint + for message in kwargs["messages"] + for block in message.content + if isinstance(block, HintBlock) and block.source == "home_run_time_guard" + ) + now[0] += 65 + count = len(self.calls) + content = ( + [ToolCallBlock(id=f"inspect-{count}", name="inspect_evidence", input="{}")] + if count < 3 + else [TextBlock(text="finished")] + ) + return ChatResponse( + content=content, + is_last=True, + usage=ChatUsage(input_tokens=10, output_tokens=5, time=0), + ) + + model = MultiRoundModel() + wrapper.as_llm.model = model + options = { + "react_config": {"max_iters": 4}, + "time_budget_minutes": 1 if timed else None, + "toolkit": Toolkit(tools=[FunctionTool(inspect_evidence)]), + } + result = await wrapper.reply("Investigate", **options) + assert result["usage"] == {"input_tokens": 30, "output_tokens": 15, "total_tokens": 45} + assert result["last_message"]["finished_reason"] == "completed" + assert len(executions) == 2 + assert executions[-1] > 60 # A tool really executes after the soft deadline. + assert len(model.calls) == 3 + assert all(call["tools"] for call in model.calls) + assert bool(model.time_reminders) is timed + assert not timer.active + metadata = wrapper.app_context.metadata + assert global_counter_get(metadata, ["__token_counter", wrapper.name, "total_tokens"]) == 45 + resumed = await wrapper.reply("Continue", resume=result["session_id"], **options) + assert resumed["usage"]["total_tokens"] == 15 + assert global_counter_get(metadata, ["__token_counter", wrapper.name, "total_tokens"]) == 60 diff --git a/plugins/home/tests/test_code.py b/plugins/home/tests/test_code.py new file mode 100644 index 00000000..d007b822 --- /dev/null +++ b/plugins/home/tests/test_code.py @@ -0,0 +1,316 @@ +"""Candidate version Jobs exercised against real isolated Git repositories.""" + +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name,unused-argument + +import asyncio +from pathlib import Path +import subprocess +from threading import Event + +import pytest +import pytest_asyncio +from reme_home.code import snapshot +from reme_home.code.models import ARGUMENTS +from reme_home.workspace import _initialization_lock + +from test_preparation import ( # noqa: F401 # pylint: disable=unused-import + home_application, + preparation_inputs, + command, +) + + +from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper + + +@pytest_asyncio.fixture +async def prepared(app, inputs): + spec, _ = inputs + response = await app.run_job("home_prepare", spec=spec) + assert response.success, response.answer + repo = Path(app.config.workspace_dir) / "home/code/repo" + return app, repo, spec + + +async def run(app, job, **kwargs): + response = await app.run_job(job, **kwargs) + assert response.success, response.answer + return response.answer + + +async def create(app, name="candidate"): + return await run(app, "home_create_candidate_branch", branch_name=name) + + +async def commit(app, repo, text="value = 2\n"): + (repo / "plugins/beam/src/reme_beam/__init__.py").write_text(text) + return await run(app, "home_commit_candidate", message="Improve candidate") + + +@pytest.mark.asyncio +async def test_branch_commit_diff_and_immutable_read(prepared): + app, repo, _ = prepared + initial = await run(app, "home_list_candidate_branches") + assert initial["current_branch"] == "init" + assert initial["candidate_branches"] == [] + created = await create(app) + assert created["candidate_branch_count"] == 1 + result = await commit(app, repo) + assert result["commit_sha"] != initial["commit_sha"] + assert (await run(app, "home_code_status"))["clean"] + diff = await run(app, "home_code_diff", base_revision="init", revision=result["commit_sha"]) + assert "+value = 2" in diff["content"] + (repo / "plugins/beam/src/reme_beam/__init__.py").write_text("dirty = 3\n") + status = await run(app, "home_code_status") + assert status["changes"] == [{"status": " M", "path": "plugins/beam/src/reme_beam/__init__.py"}] + assert not status["clean"] + assert "+dirty = 3" in (await run(app, "home_code_diff"))["content"] + read = await run( + app, + "home_read_code_file", + path="plugins/beam/src/reme_beam/__init__.py", + revision=initial["commit_sha"], + ) + assert read["content"] == "value = 1\n" + with pytest.raises(ValueError, match="clean"): + await app.context.components["home.code"]["default"].run("switch_branch", branch_name="init") + + +@pytest.mark.asyncio +async def test_host_pinned_job_tool_and_schema(prepared): + app, repo, _ = prepared + initial = command(repo, "rev-parse", "HEAD") + await create(app) + await commit(app, repo) + job = app.context.jobs["home_read_code_file"] + tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"revision": initial}) + assert "revision" not in tool.input_schema["properties"] + # Test the actual wrapper's merge contract through its executable FunctionTool. + response = await tool(path="plugins/beam/src/reme_beam/__init__.py") + assert "value = 1" in response.content[0].text + assert "value = 2" not in response.content[0].text + with pytest.raises(ValueError, match="injected"): + await tool(path="plugins/beam/src/reme_beam/__init__.py", revision="HEAD") + assert all(model.model_json_schema()["additionalProperties"] is False for model in ARGUMENTS.values()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("name", ["init", "a/b", "../escape", "-bad", "a\\b", "@{-1}"]) +async def test_invalid_candidate_branch(prepared, name): + app, _, _ = prepared + response = await app.run_job("home_create_candidate_branch", branch_name=name) + assert not response.success + assert (await run(app, "home_code_status"))["current_branch"] == "init" + + +@pytest.mark.asyncio +async def test_protected_init_and_scope(prepared): + app, repo, _ = prepared + response = await app.run_job("home_commit_candidate", message="not allowed") + assert not response.success + await create(app) + baseline = command(repo, "rev-parse", "HEAD") + readme = repo / "plugins/beam/README.md" + readme.write_text("outside declared editable roots") + response = await app.run_job("home_commit_candidate", message="outside scope") + assert not response.success + assert command(repo, "rev-parse", "HEAD") == baseline + assert not command(repo, "diff", "--cached", "--name-only") + readme.write_text("Plugin installation instructions\n") + target = repo / "plugins/beam/src/link" + target.symlink_to("/etc/passwd") + response = await app.run_job("home_commit_candidate", message="link") + assert not response.success + target.unlink() + result = await commit(app, repo) + command(repo, "update-ref", "refs/heads/init", result["commit_sha"]) + response = await app.run_job("home_code_status") + assert not response.success + + +@pytest.mark.asyncio +async def test_merge_then_explicit_commit(prepared): + app, repo, _ = prepared + await create(app, "left") + left = await commit(app, repo) + await run(app, "home_switch_candidate_branch", branch_name="init") + await create(app, "right") + (repo / "plugins/beam/src/new.py").write_text("another mechanism\n") + right = await run(app, "home_commit_candidate", message="right") + merged = await run(app, "home_merge_candidate_branch", branch_name="left") + assert merged["status"] == "merged_not_committed" + assert merged["commit_sha"] == right["commit_sha"] + assert (await run(app, "home_code_status"))["merge_pending"] + response = await app.run_job("home_switch_candidate_branch", branch_name="init") + assert not response.success + merged = await run(app, "home_commit_candidate", message="Combine compatible improvements") + assert command(repo, "rev-list", "--parents", "-n", "1", "HEAD").split() == [ + merged["commit_sha"], + right["commit_sha"], + left["commit_sha"], + ] + + +@pytest.mark.asyncio +async def test_conflicting_merge_aborts_cleanly(prepared): + app, repo, _ = prepared + await create(app, "left") + await commit(app, repo, "left = 2\n") + await run(app, "home_switch_candidate_branch", branch_name="init") + await create(app, "right") + right = await commit(app, repo, "right = 2\n") + response = await app.run_job("home_merge_candidate_branch", branch_name="left") + assert not response.success + assert not (repo / ".git/MERGE_HEAD").exists() + state = await run(app, "home_code_status") + assert state["clean"] and state["commit_sha"] == right["commit_sha"] + + +@pytest.mark.asyncio +async def test_raw_snapshot_regex_paths_and_truncation(prepared): + app, repo, _ = prepared + await create(app) + folder = repo / "plugins/beam/src" + (folder / "space name.py").write_text("prefix\nhello\nworld\nsuffix\n" + "x\n" * 400) + (folder / ".gitattributes").write_text('"space name.py" export-ignore\n') + (folder / "binary").write_bytes(b"a\0b") + (folder / "carriage.py").write_bytes(b"first\rsecond\rthird") + (folder / "colon:name.py").write_text("hello") + # Non-portable paths fail closed instead of becoming Git pathspecs. + rejected = await app.run_job("home_commit_candidate", message="bad path") + assert not rejected.success + (folder / "colon:name.py").unlink() + result = await run(app, "home_commit_candidate", message="Navigation fixtures") + tree = await run(app, "home_list_code_tree", path="plugins/beam/src", max_results=1) + assert tree["revision"] == result["commit_sha"] and tree["truncated"] + found = await run(app, "home_search_code", pattern="hello\nworld", path="plugins/beam/src") + assert found["occurrence_count"] == 1 + assert found["matches"][0]["start_line"] == 1 + assert "hello" in found["matches"][0]["content"] + carriage = await run(app, "home_search_code", pattern="second", path="plugins/beam/src/carriage.py") + assert "> 2 | second" in carriage["matches"][0]["content"] + read = await run(app, "home_read_code_file", path="plugins/beam/src/space name.py") + assert read["end_line"] == 300 and read["truncated"] + for kwargs in [ + {"path": "../escape"}, + {"path": ".git/HEAD"}, + {"path": "/etc/passwd"}, + {"path": "plugins/beam/src/binary"}, + {"path": "plugins/beam/src", "revision": "--help"}, + ]: + assert not (await app.run_job("home_read_code_file", **kwargs)).success + for pattern in ("[", "^", ""): + assert not (await app.run_job("home_search_code", pattern=pattern)).success + assert not (await app.run_job("home_list_code_tree", path=":(glob)**")).success + + +@pytest.mark.asyncio +async def test_lock_restrictions_and_unprepared(app): + assert not (await app.run_job("home_code_status")).success + assert not (await app.run_job("home_code_status", _allowed_paths=[])).success + component = app.context.components["home.code"]["default"] + with pytest.raises(ValueError, match="Unknown"): + await component.run("reset") + with pytest.raises(ValueError): + await component.run("create_branch", branch_name=1) + with _initialization_lock(component.workspace.state_path / "workspace.lock"): + assert not (await app.run_job("home_list_code_tree")).success + + +@pytest.mark.asyncio +async def test_cancel_waits_for_owned_git_thread(prepared, monkeypatch): + app, _, _ = prepared + component = app.context.components["home.code"]["default"] + entered, release, finished = Event(), Event(), Event() + original = component._run + + def blocked(*args): + entered.set() + assert release.wait(5) + try: + return original(*args) + finally: + finished.set() + + monkeypatch.setattr(component, "_run", blocked) + task = asyncio.create_task(component.run("status")) + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + await asyncio.sleep(0) + assert not task.done() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert finished.is_set() and not component._pending + + +@pytest.mark.asyncio +async def test_snapshot_size_and_regex_timeout(prepared, monkeypatch): + app, repo, _ = prepared + await create(app) + await commit(app, repo, 'x = "' + "a" * 10000 + '!"\n') + original = snapshot.subprocess.run + + def short_timeout(*args, **kwargs): + if args[0][0] == snapshot.sys.executable: + kwargs["timeout"] = 0.05 + return original(*args, **kwargs) # pylint: disable=subprocess-run-check + + monkeypatch.setattr(snapshot.subprocess, "run", short_timeout) + with pytest.raises(subprocess.TimeoutExpired): + snapshot.search_code(repo, {}, pattern="(a+)+$", path="plugins/beam/src") + monkeypatch.setattr(snapshot.subprocess, "run", original) + monkeypatch.setattr(snapshot, "MAX_FILE_BYTES", 3) + response = await app.run_job("home_read_code_file", path="plugins/beam/src/reme_beam/__init__.py") + assert not response.success + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mutation", ["staged", "untracked"]) +async def test_branch_operations_refuse_dirty_inputs(prepared, mutation): + app, repo, _ = prepared + if mutation == "staged": + target = repo / "plugins/beam/src/reme_beam/__init__.py" + target.write_text("staged = True\n") + command(repo, "add", ".") + else: + target = repo / "plugins/beam/src/untracked file.py" + target.write_text("untracked = True\n") + before = target.read_bytes() + assert not (await app.run_job("home_create_candidate_branch", branch_name="dirty")).success + assert target.read_bytes() == before + assert command(repo, "branch", "--show-current") == "init" + + +@pytest.mark.asyncio +async def test_exact_paths_rename_deletion_and_revision_pinning(prepared): + app, repo, _ = prepared + await create(app) + original = repo / "plugins/beam/src/reme_beam/__init__.py" + name = "plugins/beam/src/space name\nand newline.py" + target = repo / name + original.rename(target) + first = await run(app, "home_commit_candidate", message="Rename a file") + assert (await run(app, "home_read_code_file", path=name))["content"] == "value = 1\n" + target.write_text("value = 9\n") + pinned = await run(app, "home_commit_candidate", message="Improve renamed code") + target.unlink() + await run(app, "home_commit_candidate", message="Remove obsolete file") + await run(app, "home_switch_candidate_branch", branch_name="init") + read = await run(app, "home_read_code_file", path=name, revision=pinned["commit_sha"]) + assert read["content"] == "value = 9\n" + assert name in (await run(app, "home_list_code_tree", revision=first["commit_sha"]))["paths"] + + +@pytest.mark.asyncio +async def test_reject_symlink_repository_and_detached_candidate(prepared): + app, repo, _ = prepared + await create(app) + commit_sha = (await commit(app, repo))["commit_sha"] + command(repo, "checkout", "--detach", commit_sha) + assert not (await app.run_job("home_commit_candidate", message="detached")).success + assert (await run(app, "home_read_code_file", path="plugins/beam/src/reme_beam/__init__.py"))["content"] + moved = repo.with_name("moved") + repo.rename(moved) + repo.symlink_to(moved, target_is_directory=True) + assert not (await app.run_job("home_code_status")).success diff --git a/plugins/home/tests/test_evaluation.py b/plugins/home/tests/test_evaluation.py index 5af421a1..0fdea79c 100644 --- a/plugins/home/tests/test_evaluation.py +++ b/plugins/home/tests/test_evaluation.py @@ -126,6 +126,8 @@ class FakeFactory: async def create_cases(self, ids, **kwargs): self.workers = [FakeWorker(self) for _ in ids] + if self.workers and self.kwargs.get("on_sandbox_started"): + self.kwargs["on_sandbox_started"]() return self.workers @@ -209,6 +211,7 @@ async def test_reuse_tar_runs_current_candidate_without_building(app, prepared): baseline = await app.run_job("home_evaluate") assert baseline.success, baseline.answer repo = prepared / "code/repo" + assert (await app.run_job("home_create_candidate_branch", branch_name="new-answer")).success (repo / "plugins/beam/src/reme_beam/__init__.py").write_text("value = 2\n") command(repo, "add", ".") command(repo, "-c", "user.name=Fixture", "-c", "user.email=test@localhost", "commit", "-m", "new answer") @@ -414,6 +417,7 @@ async def test_debug_empty_answers_fail_without_scoring(app, prepared, monkeypat @pytest.mark.asyncio async def test_current_head_is_immutable_and_does_not_ship_judge(app, prepared): repo = prepared / "code/repo" + assert (await app.run_job("home_create_candidate_branch", branch_name="changed")).success (repo / "plugins/beam/src/reme_beam/__init__.py").write_text("value = 3\n", encoding="utf-8") command(repo, "add", ".") command(repo, "-c", "user.name=Fixture", "-c", "user.email=test@localhost", "commit", "-m", "change") diff --git a/plugins/home/tests/test_evaluation_history.py b/plugins/home/tests/test_evaluation_history.py new file mode 100644 index 00000000..bb2d5c98 --- /dev/null +++ b/plugins/home/tests/test_evaluation_history.py @@ -0,0 +1,360 @@ +"""Successful-history ownership and evaluation parameter precedence through real Jobs.""" + +# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access + +import json +from pathlib import Path + +import pytest +from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state +from reme_home.sampling import SamplingPolicy, create_sampling_plan +from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import + FakeFactory, + home_application, + preparation_inputs, + prepared_home, +) + +from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper + + +def history(app): + return app.context.components["home.evaluation_history"]["default"] + + +def manifest(response): + return json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text()) + + +@pytest.mark.asyncio +async def test_history_catalog_keeps_successful_low_scores_and_deduplicates(app, prepared): + manager = history(app) + assert isinstance(manager, HomeEvaluationHistory) and manager.is_started + assert manager.list_results() == [] + app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": '{"score": 0}'} + response = await app.run_job("home_evaluate") + assert response.success, response.answer + results = manager.list_results() + assert len(results) == 1 and isinstance(results[0], EvaluationRecord) + assert results[0].mean_query_score == 0 and results[0].scored_query_count == 2 + assert manager.get_result(results[0].run_key) == results[0] + run = Path(response.answer["result_dir"]) + before = manager.sampling_state() + assert manager.record(run) == results[0] + assert len(manager.list_results()) == 1 and manager.sampling_state() == before + payload = json.loads((prepared / "evaluations/history.json").read_text()) + assert payload["results"][0]["run_key"] == results[0].run_key + assert "golden_answer" not in json.dumps(payload) + debug = await app.run_job("home_evaluate", sampling=True, debug=True) + assert debug.success and manager.record(Path(debug.answer["result_dir"])) is None + assert manager.list_results() == results + with pytest.raises(KeyError): + manager.get_result("unknown") + + +@pytest.mark.asyncio +async def test_failed_run_does_not_change_next_probabilities_or_successful_history(app, prepared, monkeypatch): + baseline = await app.run_job("home_evaluate") + assert baseline.success + manager = history(app) + records, observations = manager.list_results(), manager.sampling_state() + policy = SamplingPolicy() + from reme_home.preparation.data.models import CaseSpec + + cases = [ + CaseSpec.model_validate_json(path.read_text()) + for path in sorted( + (prepared / "datasets/training-set/cases").glob("*.json"), + ) + ] + monkeypatch.setattr("reme_home.sampling.time.time_ns", lambda: 5678) + before = create_sampling_plan(cases, observations, policy) + original = FakeFactory.create_cases + + async def create(factory, *args, **kwargs): + workers = await original(factory, *args, **kwargs) + for worker in workers: + build = worker.run_build + + async def fail_one(jobs, worker=worker, build=build): + if worker.case_id == "2": + raise ConnectionError("only one case failed") + return await build(jobs) + + worker.run_build = fail_one + return workers + + monkeypatch.setattr(FakeFactory, "create_cases", create) + failed = await app.run_job("home_evaluate", concurrency=1) + assert not failed.success + assert failed.answer["scored_query_count"] == 1 # Its healthy case is also excluded from feedback. + assert manager.record(Path(failed.answer["result_dir"])) is None + assert manager.list_results() == records and manager.sampling_state() == observations + assert create_sampling_plan(cases, manager.sampling_state(), policy) == before + assert read_state(prepared)["round"] == 2 + assert read_state(prepared)["successful_round"] == 1 + assert manager.rebuild() == records + assert manager.sampling_state() == observations + + +@pytest.mark.asyncio +async def test_history_and_observations_rebuild_without_touching_artifacts(app, prepared): + baseline = await app.run_job("home_evaluate") + sampled = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_cases": 1}) + assert baseline.success and sampled.success + manager = history(app) + expected, observations = manager.list_results(), manager.sampling_state() + source = Path(sampled.answer["result_dir"]) / "summary.json" + original = source.read_bytes() + (prepared / "evaluations/history.json").unlink() + (prepared / "sampling/state.json").unlink() + assert manager.list_results() == expected + assert manager.sampling_state() == observations + assert manager.rebuild() == expected + assert source.read_bytes() == original + assert (prepared / "evaluations/history.json").is_file() + assert (prepared / "sampling/state.json").is_file() + + +@pytest.mark.asyncio +async def test_call_policy_merges_component_defaults_without_mutating_them(app, prepared): + evaluator = app.context.components["home.evaluation"]["default"] + evaluator.sampling_policy = SamplingPolicy(max_cases=2, max_queries=2, temperature=0.7, ewma_alpha=0.6) + response = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_cases": 1}) + assert response.success, response.answer + options = manifest(response)["options"] + assert options["sampling"] is True + assert options["sampling_policy"] == { + "max_cases": 1, + "max_queries": 2, + "temperature": 0.7, + "ewma_alpha": 0.6, + "uniform_exploration": 0.2, + } + assert response.answer["case_count"] == 1 + following = await app.run_job("home_evaluate", sampling=True) + assert following.success and following.answer["case_count"] == 2 + assert evaluator.sampling_policy.max_cases == 2 + assert manifest(following)["options"]["sampling_policy"]["max_cases"] == 2 + for kwargs in ( + {"sampling": "true"}, + {"sampling": {"max_cases": 1}}, + {"record_history": "false"}, + {"sampling_policy": {"max_cases": 0}}, + {"sampling_policy": {"unknown": 1}}, + ): + assert not (await app.run_job("home_evaluate", **kwargs)).success + assert len(history(app).list_results()) == 2 + + +@pytest.mark.asyncio +async def test_job_defaults_and_hidden_agent_injection_use_one_job(app, prepared): + job = app.context.jobs["home_evaluate"] + assert "home_validate" not in app.context.jobs + job.kwargs.update(sampling=True, sampling_policy={"max_cases": 1}) + default = await app.run_job("home_evaluate") + assert default.success and default.answer["case_count"] == 1 + explicit = await app.run_job("home_evaluate", sampling=False) + assert explicit.success and explicit.answer["case_count"] == 2 + tool = AsAgentWrapper._make_tool( + job, + injected_job_kwargs={ + "sampling": True, + "sampling_policy": {"max_cases": 1, "max_queries": 1}, + }, + ) + assert "sampling" not in tool.input_schema["properties"] + await tool.call() + records = history(app).list_results() + assert records[-1].case_count == records[-1].scored_query_count == 1 + assert len(records) == 3 + with pytest.raises(ValueError, match="injected"): + await tool.call(sampling=False) + with pytest.raises(ValueError, match="injected"): + await tool.call(sampling_policy={"max_cases": 100}) + assert len(history(app).list_results()) == 3 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("baseline", [False, True]) +async def test_standalone_full_evaluation_preserves_history_and_rebuild_excludes_it(app, prepared, baseline): + manager = history(app) + if baseline: + assert (await app.run_job("home_evaluate")).success + records, observations = manager.list_results(), manager.sampling_state() + cache_paths = [prepared / name for name in ("evaluations/history.json", "sampling/state.json")] + caches = [path.read_bytes() if path.exists() else None for path in cache_paths] + response = await app.run_job("home_evaluate", sampling=False, record_history=False) + assert response.success, response.answer + assert response.answer["case_count"] == response.answer["scored_query_count"] == 2 + assert response.answer["mean_query_score"] == 0.5 and response.answer["record_history"] is False + run = Path(response.answer["result_dir"]) + payload = manifest(response) + assert payload["status"] == "completed" and payload["success"] is True + assert payload["options"]["record_history"] is False and payload["sandbox_started_at"] + assert "sampling" not in payload and "selection_plan" not in payload + assert (run / "summary.json").is_file() and (run / "cases/1/queries/result.json").is_file() + assert manager.record_started(run, SamplingPolicy()) == {} + assert manager.record(run) is None + assert [path.read_bytes() if path.exists() else None for path in cache_paths] == caches + assert manager.list_results() == records and manager.sampling_state() == observations + assert manager.rebuild() == records and manager.sampling_state() == observations + following = await app.run_job("home_evaluate") + assert following.success and following.answer["record_history"] is True + state = read_state(prepared) + assert state["round"] == state["successful_round"] == 1 + int(baseline) + # Legacy manifests without the new flag keep the previous opt-in behavior. + tracked_run = Path(following.answer["result_dir"]) + tracked_manifest = manifest(following) + tracked_manifest["options"].pop("record_history") + (tracked_run / "manifest.json").write_text(json.dumps(tracked_manifest)) + assert len(manager.rebuild()) == 1 + int(baseline) + + +@pytest.mark.asyncio +async def test_untracked_full_evaluation_does_not_consult_history(app, prepared, monkeypatch): + manager = history(app) + + def fail(*args, **kwargs): + raise AssertionError("standalone full evaluation must not access history") + + for name in ("sampling_state", "record_started", "record", "invalidate"): + monkeypatch.setattr(manager, name, fail) + response = await app.run_job("home_evaluate", record_history=False) + assert response.success, response.answer + assert response.answer["case_count"] == response.answer["scored_query_count"] == 2 + + +@pytest.mark.asyncio +async def test_sampling_and_history_are_independent_and_selection_evidence_is_local(app, prepared): + assert (await app.run_job("home_evaluate")).success + manager = history(app) + records, observations = manager.list_results(), manager.sampling_state() + response = await app.run_job( + "home_evaluate", + sampling=True, + record_history=False, + sampling_policy={"max_cases": 1}, + ) + assert response.success and response.answer["case_count"] == 1 + payload = manifest(response) + assert "sampling" not in payload + assert payload["selection_plan"]["case_ids"] == payload["case_ids"] + assert payload["selection_plan"]["seed"] and payload["selection_plan"]["policy"]["max_cases"] == 1 + assert manager.rebuild() == records and manager.sampling_state() == observations + assert read_state(prepared)["round"] == read_state(prepared)["successful_round"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["infra", "exception"]) +async def test_untracked_failures_preserve_existing_history(app, prepared, monkeypatch, failure): + assert (await app.run_job("home_evaluate")).success + manager = history(app) + records, observations = manager.list_results(), manager.sampling_state() + paths = [prepared / name for name in ("evaluations/history.json", "sampling/state.json")] + caches = [path.read_bytes() for path in paths] + if failure == "infra": + monkeypatch.setattr(FakeFactory, "bad_export", True) + else: + original = FakeFactory.create_cases + + async def fail(factory, *args, **kwargs): + await original(factory, *args, **kwargs) + raise ConnectionError("pool failed after first initialized sandbox") + + monkeypatch.setattr(FakeFactory, "create_cases", fail) + response = await app.run_job("home_evaluate", record_history=False) + assert not response.success + assert [path.read_bytes() for path in paths] == caches + assert manager.rebuild() == records and manager.sampling_state() == observations + assert read_state(prepared)["round"] == read_state(prepared)["successful_round"] == 1 + + +@pytest.mark.asyncio +async def test_history_job_defaults_and_agent_injection_are_enforced(app, prepared): + job = app.context.jobs["home_evaluate"] + assert job.parameters["properties"]["record_history"]["default"] is True + job.kwargs["record_history"] = False + standalone = await app.run_job("home_evaluate") + assert standalone.success and not history(app).list_results() + baseline = await app.run_job("home_evaluate", sampling=False, record_history=True) + assert baseline.success and baseline.answer["case_count"] == 2 + tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"record_history": True}) + assert "record_history" not in tool.input_schema["properties"] + await tool.call() + assert len(history(app).list_results()) == 2 + with pytest.raises(ValueError, match="injected"): + await tool.call(record_history=False) + + +@pytest.mark.asyncio +async def test_legacy_cache_rebuild_and_explicit_null_query_budget(app, prepared): + evaluator = app.context.components["home.evaluation"]["default"] + evaluator.sampling_policy = SamplingPolicy(max_cases=2, max_queries=1) + result = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_queries": None}) + assert result.success and result.answer["scored_query_count"] == 2 + assert manifest(result)["options"]["sampling_policy"]["max_queries"] is None + assert evaluator.sampling_policy.max_queries == 1 + manager = history(app) + observations = manager.sampling_state() + legacy = read_state(prepared) + legacy["schema_version"] = 1 + legacy.pop("successful_round") + legacy["cases"]["1"]["ewma_loss"] = 999 + (prepared / "sampling/state.json").write_text(json.dumps(legacy)) + assert manager.sampling_state() == observations + manager.rebuild() + assert read_state(prepared)["schema_version"] == 2 + assert read_state(prepared)["successful_round"] == 1 + + +@pytest.mark.asyncio +async def test_cancellation_after_durable_completion_keeps_history_consistent(app, prepared, monkeypatch): + import asyncio + from threading import Event + + manager = history(app) + original = manager.record + entered, release = Event(), Event() + + def blocked(run): + entered.set() + assert release.wait(5) + return original(run) + + monkeypatch.setattr(manager, "record", blocked) + task = asyncio.create_task(app.run_job("home_evaluate")) + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + records = manager.list_results() + assert len(records) == 1 + run = prepared / "evaluations" / records[0].run_key + completed = json.loads((run / "manifest.json").read_text()) + assert completed["status"] == "completed" and completed["success"] is True + assert manager.rebuild() == records + + +@pytest.mark.asyncio +async def test_registration_failure_invalidates_only_caches(app, prepared, monkeypatch): + manager = history(app) + original = manager._index + + def fail(): + raise OSError("history index could not be loaded") + + monkeypatch.setattr(manager, "_index", fail) + response = await app.run_job("home_evaluate") + assert not response.success + runs = list((prepared / "evaluations").glob("*/*/*/manifest.json")) + assert len(runs) == 1 + run = runs[0].parent + failed = json.loads(runs[0].read_text()) + assert failed["status"] == "failed" and failed["success"] is False + assert (run / "summary.json").is_file() and (run / "cases/1/queries/result.json").is_file() + assert not (prepared / "sampling/state.json").exists() + assert read_state(prepared)["round"] == 1 and read_state(prepared)["successful_round"] == 0 + monkeypatch.setattr(manager, "_index", original) + assert manager.list_results() == [] and not manager.sampling_state()["cases"] diff --git a/plugins/home/tests/test_evaluation_inspection.py b/plugins/home/tests/test_evaluation_inspection.py new file mode 100644 index 00000000..8fe604ed --- /dev/null +++ b/plugins/home/tests/test_evaluation_inspection.py @@ -0,0 +1,312 @@ +"""Read-only result Jobs against real Home applications and isolated Git history.""" + +# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access + +import asyncio +from pathlib import Path +from threading import Event + +import pytest +from reme_home.evaluation import inspection_step as inspection +from reme_home.evaluation.artifacts import write_json + +from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import + FakeFactory, + home_application, + preparation_inputs, + prepared_home, +) +from test_preparation import command + +from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper + + +async def call(app, name, **kwargs): + response = await app.run_job(name, **kwargs) + assert response.success, response.answer + return response.answer + + +def snapshot(home): + return {path.relative_to(home): path.read_bytes() for path in home.rglob("*") if path.is_file()} + + +def write_run(home, run_id="1", *, branch="candidate", commit="a" * 40, **updates): + run = home / "evaluations" / branch / commit[:7] / run_id + manifest = { + "branch_name": branch, + "commit_sha": commit, + "validation_id": run_id, + "case_ids": ["case~a", "missing-case"], + "query_ids": {"case~a": ["q1", "q2"], "missing-case": ["q3"]}, + "mode": "scored", + "dataset_kind": "training-set", + "status": "failed", + "started_at": f"2026-01-01T00:00:{int(run_id):02d}+00:00", + "options": {"record_history": False}, + **updates, + } + write_json(run / "manifest.json", manifest) + return run + + +@pytest.mark.asyncio +async def test_real_evaluation_catalog_and_case_query_navigation_are_read_only(app, prepared): + scored = await call(app, "home_evaluate", record_history=False) + debug = await call(app, "home_evaluate", debug=True) + before = snapshot(prepared) + page = await call(app, "home_list_validation_results", limit=1) + assert page["row_count"] == 2 and page["next_offset"] == 1 + assert page["rows"][0]["mode"] == "debug" and page["rows"][0]["record_history"] is False + last = await call(app, "home_list_validation_results", offset=1, limit=1) + assert last["next_offset"] is None and last["rows"][0]["validation_path"] == scored["result_dir"] + assert last["rows"][0]["totals"]["scored_query_count"] == 2 + run = Path(scored["result_dir"]) + result = await call(app, "home_inspect_validation", validation_path=run.relative_to(prepared).as_posix(), limit=1) + assert result["row_count"] == 2 and result["next_offset"] == 1 + case = result["rows"][0] + assert case["mean_score"] == 0.5 and case["pointer"].startswith("/cases/") + result = await call(app, "home_inspect_validation", validation_path=str(run), case_id=case["case_id"]) + query = result["rows"][0] + assert query["score"] == 0.5 and query["pointer"] == "/queries/0" + assert "fields" not in query and result["construction"]["failed_jobs"]["rows"] == [] + result = await call( + app, + "home_inspect_validation", + validation_path=str(run), + case_id=case["case_id"], + query_id=query["query_id"], + ) + fields = result["rows"][0]["fields"] + assert fields["answer"] == "hello" and fields["score"] == 0.5 and "judge_result" in fields + assert result["query_missing"] is False and result["answer_log"].endswith("answer.log") + assert snapshot(prepared) == before + assert app.context.components["home.evaluation_history"]["default"].list_results() == [] + assert debug["result_dir"] != scored["result_dir"] + + +@pytest.mark.asyncio +async def test_partial_missing_and_corrupt_runs_are_visible_without_rebuilding(app, prepared): + run = write_run(prepared) + write_json( + run / "partial_summary.json", + { + "case_count": 1, + "cases": { + "case~a": { + "mean_score": 0.0, + "construction": {"status": "partial_success"}, + "construction_error_count": 2, + }, + }, + }, + ) + write_json(run / "failure.json", {"error": "failure" * 300}) + missing = write_run(prepared, "2", mode="debug", status="running") + corrupt = write_run(prepared, "3") + (corrupt / "manifest.json").write_text("invalid JSON") + before = snapshot(prepared) + result = await call(app, "home_list_validation_results") + assert result["row_count"] == 3 + assert sum("read_error" in row for row in result["rows"]) == 1 + row = next(row for row in result["rows"] if row["validation_path"] == str(run)) + assert row["summary_path"].endswith("partial_summary.json") and "truncated" in row["failure"] + result = await call(app, "home_inspect_validation", validation_path=str(run)) + assert result["rows"][0]["case_id"] == "missing-case" and result["rows"][0]["mean_score"] is None + assert result["rows"][1]["pointer"] == "/cases/case~0a" + result = await call(app, "home_inspect_validation", validation_path=str(missing), case_id="case~a", query_id="q1") + assert result["query_missing"] and result["rows"][0]["score"] is None + assert len(result["missing_files"]) == 2 + result = await call(app, "home_inspect_validation", validation_path=str(missing)) + assert result["summary_missing"] and result["rows_source"] is None and result["row_count"] == 2 + assert snapshot(prepared) == before and not (missing / "partial_summary.json").exists() + + +@pytest.mark.asyncio +async def test_failure_and_query_paging_previews_and_source_coordinates(app, prepared): + run = write_run(prepared) + folder = run / "cases/case~a" + write_json( + folder / "memory_construction/result.json", + { + "success": False, + "jobs": [{"job": f"job-{index}", "result": {"success": False, "error": "x" * 900}} for index in range(3)], + }, + ) + write_json( + folder / "queries/result.json", + { + "queries": [ + { + "query_id": "q2", + "score": 0, + "answer": "a" * 3000, + "judge_result": {"success": True}, + "golden_answer": ["rubric"], + "answer_result": None, + }, + {"query_id": "q1", "score": None, "error": "e" * 900}, + ], + }, + ) + result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", limit=1) + assert result["rows"][0]["query_id"] == "q1" and result["rows"][0]["pointer"] == "/queries/1" + assert result["next_offset"] == 1 and "truncated" in result["rows"][0]["error"] + failed_jobs = result["construction"]["failed_jobs"] + assert failed_jobs["row_count"] == 3 and failed_jobs["next_offset"] == 1 + assert failed_jobs["rows"][0]["pointer"] == "/jobs/0/result" + result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", offset=2, limit=1) + assert result["rows"] == [] and result["construction"]["failed_jobs"]["rows"][0]["job"] == "job-2" + result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", query_id="q2") + assert result["rows"][0]["pointer"] == "/queries/0" + assert "truncated" in result["rows"][0]["fields"]["answer"] + assert result["rows"][0]["fields"]["golden_answer"] == ["rubric"] + assert result["rows"][0]["score"] == 0 + + +@pytest.mark.asyncio +async def test_prefix_resolves_on_branch_history_not_only_evaluated_commits(app, prepared): + repo = prepared / "code/repo" + initial = command(repo, "rev-parse", "HEAD") + await call(app, "home_create_candidate_branch", branch_name="candidate") + write_run(prepared, commit=initial) + write_run(prepared, branch="other", commit=initial) + result = await call(app, "home_list_validation_results", commit_sha=initial[:7].upper()) + assert result["branch_name"] == "candidate" and result["commit_sha"] == initial + assert result["row_count"] == 1 + # An unevaluated reachable commit can make a short prefix ambiguous. + tree = command(repo, "rev-parse", "HEAD^{tree}") + for index in range(500): + sha = command( + repo, + "-c", + "user.name=Home", + "-c", + "user.email=home@local", + "commit-tree", + tree, + "-p", + initial, + "-m", + f"unevaluated {index}", + ) + if sha[0] == initial[0]: + break + else: + pytest.fail("Could not construct a prefix collision") + command(repo, "update-ref", "refs/heads/candidate", sha) + response = await app.run_job("home_list_validation_results", commit_sha=initial[0]) + assert not response.success and "2 matches" in response.answer + # The very same prefix remains unique on init, despite the other Git object. + result = await call(app, "home_list_validation_results", branch_name="init", commit_sha=initial[0]) + assert result["commit_sha"] == initial and result["row_count"] == 0 + result = await call(app, "home_list_validation_results", commit_sha=sha) + assert result["commit_sha"] == sha and result["rows"] == [] + absent = "f" * len(initial) + assert not (await app.run_job("home_list_validation_results", commit_sha=absent)).success + + +@pytest.mark.asyncio +async def test_branch_filters_and_deleted_historical_branch_do_not_require_git(app, prepared): + write_run(prepared, branch="removed") + write_run(prepared, "2", branch="candidate") + result = await call(app, "home_list_validation_results", branch_name="removed") + assert result["row_count"] == 1 and result["rows"][0]["branch_name"] == "removed" + response = await app.run_job("home_list_validation_results", branch_name="removed", commit_sha="a") + assert not response.success + + +@pytest.mark.asyncio +@pytest.mark.parametrize("corruption", ["summary", "options"]) +async def test_commit_filtered_catalog_reports_corrupt_run_with_identity(app, prepared, corruption): + sha = command(prepared / "code/repo", "rev-parse", "HEAD") + run = write_run(prepared, branch="init", commit=sha, options=None if corruption == "options" else {}) + if corruption == "summary": + write_json(run / "summary.json", []) + result = await call(app, "home_list_validation_results", commit_sha=sha[:7]) + assert result["row_count"] == 1 + assert result["rows"][0]["commit_sha"] == sha and "read_error" in result["rows"][0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "job,kwargs", + [ + ("home_list_validation_results", {"offset": -1}), + ("home_list_validation_results", {"limit": 0}), + ("home_list_validation_results", {"limit": 101}), + ("home_list_validation_results", {"limit": True}), + ("home_list_validation_results", {"offset": "1"}), + ("home_list_validation_results", {"commit_sha": "HEAD"}), + ("home_list_validation_results", {"commit_sha": ""}), + ("home_list_validation_results", {"branch_name": "../init"}), + ("home_inspect_validation", {}), + ("home_inspect_validation", {"validation_path": "evaluations"}), + ("home_inspect_validation", {"validation_path": "../outside"}), + ("home_inspect_validation", {"validation_path": "/tmp/outside"}), + ("home_inspect_validation", {"validation_path": "evaluations/a/b/c", "query_id": "q"}), + ("home_inspect_validation", {"validation_path": "evaluations/a/b/c", "case_id": "a/b"}), + ("home_list_validation_results", {"_allowed_paths": []}), + ], +) +async def test_invalid_input_fails_closed(app, prepared, job, kwargs): + assert not (await app.run_job(job, **kwargs)).success + + +@pytest.mark.asyncio +async def test_unknown_ids_and_symlink_artifacts_are_rejected(app, prepared, tmp_path): + run = write_run(prepared) + assert not (await app.run_job("home_inspect_validation", validation_path=str(run), case_id="other")).success + assert not ( + await app.run_job( + "home_inspect_validation", + validation_path=str(run), + case_id="case~a", + query_id="other", + ) + ).success + secret = tmp_path / "secret.json" + secret.write_text('{"secret": "must not be read"}') + (run / "summary.json").symlink_to(secret) + response = await app.run_job("home_inspect_validation", validation_path=str(run)) + assert not response.success and "must not be read" not in response.answer + result = await call(app, "home_list_validation_results") + assert "read_error" in result["rows"][0] and "secret" not in str(result["rows"]) + (run / "summary.json").unlink() + (run / "cases").symlink_to(tmp_path, target_is_directory=True) + assert not (await app.run_job("home_inspect_validation", validation_path=str(run), case_id="case~a")).success + + +@pytest.mark.asyncio +async def test_hidden_case_scope_is_enforced_by_agent_job_injection(app, prepared): + run = write_run(prepared) + job = app.context.jobs["home_inspect_validation"] + tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"validation_path": str(run), "case_id": "case~a"}) + assert "validation_path" not in tool.input_schema["properties"] and "case_id" not in tool.input_schema["properties"] + await tool.call(query_id="q1") + with pytest.raises(ValueError, match="injected"): + await tool.call(case_id="missing-case") + with pytest.raises(ValueError, match="injected"): + await tool.call(validation_path="evaluations/other/commit/1") + + +@pytest.mark.asyncio +async def test_cancelled_job_drains_read_thread(app, prepared, monkeypatch): + entered, release, finished = Event(), Event(), Event() + + def read(*args): + entered.set() + assert release.wait(5) + finished.set() + return {} + + monkeypatch.setattr(inspection, "_list_results", read) + task = asyncio.create_task(app.run_job("home_list_validation_results")) + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + await asyncio.sleep(0) + assert not task.done() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert finished.is_set() diff --git a/plugins/home/tests/test_memory_inspection.py b/plugins/home/tests/test_memory_inspection.py new file mode 100644 index 00000000..dd3482c9 --- /dev/null +++ b/plugins/home/tests/test_memory_inspection.py @@ -0,0 +1,230 @@ +"""Memory evidence Jobs against isolated real Git repositories.""" + +# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access + +import shutil + +import pytest +from pydantic import ValidationError +from reme_home.evaluation import memory_inspection_step as memory +from reme_home.preparation.code import git + +from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import + FakeFactory, + home_application, + preparation_inputs, + prepared_home, +) +from test_evaluation_inspection import call, snapshot, write_run + + +@pytest.fixture +def evidence(tmp_path): + home = tmp_path / "home" + run = write_run(home, case_ids=["case1"]) + repo = run / "cases/case1/memory_construction/reme_workspace" + repo.mkdir(parents=True) + git(repo, "init") + git(repo, "config", "user.name", "test") + git(repo, "config", "user.email", "test@localhost") + (repo / "memory").mkdir() + (repo / "memory/note.md").write_text("old fact\n") + (repo / "removed.txt").write_text("removed\n") + first = commit(repo, "session: session1") + (repo / "memory/note.md").write_text("new fact\n") + (repo / "removed.txt").unlink() + (repo / "added.bin").write_bytes(b"\x00\xff") + second = commit(repo, "session: session2") + return home, run, repo, first, second + + +def commit(repo, subject): + git(repo, "add", "--all") + git(repo, "commit", "-m", subject) + return git(repo, "rev-parse", "HEAD") + + +def history(run, **kwargs): + return memory.MemoryHistoryInput(validation_path=str(run), case_id="case1", **kwargs) + + +def comparison(run, **kwargs): + return memory.CompareMemoryInput(validation_path=str(run), case_id="case1", **kwargs) + + +def test_history_paging_filter_and_no_writes(evidence): + home, run, repo, first, second = evidence + (repo / "memory/note.md").write_text("dirty checkout ignored\n") + before = snapshot(home) + result = memory.inspect_history(home, history(run, limit=1)) + assert result["head"] == second and result["row_count"] == 2 + assert result["rows"][0]["session_id"] == "session2" and result["next_offset"] == 1 + result = memory.inspect_history(home, history(run, offset=1, limit=1)) + assert result["rows"][0]["commit_sha"] == first and result["next_offset"] is None + assert memory.inspect_history(home, history(run, path="added.bin"))["row_count"] == 1 + assert memory.inspect_history(home, history(run, path="memory/*"))["rows"] == [] + assert snapshot(home) == before + + +def test_same_repository_diff_and_unique_prefix(evidence): + home, run, repo, first, second = evidence + (repo / "memory/note.md").write_text("dirty checkout ignored\n") + before = snapshot(home) + result = memory.compare_memory(home, comparison(run, base_ref=first[:7].upper())) + assert result["base_commit"] == first and result["target_commit"] == second + assert result["added"] == ["added.bin"] and result["removed"] == ["removed.txt"] + assert result["modified"] == ["memory/note.md"] + assert "-old fact" in result["diff"] and "+new fact" in result["diff"] + assert "Binary files differ" in result["diff"] and not result["truncated"] + assert "dirty checkout" not in result["diff"] + assert memory.compare_memory(home, comparison(run, base_ref="HEAD~1", path="memory"))["added"] == [] + assert snapshot(home) == before + + +def test_cross_evaluation_final_snapshots(evidence): + home, run, _repo, _first, second = evidence + other = write_run(home, "2", case_ids=["case1"], commit="b" * 40) + target = other / "cases/case1/memory_construction/reme_workspace" + target.mkdir(parents=True) + git(target, "init") + git(target, "config", "user.name", "test") + git(target, "config", "user.email", "test@localhost") + (target / "target.txt").write_text("target\n") + target_head = commit(target, "new history") + before = snapshot(home) + result = memory.compare_memory(home, comparison(run, target_validation_path=str(other))) + assert result["base_commit"] == second and result["target_commit"] == target_head + assert result["added"] == ["target.txt"] + assert result["removed"] == ["added.bin", "memory/note.md"] + assert result["base_tree_sha256"] != result["target_tree_sha256"] + assert snapshot(home) == before + + +@pytest.mark.parametrize("state", ["workspace_missing", "history_missing", "history_empty"]) +def test_missing_and_empty_history(tmp_path, state): + home = tmp_path / "home" + run = write_run(home, case_ids=["case1"]) + repo = run / "cases/case1/memory_construction/reme_workspace" + if state != "workspace_missing": + repo.mkdir(parents=True) + if state == "history_empty": + git(repo, "init") + result = memory.inspect_history(home, history(run)) + assert result["status"] == state and result["rows"] == [] and not result["has_git_history"] + with pytest.raises(ValueError, match="requires memory commits"): + memory.compare_memory(home, comparison(run, base_ref="HEAD")) + + +@pytest.mark.parametrize( + "kwargs", + [ + {"offset": -1}, + {"limit": 101}, + {"limit": True}, + {"case_id": "../bad"}, + {"path": "../escape"}, + {"path": ".git/config"}, + ], +) +def test_input_validation(kwargs): + with pytest.raises(ValidationError): + memory.MemoryHistoryInput.model_validate({"validation_path": "a", "case_id": "case1", **kwargs}) + + +def test_comparison_requires_base(): + with pytest.raises(ValidationError, match="base_ref is required"): + comparison("a") + + +@pytest.mark.parametrize("kind", ["gitfile", "symlink", "alternates", "corrupt"]) +def test_invalid_or_external_git_storage(evidence, tmp_path, kind): + home, run, repo, _first, _second = evidence + if kind == "gitfile": + (repo / ".git").rename(repo / "saved-git") + (repo / ".git").write_text("gitdir: saved-git\n") + elif kind == "symlink": + (repo / ".git/objects/external").symlink_to(tmp_path) + elif kind == "alternates": + (repo / ".git/objects/info/alternates").write_text(str(tmp_path)) + else: + (repo / ".git/HEAD").write_text("broken\n") + with pytest.raises((ValueError, memory.subprocess.CalledProcessError)): + memory.inspect_history(home, history(run)) + + +def test_scope_and_limits(evidence, monkeypatch): + home, run, _repo, first, _second = evidence + with pytest.raises(ValueError, match="case_id"): + memory.inspect_history(home, memory.MemoryHistoryInput(validation_path=str(run), case_id="other")) + with pytest.raises(ValueError): + memory.inspect_history(home, history(home)) + monkeypatch.setattr(memory, "MAX_OUTPUT", 20) + result = memory.compare_memory(home, comparison(run, base_ref=first)) + assert result["truncated"] and len(result["diff"]) <= 20 + monkeypatch.setattr(memory, "MAX_TREE_FILES", 1) + with pytest.raises(ValueError, match="2000 files"): + memory.compare_memory(home, comparison(run, base_ref=first)) + + +def test_ambiguous_and_unreachable_revisions(evidence): + home, run, repo, first, second = evidence + # Publish real commits until two reachable SHAs share a one-character prefix. + previous = second + seen = {first[0]: first, second[0]: second} + for index in range(100): + sha = git(repo, "commit-tree", "HEAD^{tree}", "-p", previous, "-m", str(index)) + previous = sha + if sha[0] in seen: + prefix = sha[0] + break + seen[sha[0]] = sha + git(repo, "update-ref", "HEAD", previous) + with pytest.raises(ValueError, match="matches reachable"): + memory.compare_memory(home, comparison(run, base_ref=prefix)) + with pytest.raises(ValueError, match="0 matches"): + memory.compare_memory(home, comparison(run, base_ref="f" * 40)) + detached = git(repo, "commit-tree", "HEAD^{tree}", "-m", "unreachable") + with pytest.raises(memory.subprocess.CalledProcessError): + memory.compare_memory(home, comparison(run, base_ref=detached + "~0")) + + +@pytest.mark.asyncio +async def test_registered_jobs_and_restricted_scope(app, prepared, evidence): + run = write_run(prepared, case_ids=["case1"]) + result = await call(app, "home_inspect_memory_history", validation_path=str(run), case_id="case1") + assert result["status"] == "workspace_missing" + response = await app.run_job("home_compare_memory", validation_path=str(run), case_id="case1") + assert not response.success and "base_ref" in str(response.answer) + response = await app.run_job( + "home_inspect_memory_history", + validation_path=str(run), + case_id="case1", + _allowed_paths=[str(run)], + ) + assert not response.success and "unrestricted" in str(response.answer) + + _home, _run, repository, first, second = evidence + destination = run / "cases/case1/memory_construction/reme_workspace" + shutil.copytree(repository, destination) + result = await call(app, "home_compare_memory", validation_path=str(run), case_id="case1", base_ref=first) + assert result["target_commit"] == second and result["modified"] == ["memory/note.md"] + + +def test_unterminated_lines_and_snapshot_byte_limit(evidence, monkeypatch): + home, run, repo, _first, second = evidence + (repo / "memory/note.md").write_text("unterminated") + commit(repo, "ordinary checkpoint") + result = memory.compare_memory(home, comparison(run, base_ref=second, path="memory")) + assert "+unterminated\n\\ No newline at end of file\n" in result["diff"] + assert memory.inspect_history(home, history(run))["rows"][0]["session_id"] is None + monkeypatch.setattr(memory, "MAX_TREE_BYTES", 1) + with pytest.raises(ValueError, match="20 MB"): + memory.compare_memory(home, comparison(run, base_ref=second)) + + +def test_tracked_symlinks_are_not_followed(evidence): + home, run, repo, _first, second = evidence + (repo / "memory/link").symlink_to("/etc/passwd") + commit(repo, "link") + with pytest.raises(ValueError, match="regular files"): + memory.compare_memory(home, comparison(run, base_ref=second)) diff --git a/plugins/home/tests/test_sampling.py b/plugins/home/tests/test_sampling.py new file mode 100644 index 00000000..2ca98fdc --- /dev/null +++ b/plugins/home/tests/test_sampling.py @@ -0,0 +1,412 @@ +"""Adaptive sampling parity and actual-start accounting through real Home Jobs.""" + +# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access + +import asyncio +import copy +import json +from pathlib import Path + +import pytest +from reme_home import sampling +from reme_home.evaluation import history as history_module +from reme_home.preparation.data.models import CaseSpec, QuerySpec +from reme_home.workspace import _initialization_lock +from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import + FakeFactory, + home_application, + preparation_inputs, + prepared_home, +) + + +def cases(): + return [ + CaseSpec( + case_id=key, + sessions=[], + queries=[ + QuerySpec(query_id=f"query-{key}-{index}", question="?", golden_answer="answer") for index in range(4) + ], + ) + for key in "abc" + ] + + +def test_plan_matches_dev_meta_golden_draw(monkeypatch): + # Golden output computed with dev_meta b53048e4 sampling.py, seed=123456. + monkeypatch.setattr(sampling.time, "time_ns", lambda: 123456) + state = { + "round": 2, + "cases": { + "a": { + "observed_count": 2, + "ewma_loss": 0.9, + "last_selected_round": 2, + "last_error_round": 1, + }, + }, + "queries": {}, + } + before = copy.deepcopy(state) + policy = sampling.SamplingPolicy(max_cases=2, max_queries=3, temperature=0.8) + plan = sampling.create_sampling_plan(cases(), state, policy) + assert plan["case_ids"] == ["a", "b"] + assert plan["query_ids"] == {"a": ["query-a-2", "query-a-0"], "b": ["query-b-3"]} + assert plan["case_draw_weights"] == pytest.approx( + { + "a": 0.40033106216673214, + "b": 0.29983446891663396, + "c": 0.29983446891663396, + }, + ) + assert plan["query_draw_weights"]["a"] == {f"query-a-{index}": 0.25 for index in range(4)} + assert state == before + assert sampling.create_sampling_plan(cases(), state, policy) == plan + + +def test_replay_eligibility_and_unrestricted_next_draw(monkeypatch): + monkeypatch.setattr(sampling.time, "time_ns", lambda: 100) + policy = sampling.SamplingPolicy(max_cases=3, max_queries=3) + eligible = {"b": ["query-b-1", "query-b-3"], "c": ["query-c-2"]} + plan = sampling.create_sampling_plan(cases(), {}, policy, eligible_query_ids=eligible) + assert set(plan["case_ids"]) == {"b", "c"} + assert all(set(values) <= set(eligible[key]) for key, values in plan["query_ids"].items()) + full = sampling.create_sampling_plan(cases(), {}, policy) + assert set(full["case_draw_weights"]) == {"a", "b", "c"} + assert full["eligibility"] is None + + +@pytest.mark.parametrize( + "options", + [ + {"max_cases": 0}, + {"max_queries": 0}, + {"max_cases": True}, + {"max_queries": True}, + {"temperature": 0}, + {"temperature": float("nan")}, + {"temperature": float("inf")}, + {"ewma_alpha": 0}, + {"ewma_alpha": 1.1}, + {"uniform_exploration": -0.1}, + {"uniform_exploration": 1.1}, + {"unexpected": 1}, + ], +) +def test_policy_rejects_invalid_limits(options): + with pytest.raises(ValueError): + sampling.SamplingPolicy(**options) + + +def test_uniform_mass_and_unobserved_priority(): + records = { + "easy": {"ewma_loss": 0.0, "observed_count": 100, "last_selected_round": 10}, + "hard": {"ewma_loss": 1.0, "observed_count": 1, "last_selected_round": 1, "last_error_round": 1}, + "unseen": {}, + } + weights = sampling._mixed_weights(records, 11, sampling.SamplingPolicy(uniform_exploration=0.3)) + assert sum(weights.values()) == pytest.approx(1) + assert all(weight >= 0.1 for weight in weights.values()) + assert weights["hard"] > weights["easy"] and weights["unseen"] > weights["easy"] + uniform = sampling._mixed_weights(records, 11, sampling.SamplingPolicy(uniform_exploration=1)) + assert set(uniform.values()) == {1 / 3} + + +def state(home): + return history_module.read_state(home) + + +@pytest.mark.asyncio +async def test_baseline_then_adaptive_job_updates_only_selected_cases(app, prepared): + baseline = await app.run_job("home_evaluate") + assert baseline.success, baseline.answer + assert state(prepared)["round"] == 1 + assert all(record["ewma_loss"] == 0.5 for record in state(prepared)["cases"].values()) + evaluation = app.context.components["home.evaluation"]["default"] + evaluation.sampling_policy = sampling.SamplingPolicy(max_cases=1, max_queries=1, ewma_alpha=0.5) + app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": '{"score": 1}'} + before = copy.deepcopy(state(prepared)) + response = await app.run_job("home_evaluate", sampling=True) + assert response.success, response.answer + run = Path(response.answer["result_dir"]) + manifest = json.loads((run / "manifest.json").read_text()) + plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text()) + selected = manifest["case_ids"][0] + unselected = ({"1", "2"} - {selected}).pop() + assert manifest["case_ids"] == plan["case_ids"] and manifest["query_ids"] == plan["query_ids"] + assert plan["mode"] == "sampled" and plan["round"] == 2 + assert response.answer["case_count"] == response.answer["scored_query_count"] == 1 + after = state(prepared) + assert after["cases"][selected]["ewma_loss"] == 0.25 + assert after["cases"][selected]["observed_count"] == 2 + assert after["cases"][selected]["successful_observations_after_last_error"] == 1 + assert after["cases"][unselected] == before["cases"][unselected] + evaluation._finish_sampling(run) + assert state(prepared) == after # A summary is consumed once, including after reopening. + assert "home_validate" not in app.context.jobs + assert app.context.jobs["home_evaluate"].parameters["properties"]["sampling"]["type"] == "boolean" + assert "sampling_policy" not in app.context.jobs["home_evaluate"].parameters["properties"] + assert "case_ids" not in app.context.jobs["home_evaluate"].parameters["properties"] + + +@pytest.mark.asyncio +async def test_debug_never_reads_or_changes_sampling_state(app, prepared): + before = prepared / "sampling/state.json" + for enabled in (False, True): + response = await app.run_job("home_evaluate", sampling=enabled, debug=True) + assert response.success, response.answer + assert not before.exists() + assert not list((prepared / "sampling/plans").rglob("*.json")) + # Corrupt sampling state cannot prevent an unrelated debug smoke test. + before.write_text("corrupt cached state") + response = await app.run_job("home_evaluate", sampling=True, debug=True) + assert response.success and before.read_text() == "corrupt cached state" + manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text()) + assert "sandbox_started_at" not in manifest and "sampling" not in manifest + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["dirty", "factory_constructor", "startup", "selection", "replay"]) +async def test_prestart_failure_never_counts(app, prepared, monkeypatch, failure): + await app.run_job("home_evaluate") + previous = (prepared / "sampling/state.json").read_bytes() + plans = sorted((prepared / "sampling/plans").rglob("*.json")) + options = {} + if failure == "dirty": + (prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py").write_text("dirty") + elif failure == "selection": + options["case_ids"] = ["1"] + elif failure == "replay": + options["reuse_construction_from"] = "evaluations/not-a-run" + elif failure == "factory_constructor": + + def fail(*args, **kwargs): + raise RuntimeError("factory never created") + + monkeypatch.setattr("reme_home.evaluation.component.DockerReMeSandboxFactory", fail) + else: + + async def fail(*args, **kwargs): + raise RuntimeError("no sandbox initialized") + + monkeypatch.setattr(FakeFactory, "create_cases", fail) + response = await app.run_job("home_evaluate", sampling=True, **options) + assert not response.success + assert (prepared / "sampling/state.json").read_bytes() == previous + assert sorted((prepared / "sampling/plans").rglob("*.json")) == plans + + +@pytest.mark.asyncio +async def test_started_infrastructure_failure_is_not_a_quality_zero(app, prepared, monkeypatch): + async def fail(*args, **kwargs): + raise ConnectionError("construction provider unavailable") + + monkeypatch.setattr("test_evaluation.FakeWorker.run_build", fail) + response = await app.run_job("home_evaluate", case_ids=["1"]) + assert not response.success + observed = state(prepared) + assert observed["round"] == 1 + assert not observed["cases"] and not observed["queries"] + assert app.context.components["home.evaluation_history"]["default"].list_results() == [] + + +@pytest.mark.asyncio +async def test_started_candidate_failure_does_not_fabricate_a_score(app, prepared, monkeypatch): + factory = FakeFactory.create_cases + + async def create(self, *args, **kwargs): + workers = await factory(self, *args, **kwargs) + + async def empty(_queries): + return {"success": True, "queries": [{"query_id": "fact:1", "answer": "", "error": None}]} + + for worker in workers: + worker.run_queries = empty + return workers + + monkeypatch.setattr(FakeFactory, "create_cases", create) + response = await app.run_job("home_evaluate", case_ids=["1"]) + assert response.success, response.answer + observed = state(prepared) + assert observed["round"] == 1 + record = observed["cases"]["1"] + assert record["candidate_failure_count"] == 1 + assert record["observed_count"] == 0 and record["ewma_loss"] is None + query = observed["queries"][sampling._query_key("1", "fact:1")] + assert query["candidate_failure_count"] == 1 and query["observed_count"] == 0 + + +@pytest.mark.asyncio +async def test_started_cancellation_retains_round_but_no_partial_quality(app, prepared, monkeypatch): + monkeypatch.setattr(FakeFactory, "block", asyncio.Event()) + task = asyncio.create_task(app.run_job("home_evaluate", sampling=True, concurrency=1)) + for _ in range(200): + if FakeFactory.instances and FakeFactory.instances[0].entered.is_set(): + break + await asyncio.sleep(0.01) + assert FakeFactory.instances[0].entered.is_set() + assert state(prepared)["round"] == 1 + concurrent = await app.run_job("home_evaluate", sampling=True) + assert not concurrent.success # Formal validations follow dev_meta's serialization boundary. + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + observed = state(prepared) + assert observed["round"] == 1 and not observed["cases"] and not observed["queries"] + assert not observed["processed_validations"] + with _initialization_lock(prepared / "metadata/validation.lock"): + pass + + +@pytest.mark.asyncio +async def test_replay_sampling_is_limited_to_donor_cases(app, prepared): + baseline = await app.run_job("home_evaluate", case_ids=["1"]) + assert baseline.success, baseline.answer + app.context.components["home.evaluation"]["default"].sampling_policy = sampling.SamplingPolicy(max_cases=1) + response = await app.run_job("home_evaluate", sampling=True, reuse_construction_from=baseline.answer["result_dir"]) + assert response.success, response.answer + manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text()) + assert manifest["case_ids"] == ["1"] + plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text()) + assert plan["eligibility"] == {"1": ["fact:1"]} + # The next full rebuild must see the entire training set again. + response = await app.run_job("home_evaluate", sampling=True) + manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text()) + plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text()) + assert set(plan["case_draw_weights"]) == {"1", "2"} and plan["eligibility"] is None + + +@pytest.mark.asyncio +async def test_sampling_cache_rebuild_and_new_round_after_reopen(app, prepared): + baseline = await app.run_job("home_evaluate") + assert baseline.success + expected = state(prepared) + (prepared / "sampling/state.json").unlink() + assert state(prepared) == expected + assert not (prepared / "sampling/state.json").exists() # Rebuilding reads never claim a new round. + with _initialization_lock(prepared / "metadata/workspace.lock"): + assert history_module.rebuild_sampling_state(prepared) == expected + result = await app.run_job("home_evaluate", sampling=True) + assert result.success, result.answer + assert state(prepared)["round"] == 2 + + +@pytest.mark.asyncio +async def test_fail_fast_started_round_without_summary_is_not_quality_feedback(app, prepared): + app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": "invalid judge output"} + response = await app.run_job("home_evaluate", sampling=True, fail_fast=True) + assert not response.success + observed = state(prepared) + assert observed["round"] == 1 + assert not observed["processed_validations"] and not observed["cases"] + assert not observed["queries"] + (prepared / "sampling/state.json").unlink() + assert state(prepared) == observed + + +@pytest.mark.asyncio +async def test_partial_pool_start_failure_counts_started_round(app, prepared, monkeypatch): + original = FakeFactory.create_cases + + async def fail_after_first_start(factory, *args, **kwargs): + await original(factory, *args, **kwargs) + raise ConnectionError("a later worker failed after first initialized") + + monkeypatch.setattr(FakeFactory, "create_cases", fail_after_first_start) + result = await app.run_job("home_evaluate", sampling=True) + assert not result.success + observed = state(prepared) + assert observed["round"] == 1 and not observed["processed_validations"] + assert not observed["cases"] and not observed["queries"] + + +@pytest.mark.asyncio +async def test_started_callback_is_idempotent(app, prepared): + response = await app.run_job("home_evaluate", sampling=True) + assert response.success + run = Path(response.answer["result_dir"]) + manifest = json.loads((run / "manifest.json").read_text()) + original = state(prepared) + component = app.context.components["home.evaluation"]["default"] + component._sampling_started(run, manifest, None) + assert state(prepared) == original + + +@pytest.mark.parametrize("missing", [1, 2]) +def test_replay_eligibility_uses_dev_meta_five_percent_threshold(tmp_path, missing): + from reme_home.evaluation.inputs import EvaluationSpec, replay_sampling_eligibility + from reme_home.evaluation.artifacts import write_json + + home = tmp_path / "home" + source = home / "evaluations/init/commit/1" + selected = [ + CaseSpec( + case_id=f"case-{index}", + sessions=[], + queries=[ + QuerySpec(query_id="q", question="?", golden_answer="answer"), + ], + ) + for index in range(20) + ] + write_json( + source / "manifest.json", + { + "dataset_kind": "training-set", + "mode": "scored", + "status": "completed", + "case_ids": [case.case_id for case in selected], + }, + ) + write_json(source / "summary.json", {}) + for case in selected[missing:]: + folder = source / "cases" / case.case_id + write_json(folder / "memory_construction/result.json", {"success": True}) + (folder / "memory_construction/reme_workspace.tar.gz").write_bytes(b"fixture") + write_json(folder / "queries/result.json", {"queries": [{"answer": "answer", "error": None}]}) + spec = EvaluationSpec(reuse_construction_from=str(source)) + if missing == 1: + eligible = replay_sampling_eligibility(home, spec, selected) + assert len(eligible) == 19 and "case-0" not in eligible + else: + with pytest.raises(ValueError, match="5%"): + replay_sampling_eligibility(home, spec, selected) + + +@pytest.mark.asyncio +async def test_query_budget_and_draw_order_reach_actual_execution(app, inputs, monkeypatch): + from unittest.mock import AsyncMock + + source = Path(inputs[0]["benchmark"]["source"]) + for path in source.glob("chats/*/*/probing_questions/probing_questions.json"): + payload = json.loads(path.read_text()) + payload["fact"] = [ + dict(payload["fact"][0], question=f"question {index}", rubric=["says hello"]) for index in range(4) + ] + path.write_text(json.dumps(payload)) + prepared = await app.run_job("home_prepare", spec=inputs[0]) + assert prepared.success + monkeypatch.setattr("reme_home.evaluation.component.DockerReMeSandboxFactory", FakeFactory) + monkeypatch.setattr( + app.context.components["agent_wrapper"]["judge"], + "reply", + AsyncMock(return_value={"result": '{"score": 0.5}'}), + ) + monkeypatch.setattr(sampling.time, "time_ns", lambda: 123456) + component = app.context.components["home.evaluation"]["default"] + component.sampling_policy = sampling.SamplingPolicy(max_cases=2, max_queries=3) + response = await app.run_job("home_evaluate", sampling=True, concurrency=1) + assert response.success, response.answer + run = Path(response.answer["result_dir"]) + manifest = json.loads((run / "manifest.json").read_text()) + home = Path(app.config.workspace_dir) / "home" + plan = json.loads((home / manifest["sampling"]["plan_path"]).read_text()) + assert manifest["query_ids"] == plan["query_ids"] + assert len(manifest["case_ids"]) == 2 and sum(map(len, manifest["query_ids"].values())) == 3 + worker = FakeFactory.instances[-1].workers[0] + assert [query.query_id for query in worker.queries] == manifest["query_ids"][worker.case_id] + for case_id, query_ids in manifest["query_ids"].items(): + result = json.loads((run / "cases" / case_id / "queries/result.json").read_text()) + assert [row["query_id"] for row in result["queries"]] == query_ids + assert response.answer["scored_query_count"] == 3 diff --git a/reme/components/agent_wrapper/as_agent_wrapper.py b/reme/components/agent_wrapper/as_agent_wrapper.py index 935c39e0..38eccbbb 100644 --- a/reme/components/agent_wrapper/as_agent_wrapper.py +++ b/reme/components/agent_wrapper/as_agent_wrapper.py @@ -299,6 +299,11 @@ class AsAgentWrapper(BaseAgentWrapper): """Resolve configured skill names to AgentScope local skill directories.""" return [str(path) for path in self._resolve_project_skills(skills).values()] + def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> Agent: + """Construct the SDK agent while subclasses reuse ReMe's tool and session wiring.""" + del options + return Agent(**kwargs) + async def _build_agent(self, inputs: Any, **kwargs) -> tuple[Agent, Any]: """Build an Agent instance from kwargs. Returns (agent, processed_inputs).""" model = self.as_llm.model if self.as_llm else None @@ -327,7 +332,8 @@ class AsAgentWrapper(BaseAgentWrapper): perm_mode = PermissionMode(kwargs.get("permission_mode", "bypass")) state = await self._load_state(kwargs, perm_mode) - agent = Agent( + agent = self._create_agent( + options=kwargs, name=self.name, system_prompt=system_prompt, model=model, diff --git a/tests/unit/test_benchmark_docker_start_callback.py b/tests/unit/test_benchmark_docker_start_callback.py new file mode 100644 index 00000000..2dfa5294 --- /dev/null +++ b/tests/unit/test_benchmark_docker_start_callback.py @@ -0,0 +1,84 @@ +"""Observe the first initialized sandbox without counting failed startup attempts.""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from deploy.benchmark_docker import DockerReMeSandboxFactory, ImageCandidate +from deploy.benchmark_docker import docker_sandbox + + +@pytest.mark.asyncio +async def test_start_callback_only_once_after_initialization(monkeypatch): + """Initialization is the boundary, including replacement workers and partial pool failure.""" + initialized, reported = [], [] + + async def initialize(case): + await asyncio.sleep(0) + initialized.append(case.case_id) + if case.case_id == "bad": + raise RuntimeError("startup failed") + + monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", initialize) + workspace = AsyncMock() + factory = DockerReMeSandboxFactory( + ImageCandidate("fixture"), + workspace_builder=lambda *_: workspace, + on_sandbox_started=lambda: reported.append(list(initialized)), + ) + with pytest.raises(RuntimeError, match="startup failed"): + await factory.create_case("bad") + assert not reported + await factory.create_case("good") + assert reported == [["bad", "good"]] + await factory.create_case("replacement") + assert len(reported) == 1 + + +@pytest.mark.asyncio +async def test_start_callback_failure_closes_initialized_resource(monkeypatch): + """Persisting a start marker must not orphan an already-created workspace on failure.""" + monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", AsyncMock()) + workspace = AsyncMock() + + def fail(): + raise OSError("start marker unavailable") + + factory = DockerReMeSandboxFactory( + ImageCandidate("fixture"), + workspace_builder=lambda *_: workspace, + on_sandbox_started=fail, + ) + with pytest.raises(OSError, match="start marker unavailable"): + await factory.create_case("created") + workspace.close.assert_awaited_once() + assert not factory._sandbox_started_reported # pylint: disable=protected-access + + +@pytest.mark.asyncio +async def test_partial_pool_start_reports_first_success_before_failure(monkeypatch): + """An initialized worker counts even if a subsequent worker prevents pool startup.""" + workspaces, started = [], [] + + def workspace_builder(*_args): + workspace = AsyncMock() + workspaces.append(workspace) + return workspace + + async def initialize(case): + if case.case_id == "bad": + raise ConnectionError("second worker failed") + + monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", initialize) + factory = DockerReMeSandboxFactory( + ImageCandidate("fixture"), + workspace_builder=workspace_builder, + on_sandbox_started=lambda: started.append(True), + ) + with pytest.raises(ConnectionError, match="second worker failed"): + await factory.create_cases(["good", "bad"], concurrency=1) + assert started == [True] + assert len(workspaces) == 2 + for workspace in workspaces: + workspace.close.assert_awaited_once()