feat(home): migrate agent, sampling and evidence inspection tooling

This commit is contained in:
酱牛肉 2026-10-29 16:57:58 +08:00
parent 95f9ea7b61
commit 91b7c14df4
41 changed files with 5428 additions and 20 deletions

View file

@ -158,6 +158,12 @@ the host application loads `home`. `source_ref: HEAD` exports committed source
only, excluding uncommitted modifications. Preparation failure rolls back its
outputs; after success, another call fails instead of resetting the experiment.
The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`,
`max_queries` (null means unlimited), `temperature`, `ewma_alpha` and
`uniform_exploration`. The runner validates these and passes them to the host
`home.evaluation` Component. It still only prepares; it does not run evaluations.
Each evaluation controls `sampling` and `record_history` separately.
Python callers can use the same lifecycle:
```python

View file

@ -149,4 +149,9 @@ python benchmark/beam/run_home.py
准备失败会回滚;成功后禁止再次准备同一工作区。
Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。
顶层 `sampling_policy` 配置采样默认值:`max_cases`、`max_queries`(null 表示不限)、
`temperature`、`ewma_alpha` 和 `uniform_exploration`。入口校验后将其传给宿主
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
和 `record_history`。
默认选择 BEAM **100K 全部 case**。

View file

@ -1,7 +1,13 @@
# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
workspace_dir: benchmark/beam/workspaces/home-100K
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
sampling_policy:
max_cases: 5
max_queries: null
temperature: 1.0
ewma_alpha: 0.3
uniform_exploration: 0.2
preparation:
schema_version: 1
benchmark:
name: beam
source: benchmark/beam/dataset/BEAM

View file

@ -9,6 +9,7 @@ from pydantic import BaseModel, ConfigDict, Field
import yaml
from reme_home.preparation.spec import PreparationSpec
from reme_home.judge import JUDGE_PLUGINS
from reme_home.sampling import SamplingPolicy
from reme.application import Application
@ -21,6 +22,7 @@ class RunHomeConfig(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
workspace_dir: str = Field(min_length=1)
preparation: dict
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
def project_path(value: str, project_root: Path) -> Path:
@ -30,7 +32,7 @@ def project_path(value: str, project_root: Path) -> Path:
return (project_root / value).resolve()
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
"""Resolve source paths and validate explicit Home preparation parameters."""
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
config = RunHomeConfig.model_validate(raw)
@ -40,15 +42,22 @@ def load_config(config_path: str | Path, *, workspace_dir: str | None = None, pr
spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
return workspace, spec.model_dump(mode="json")
return workspace, spec.model_dump(mode="json"), config.sampling_policy
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
"""Return resolved preparation inputs, preserving the preparation-only caller contract."""
workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
return workspace, spec
async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
"""Start Home, prepare once, and close all resources without evaluating cases."""
workspace, spec = load_config(config_path, workspace_dir=workspace_dir)
workspace, spec, policy = _load_config(config_path, workspace_dir=workspace_dir)
app = Application(
workspace_dir=str(workspace),
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
components={"home.evaluation": {"default": {"sampling_policy": policy.model_dump(mode="json")}}},
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,

View file

@ -131,6 +131,12 @@ the host application loads `home`. `source_ref: HEAD` exports committed source
only, excluding uncommitted modifications. Preparation failure rolls back its
outputs; after success, another call fails instead of resetting the experiment.
The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`,
`max_queries` (null means unlimited), `temperature`, `ewma_alpha` and
`uniform_exploration`. The runner validates these and passes them to the host
`home.evaluation` Component. It still only prepares; it does not run evaluations.
Each evaluation controls `sampling` and `record_history` separately.
Python callers can use the same lifecycle:
```python

View file

@ -120,6 +120,11 @@ python benchmark/longmemeval/run_home.py
准备失败会回滚;成功后禁止再次准备同一工作区。
Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=...)`。
顶层 `sampling_policy` 配置采样默认值:`max_cases`、`max_queries`(null 表示不限)、
`temperature`、`ewma_alpha` 和 `uniform_exploration`。入口校验后将其传给宿主
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
和 `record_history`。
默认使用 `dev_meta` 已保存的 **group_1–group_5,共 250 个 case**,
ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchmark.case_ids`,
运行时无需读取分组文件,group_6–group_10 保持为留出集。

View file

@ -1,8 +1,14 @@
# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
# Preserve dev_meta's fixed training split; groups 6-10 remain held out.
workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
sampling_policy:
max_cases: 5
max_queries: null
temperature: 1.0
ewma_alpha: 0.3
uniform_exploration: 0.2
preparation:
schema_version: 1
benchmark:
name: longmemeval
source: benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json

View file

@ -272,8 +272,11 @@ class DockerReMeSandboxFactory:
plugins: tuple[str, ...] | list[str] = (),
command_timeout: float = 1800.0,
workspace_builder: WorkspaceBuilder | None = None,
on_sandbox_started: Callable[[], None] | None = None,
) -> None:
self.candidate = candidate
self._on_sandbox_started = on_sandbox_started
self._sandbox_started_reported = False
self.env = dict(env or {})
self.config = config
self.plugins = tuple(plugins)
@ -309,6 +312,9 @@ class DockerReMeSandboxFactory:
)
try:
await case.initialize()
if self._on_sandbox_started is not None and not self._sandbox_started_reported:
self._on_sandbox_started()
self._sandbox_started_reported = True
except BaseException as exc:
await cleanup_on_error([(f"workspace {container_id!r}", workspace.close())], exc)
raise

View file

@ -315,3 +315,394 @@ processes must not write there. Recovery requires a writable, functioning
filesystem; if rollback fails, its journal remains for the next attempt and the
Job reports the error. Calls with `_allowed_paths` restrictions are rejected
because preparation owns multiple workspace output locations.
## Bounded agent foundation
`reme_home.agent.BoundedAgentWrapper` inherits ReMe's `AsAgentWrapper` and is
registered as the `home_bounded` agent-wrapper backend when Home is enabled.
It reuses ReMe's Job tools, hidden `injected_job_kwargs`, `tool_context_id`,
built-in tools, permissions, model configuration, skills, token accounting and
persisted sessions. No agent component or optimization run is enabled automatically.
Configure a component explicitly, alongside the required model and Home plugins:
```yaml
components:
agent_wrapper:
investigator:
backend: home_bounded
as_llm: default
react_config:
max_iters: 40
time_budget_minutes: 30 # Optional advisory budget; null disables it.
job_tools: [home_evaluate]
injected_job_kwargs:
case_ids: ["1"] # Hidden from the model and enforced by the host.
```
Call `reply()` or `reply_stream()` through the normal ReMe wrapper interface;
call-time options override component defaults. Each built agent owns its own
timer and request state. The internal `BoundedReActAgent` still subclasses the AS
runtime to implement loop hooks; callers use `BoundedAgentWrapper` for ReMe wiring.
The lower-level `BoundedReActAgent` and `RunTimeBudgetTimer` exports remain available.
Each reply invocation starts and stops its advisory timer, including on errors and
cancellation. Iteration reminders occur at 50%, 75%, 90% and 95%; coincident or
skipped thresholds produce one reminder. Time reminders occur at 60%, 75%, 90%,
100%, then each additional 10%, alternating temporary and retained messages. Time
budgets do not cancel model calls or tools. Completed context replacements are
counted in time reminders. Reminders use native AS hint blocks, preserving the
reply's cumulative token usage across reasoning and tool calls. Use separate
agent instances for concurrent work.
The AgentScope loop owns the iteration cap and its single finalization call.
For text reports, Home removes all tools from that call, retains the original
task, and trims oldest messages if needed to fit the context budget. The final
report is still attempted if its preceding context compression fails; ordinary
compression errors outside finalization and cancellation continue to propagate.
The final response is buffered until complete, including for streaming models; unsolicited
tool calls are discarded. If finalization fails or returns no text, the reply
explicitly identifies any retained prior model output as incomplete. Cancellation
and provider interruption keep AgentScope's interruption behavior. Native
structured-output requests retain the SDK's schema and grace-iteration behavior.
Events and final-message token usage retain AgentScope's existing semantics.
## Candidate versions and code inspection
Home registers the `home.code` Component and ten ordinary Step-backed Jobs. They
operate only on the prepared `home/code/repo`, sharing the preparation/evaluation
process lock. Enable Home and prepare the workspace before calling them.
| Job | Inputs and behavior |
| --- | --- |
| `home_code_status` | Current branch, full commit SHA, exact changed paths and pending merge state. |
| `home_list_candidate_branches` | Local branches and candidates, excluding protected `init` from the candidate count. |
| `home_create_candidate_branch` | `branch_name`: fork current HEAD and switch to a new candidate. |
| `home_switch_candidate_branch` | `branch_name`: switch to an existing local branch, including `init`. |
| `home_merge_candidate_branch` | `branch_name`: merge another candidate without committing; conflicts are aborted. |
| `home_commit_candidate` | `message`: stage and commit all visible changes within prepared `candidate_paths`. |
| `home_code_diff` | Omit inputs for tracked changes against HEAD; supply `base_revision` and `revision` to compare commits. With only `base_revision`, compare the working tree against that commit. |
| `home_list_code_tree` | Optional `revision`, literal `path`, and `max_results` (default 500, maximum 2000). |
| `home_read_code_file` | Required `path`; optional `revision`, one-based `start_line` and inclusive `end_line`. |
| `home_search_code` | Required Python regex `pattern`; optional `revision`, literal `path`, and `max_results` (maximum 30). |
Start the configured service in one terminal, then use a second terminal:
```bash
reme start config=/absolute/path/to/home.yaml
```
```bash
reme home_code_status
reme home_create_candidate_branch branch_name=retrieval-routing
# Edit files under the prepared candidate_paths, then inspect and commit.
reme home_code_diff
reme home_commit_candidate message='Improve evidence routing'
reme home_evaluate debug=true
reme home_evaluate
```
Branch names must be single path-safe names without slashes. Creation, switching
and merging require a clean checkout, including untracked files and pending
merges. `init` must continue to point to the preparation's initial commit; commits
and merges into it are rejected. Detached candidates, symlinks, submodules and
changes outside `candidate_paths` are rejected by the managed mutation tools.
Merges require an explicit later commit. No remote operations, branch deletion,
amend, reset or rebase are exposed. Empty ordinary commits fail through Git.
These guarantees apply to cooperating Home Jobs; do not grant an optimizer
unrestricted shell writes or direct access to `.git` if it must obey this boundary.
Cancellation waits for already-started Git work to finish before returning.
Code-navigation Jobs resolve `revision` (default `HEAD`) to a full SHA once per
call, read raw committed objects and never execute candidate code. Dirty working
files and Git archive attributes do not affect these reads. Pin the same SHA
across a diagnostic session through ReMe's hidden injection:
```yaml
job_tools: [home_list_code_tree, home_read_code_file, home_search_code]
injected_job_kwargs:
revision: "<full-commit-sha>"
```
The wrapper hides injected fields from the model and rejects attempts to supply
them. For comparisons, configure two agent invocations or two named Job aliases
with separately pinned revisions. `HomeCodeStep` takes a fixed `operation` in
Step configuration; repository location and operation are never model-selected.
The Python entry point is `await home_code.run(operation, **arguments)`.
Reads are limited to 300 lines and 100,000 content characters, with explicit
truncation flags. Search accepts Python multiline regex, merges adjacent match
contexts, and displays at most five lines per result block and 100,000 content
characters. Files above 5 MB and searches above 100 MB are rejected; regex
matching runs in a separate process with a ten-second timeout. Use narrower
paths or line ranges to continue inspecting. These tool-operation timeouts are
independent of the bounded agent's advisory time budget.
Evaluation also checks the standalone repository identity and protected baseline
before taking its existing clean-HEAD snapshot. Formal evaluation deduplication,
automatic ancestor/donor selection and optimization scheduling remain separate
policies; these Jobs do not enable an optimizer or change evaluation selection.
## Evaluation sampling and history
`home_evaluate` is the single evaluation Job. Set `sampling=true` for adaptive
case/query selection, or leave it false for a full baseline or an explicit
host-provided `case_ids`/`query_ids` selection. `sampling=true` conflicts with
explicit selections in scored mode. `debug=true` ignores the sampling flag,
uses debug cases and never reads, writes or registers evaluation history.
The former `home_validate` Job is removed; use `home_evaluate sampling=true`.
Selection and optimization-history recording are independent. `record_history`
defaults to `true` for compatibility: a first full optimization baseline contributes
observations even with `sampling=false`. Set `record_history=false` to score code
without feeding optimization history. This still saves the manifest, scores and
result artifacts, but never registers a history entry, increments either counter
or updates sampling observations. Rebuilding history preserves that exclusion.
Debug always skips history regardless of either flag.
Evaluation parameters have distinct names and lifetimes:
| Parameter | Where it belongs | Meaning |
| --- | --- | --- |
| `sampling` | Job input, default `false` | Enable adaptive selection for this invocation. |
| `record_history` | Job input, default `true` | Record started rounds and successful results for optimization. |
| `sampling_policy` | Component defaults or host Job input | Case/query limits and weighting settings. |
Home registers these Component defaults automatically; override the policy in
application configuration when needed:
```yaml
components:
home.evaluation:
default:
backend: home_evaluation
sampling_policy:
max_cases: 5
max_queries: null
temperature: 1.0
ewma_alpha: 0.3
uniform_exploration: 0.2
```
The Component policy is immutable. Each invocation validates its own policy,
merging a partial `sampling_policy` over Component defaults and persisting the
effective values with that run. Explicit null `max_queries` removes the query
budget. The previous Component configuration key `sampling` is now
`sampling_policy`; passing a policy object as the Job's boolean `sampling` fails.
Normal ReMe Job defaults also work: set `sampling: true` and optionally
`sampling_policy` directly under `jobs.home_evaluate`. Call-time values take
precedence over same-named Job defaults; a call-time policy object replaces the
Job-default policy object, then merges with Component defaults.
With the configured service running, use a second terminal:
```bash
reme home_evaluate sampling=false # Full baseline.
reme home_evaluate sampling=false record_history=false # Standalone full evaluation.
reme home_evaluate sampling=true # Adaptive formal evaluation.
reme home_evaluate debug=true sampling=true # Debug only; no history feedback.
```
Host Python callers can override the policy for one invocation without changing
the next invocation's defaults:
```python
result = await app.run_job(
"home_evaluate",
sampling=True,
sampling_policy={"max_cases": 3, "max_queries": 12},
)
# Score specified prepared cases without affecting the optimizer's history.
standalone = await app.run_job(
"home_evaluate",
sampling=False,
record_history=False,
case_ids=["case-a", "case-b"], # Omit to run the full prepared dataset.
)
```
`sampling_policy` and explicit case/query IDs stay out of the default agent tool
schema. An optimizer that must obey a fixed budget should receive hidden,
enforced values through the normal wrapper injection:
```yaml
job_tools: [home_evaluate]
injected_job_kwargs:
sampling: true
record_history: true
sampling_policy:
max_cases: 3
max_queries: 12
```
The wrapper hides injected parameters and rejects model attempts to provide
those keys. Use ordinary call arguments when the host should decide the mode
per invocation; use injection when that decision must remain fixed for an agent.
Case/query selection retains dev_meta b53048e4's EWMA loss, staleness, observation
uncertainty, un-rechecked error priority, normalized adaptive/uniform mixture,
weighted drawing without replacement and fair-share query budget allocation.
Policy defaults are unchanged. Positive integer case/query limits, positive
finite temperature, alpha in `(0, 1]` and uniform exploration in `[0, 1]` are
validated. Seeds, draw weights, eligibility and exact selections are saved.
An explicit construction donor uses the existing dev_meta eligibility rules and
checksum checks. A later full rebuild considers the full training set again.
### Successful evaluation history
`reme_home.evaluation.history.HomeEvaluationHistory` is the dedicated
`home.evaluation_history` Component, automatically bound by `HomeEvaluation`.
It owns successful-result registration, lookup, deduplication, cache rebuilding
and the observations used for the next draw. Steps remain invocation adapters;
they do not own cross-evaluation state.
A successful history entry requires `record_history=true` and a started, completed, scored training-set
run with a full summary and no infrastructure errors, following the existing
`Response.success` contract. Low or zero quality scores remain valid observations.
Classified candidate failures retain the existing response semantics; only their
actual numeric scores train EWMA. A run with any infrastructure error is entirely
excluded, including scores from its healthy cases. Debug, cancelled, fail-fast
and pre-start failures never enter successful history. A full summary is the durable
completion boundary: cancelling the caller during subsequent history bookkeeping
does not relabel that already-completed evaluation. Failed registration discards
only derived caches so they can rebuild from the final source run status.
```python
history = app.context.components["home.evaluation_history"]["default"]
results = history.list_results() # Typed EvaluationRecord objects, chronological.
result = history.get_result(results[-1].run_key)
history.rebuild() # Rebuild derived indexes from saved artifacts.
```
The Component owns the workspace lock for these synchronous filesystem methods;
async host code can run them in a worker thread. Its compact catalog lives at
`home/evaluations/history.json`. Each record points to its run via `run_key` and
contains commit/branch identity, selected IDs, completion time and summary metrics;
questions, answers and memory files stay in their original result directory.
The separate startup ledger advances only after the first successfully initialized
sandbox, including when a later worker or execution fails. Extra/replacement
workers do not count twice. `home/sampling/state.json` distinguishes started
`round` from `successful_round`. Only successful history advances the probability
clock or changes EWMA; failed runs cannot age cases or alter the next draw's
weights. Started plans live under
`home/sampling/plans/<branch>/<commit-prefix>/<run-id>.json`. Their `round` is the
startup ledger round; `selection_round` records the successful-history snapshot
used for an adaptive draw.
For `record_history=false`, full or explicit evaluation does not access the history
manager. Adaptive selection can still read existing observations, while leaving
them unchanged; its draw is saved locally as `manifest.selection_plan`, without
a startup-ledger round or shared sampling plan. `manifest.options.record_history`
persists the requested setting, and the response's `record_history` indicates
whether the run participates in history (always false for debug). Legacy manifests
without the setting retain the previous history behavior.
Formal runs remain serialized by `home/metadata/validation.lock`; debug bypasses
that lock. Missing caches reconstruct from started plans and successful scored
artifacts. Older sampling caches are rebuilt under the success-only rule.
`history.rebuild()` rewrites only the catalog and sampling cache. Preserve source
manifests, plans and result artifacts when rebuilding; candidate code, datasets
and memory files are never modified.
## Evaluation result inspection
Two read-only Jobs navigate saved evaluations independently of successful
optimization history. They include debug, failed, cancelled, running and standalone
(`record_history=false`) runs. Reads never register results, update sampling,
rebuild summaries or write artifacts.
`home_list_validation_results` accepts optional `branch_name` and `commit_sha`
filters plus `offset=0`, `limit=20` (1–100). Without filters it lists all runs,
newest first, with a stable path tie-breaker. It returns `row_count`, `rows` and
`next_offset`, compact identity/options, saved summary counts and scores, failure
previews and artifact paths. A malformed run is reported as a `read_error` row
without stopping the unfiltered catalog.
`commit_sha` accepts a full SHA or hexadecimal prefix. A prefix must match exactly
one of **all commits reachable from the selected local branch**, including commits
that have never been evaluated. Omitting `branch_name` with a commit filter uses
the current candidate branch. Missing or ambiguous matches fail explicitly; the
response includes the resolved branch and full SHA. A branch-only filter can also
browse historical runs from a deleted branch.
`home_inspect_validation` requires `validation_path` (absolute or relative to
`home/`) and accepts optional `case_id`, `query_id`, `offset=0` and `limit=20`:
| Selection | Returned rows |
| --- | --- |
| Run only | Case summaries, missing scores first, then low scores. |
| Run and case | Query summaries in manifest order, plus independently paged failed construction Jobs. |
| Run, case and query | One query's question, answer, rubric, score and judge/error previews. |
`query_id` requires `case_id`; both must be selected by the run manifest. Cases or
queries without published evidence are explicitly marked `missing`, with null
scores. The tool uses a saved full summary, or an existing partial summary when
the full summary is absent. Missing summaries are reported without reconstruction.
Errors are previewed to 600 characters, and selected query fields to 2,000 each;
truncated fields carry a marker. `rows_source`, JSON `pointer` values, and log and
memory paths identify the original evidence. Failed construction Jobs have their
own `row_count`, `rows`, `next_offset` under `construction.failed_jobs`; their
pagination is independent of the query rows, using the supplied offset/limit.
With the configured service running, use a second terminal:
```bash
reme home_list_validation_results branch_name=candidate commit_sha=abc123 limit=10
reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1
reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1 case_id=case-a query_id=query-a
```
Host callers use the same Jobs through `app.run_job()`. For a case-scoped agent,
inject `validation_path` and `case_id` with `injected_job_kwargs`; the wrapper hides
and enforces these values. Both Jobs reject path escapes and symlinks, and fail
closed for restricted `_allowed_paths` contexts, like the other Home Jobs.
### Memory checkpoint inspection
Two read-only Jobs investigate a case's exported memory Git repository. Locate
an evaluation and case with `home_list_validation_results` and
`home_inspect_validation` first. Neither Job changes artifacts, Git refs, the
index, evaluation history or sampling state.
- `home_inspect_memory_history(validation_path, case_id, path=null, offset=0,
limit=20)` returns `status`, `workspace_exists`, `has_git_history`, `head`,
`repository`, and newest-first checkpoint `rows` with `row_count` and
`next_offset`. Limits are 1–100. Each row includes the full commit SHA, parents,
timestamp, subject (up to 2,000 characters) and a `session_id` when the subject
follows `session: <id>`. `path` follows a literal workspace-relative file or
directory. Missing exports, absent Git history and empty repositories have
distinct statuses; corrupt repositories fail explicitly.
- `home_compare_memory(validation_path, case_id, base_ref=null,
target_ref="HEAD", target_validation_path=null, path=null)` compares two memory
commits. Within one evaluation, `base_ref` is required. For another evaluation,
supply `target_validation_path`; both sides then default to `HEAD`. The case
must be selected in both manifests. Returns resolved commits, candidate commit
identities, manifest fingerprints, added/removed/modified paths, content-tree
hashes, a unified `diff`, and `truncated`.
References accept HEAD-relative expressions and hexadecimal SHA prefixes unique
among commits reachable from that memory repository's HEAD. Resolved expressions
must also be HEAD-reachable. This is independent of candidate-code branch history.
Comparisons read committed regular files from Git objects, ignoring dirty or
untracked checkout contents. File mode changes alone are not content differences.
Binary changes are identified without printing their bytes. Each snapshot is
limited to 2,000 files, 20 MB total and 5 MB per file; diff output is limited to
100,000 characters. Narrow `path` if a limit is reached. Tree hashes describe the
selected path scope. Linked Git storage, alternates, shallow/partial histories and
symbolic links in Git metadata are rejected.
After starting the service with the Home configuration, call from another terminal:
```bash
reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1
reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 case_id=case1 base_ref=HEAD~1
reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 target_validation_path=evaluations/candidate/1234567/1 case_id=case1
```
`validation_path` is absolute or relative to `home/`; for example the relative
form is `evaluations/candidate/abcdef0/1`. For agent tools, host `injected_job_kwargs` can pin
`validation_path` and `case_id`; the wrapper hides injected fields and rejects
model overrides. Like the other Home inspection Jobs, these Jobs reject
`_allowed_paths` restrictions rather than bypassing them. Query-to-session evidence
tracing and the case study agent are separate functionality.

View file

@ -0,0 +1,6 @@
"""Agent foundations for Home's investigation and optimization workflows."""
from .bounded import BoundedReActAgent, RunTimeBudgetTimer
from .wrapper import BoundedAgentWrapper
__all__ = ["BoundedAgentWrapper", "BoundedReActAgent", "RunTimeBudgetTimer"]

View file

@ -0,0 +1,308 @@
"""AgentScope agent that turns an exhausted ReAct budget into a final answer."""
from __future__ import annotations
import inspect
import logging
import math
import time
from dataclasses import dataclass
from typing import Any, AsyncGenerator, Callable
from agentscope.agent import Agent
from agentscope.message import HintBlock, Msg, TextBlock, UserMsg
from agentscope.model import ChatResponse, FinishedReason
from agentscope.tool import ToolChoice
logger = logging.getLogger(__name__)
_REMINDER_RATIOS = (0.50, 0.75, 0.90, 0.95)
@dataclass(frozen=True)
class TimeBudgetReminder:
"""One reminder plus whether it should survive the current reasoning call."""
content: str
persistent: bool
class RunTimeBudgetTimer:
"""Track one agent run and emit advisory reminders without stopping it."""
def __init__(
self,
time_budget_minutes: float,
*,
agent_label: str,
clock: Callable[[], float] = time.monotonic,
) -> None:
if (
not isinstance(time_budget_minutes, (int, float))
or isinstance(time_budget_minutes, bool)
or not math.isfinite(time_budget_minutes)
or time_budget_minutes <= 0
):
raise ValueError("time_budget_minutes must be a positive number")
if not isinstance(agent_label, str) or not agent_label.strip():
raise ValueError("agent_label must be a non-empty string")
self.time_budget_minutes = float(time_budget_minutes)
self.agent_label = agent_label.strip()
self._clock = clock
self._started_at: float | None = None
self._last_reminded_ratio = 0.0
self._next_reminder_is_persistent = False
self.context_compression_count = 0
@property
def active(self) -> bool:
"""Return whether a run is currently being timed."""
return self._started_at is not None
def start(self) -> None:
"""Start a fresh run timer."""
self._started_at = self._clock()
self._last_reminded_ratio = 0.0
self._next_reminder_is_persistent = False
self.context_compression_count = 0
def stop(self) -> None:
"""Stop timing the current run."""
self._started_at = None
def record_context_compression(self) -> None:
"""Record one completed context compression in the current run."""
self.context_compression_count += 1
def next_reminder(self) -> TimeBudgetReminder | None:
"""Return the latest newly reached soft-budget reminder."""
if self._started_at is None:
return None
elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0)
elapsed_ratio = elapsed_minutes / self.time_budget_minutes
reached_ratio: float | None = None
if elapsed_ratio >= 1.0:
reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10
else:
for threshold in (0.90, 0.75, 0.60):
if elapsed_ratio >= threshold:
reached_ratio = threshold
break
if reached_ratio is None or reached_ratio <= self._last_reminded_ratio:
return None
self._last_reminded_ratio = reached_ratio
persistent = self._next_reminder_is_persistent
self._next_reminder_is_persistent = not persistent
elapsed = _format_minutes(elapsed_minutes)
percentage = round(reached_ratio * 100)
return TimeBudgetReminder(
content=(
f"<system-reminder>The {self.agent_label} run has used {elapsed} ({percentage}% of its "
f"{_format_minutes(self.time_budget_minutes)} soft time budget). "
f"The run has completed {self.context_compression_count} context compressions. "
"This reminder does not terminate the run. Prioritize the highest-value remaining investigation and "
"produce the required final report as soon as the evidence is sufficient.</system-reminder>"
),
persistent=persistent,
)
def _format_minutes(minutes: float) -> str:
"""Format a timer duration compactly."""
if minutes < 1:
return f"{max(0, round(minutes * 60))} seconds"
rounded = round(minutes, 1)
value = int(rounded) if rounded.is_integer() else rounded
unit = "minute" if value == 1 else "minutes"
return f"{value} {unit}"
def _remove_hint_by_identity(messages: list[Msg], target: HintBlock) -> None:
"""Remove only the temporary hint, retaining the reply message and its usage."""
for message in messages:
for index, block in enumerate(message.content):
if block is target:
del message.content[index]
return
class BoundedReActAgent(Agent):
"""Add advisory budgets and safe finalization to AgentScope's bounded loop.
AgentScope owns iteration accounting, tool execution, events and cancellation.
Its one final reasoning call after ``max_iters`` receives no tools. Native
structured-output requests retain AgentScope's schema and grace-iteration rules.
Instances, like AgentScope agents, must not serve concurrent replies.
"""
def __init__(self, *args: Any, run_time_budget_timer: RunTimeBudgetTimer | None = None, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.run_time_budget_timer = run_time_budget_timer
self._sent_iteration_reminders: set[int] = set()
self._original_task_prompt = ""
async def _reply_impl(self, inputs: Any = None, **kwargs: Any) -> AsyncGenerator[Any, None]:
"""Own the advisory timer for this invocation, including error cleanup."""
self._sent_iteration_reminders.clear()
if isinstance(inputs, Msg):
self._original_task_prompt = inputs.get_text_content() or ""
elif isinstance(inputs, list):
self._original_task_prompt = "\n\n".join(message.get_text_content() or "" for message in inputs)
elif inputs is None:
# ReMe observes inputs before reply(); retain that task for finalization.
self._original_task_prompt = next(
(
message.get_text_content() or ""
for message in reversed(self.state.context)
if message.role == "user"
),
"",
)
timer = self.run_time_budget_timer
if timer is not None:
timer.start()
try:
async for item in super()._reply_impl(inputs=inputs, **kwargs):
yield item
finally:
if timer is not None:
timer.stop()
async def compress_context(self, *args: Any, **kwargs: Any) -> None:
"""Count successful compressions; allow final reports after compression failure."""
summary_before, context_before = self.state.summary, self.state.context
try:
await super().compress_context(*args, **kwargs)
except Exception as exc:
if not self._is_finalizing():
raise
logger.warning(
"Compression before the final report failed for Home agent %s; attempting finalization: %s",
self.name,
type(exc).__name__,
)
return
if self.run_time_budget_timer is not None and (
self.state.summary is not summary_before or self.state.context is not context_before
):
self.run_time_budget_timer.record_context_compression()
async def _reasoning(self, *args: Any, **kwargs: Any) -> AsyncGenerator[Any, None]:
"""Deliver each iteration threshold once and alternate time reminders."""
timer = self.run_time_budget_timer
reminder = timer.next_reminder() if timer is not None else None
reminder_hint = None
if reminder is not None:
reminder_hint = HintBlock(source="home_run_time_guard", hint=reminder.content)
self.state.append_context(self.name, [reminder_hint])
completed = self.state.cur_iter
thresholds = {math.ceil(self.react_config.max_iters * ratio) for ratio in _REMINDER_RATIOS}
reached = thresholds - self._sent_iteration_reminders
reached = {threshold for threshold in reached if threshold <= completed}
if reached:
remaining = max(0, self.react_config.max_iters - completed)
self.state.append_context(
self.name,
[
HintBlock(
source="home_iteration_guard",
hint=(
f"<system-reminder>You have completed {completed} reasoning-acting iterations; "
f"{remaining} iteration(s) remain.</system-reminder>"
),
),
],
)
self._sent_iteration_reminders.update(reached)
try:
async for item in super()._reasoning(*args, **kwargs):
yield item
finally:
if reminder is not None and not reminder.persistent and reminder_hint is not None:
_remove_hint_by_identity(self.state.context, reminder_hint)
def _is_finalizing(self) -> bool:
"""Only the iteration cap triggers finalization; elapsed time is advisory."""
return self.state.cur_iter >= self.react_config.max_iters and self.state.reply_context.structured_schema is None
async def _call_model(self, messages: list[Msg], tools: list[dict], tool_choice: ToolChoice | None = None):
"""Use the native finalization slot; never start a second synthesis call."""
if not self._is_finalizing():
return await super()._call_model(messages=messages, tools=tools, tool_choice=tool_choice)
last_text = next(
(
message.get_text_content()
for message in reversed(self.state.context)
if message.role == "assistant" and message.name == self.name and message.get_text_content()
),
"(no previous displayable model response)",
)
messages = list(messages)
messages.append(
UserMsg(
name="home_iteration_guard",
content=(
"The iteration budget is exhausted. Return the best final report from collected evidence. "
"Do not call tools or propose more investigation. Distinguish findings from uncertainty.\n"
f"Original task: {self._original_task_prompt or '(see the conversation above)'}"
),
),
)
response_usage = None
try:
limit = int(self.model.context_size * self.context_config.trigger_ratio)
while await self.model.count_tokens(messages=messages, tools=[]) >= limit:
if len(messages) <= 2:
raise ValueError("Final instructions exceed the model context budget")
del messages[1]
response = await super()._call_model(messages=messages, tools=[], tool_choice=ToolChoice(mode="none"))
response = await self._consume_final_response(response)
response_usage = response.usage
if response.finished_reason == FinishedReason.INTERRUPTED:
return response
content = [block for block in response.content if isinstance(block, TextBlock) and block.text.strip()]
if not content:
raise ValueError("Final model response contains no displayable answer")
# Ignore unsolicited tool calls, even when accompanied by valid text.
response.content = content
return response
except Exception as exc:
logger.exception("Final report failed for Home agent %s", self.name)
return ChatResponse(
content=[
TextBlock(
text=(
f"Reached the maximum of {self.react_config.max_iters} reasoning-acting iterations. "
f"Final report generation failed ({type(exc).__name__}); "
f"the following is incomplete prior model output:\n\n{last_text}"
),
),
],
is_last=True,
usage=response_usage,
)
@staticmethod
async def _consume_final_response(response: ChatResponse | AsyncGenerator[ChatResponse, None]) -> ChatResponse:
"""Require a completed response for streaming and non-streaming models."""
if isinstance(response, ChatResponse):
return response
if not inspect.isasyncgen(response):
raise TypeError(f"Unexpected final response: {type(response)}")
completed = None
try:
async for chunk in response:
if chunk.is_last:
completed = chunk
finally:
await response.aclose()
if completed is None:
raise ValueError("Final response stream ended without a completed response")
return completed

View file

@ -0,0 +1,21 @@
"""ReMe agent wrapper with Home's bounded investigation runtime."""
from typing import Any
from reme.components.agent_wrapper import AsAgentWrapper
from .bounded import BoundedReActAgent, RunTimeBudgetTimer
class BoundedAgentWrapper(AsAgentWrapper):
"""Reuse ReMe Job tools, injected arguments, permissions and persisted sessions.
``react_config.max_iters`` configures the iteration cap. Optional
``time_budget_minutes`` adds advisory reminders; each built SDK agent owns
its own timer so requests never share budget state.
"""
def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> BoundedReActAgent:
budget = options.get("time_budget_minutes")
timer = None if budget is None else RunTimeBudgetTimer(budget, agent_label=self.name)
return BoundedReActAgent(run_time_budget_timer=timer, **kwargs)

View file

@ -0,0 +1 @@
"""Home candidate version management and immutable code inspection."""

View file

@ -0,0 +1,72 @@
"""Application-owned Git work serialized with Home preparation and snapshots."""
import asyncio
from reme.components.base_component import BaseComponent
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from . import repository, snapshot
from .models import ARGUMENTS
OPERATIONS = {
"status": repository.status,
"list_branches": lambda repo, _record: repository.branches(repo),
"create_branch": repository.create_branch,
"switch_branch": repository.switch_branch,
"merge_branch": repository.merge_branch,
"commit": repository.commit,
"diff": repository.diff,
"list_code_tree": snapshot.list_code_tree,
"read_code_file": snapshot.read_code_file,
"search_code": snapshot.search_code,
}
class HomeCode(BaseComponent):
"""Manage only the prepared Home candidate; no remotes or history rewrites."""
component_type = "home.code"
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.workspace = self.bind("default", HomeWorkspace, optional=False)
self._pending: set[asyncio.Task] = set()
async def _close(self):
if self._pending:
await asyncio.gather(*self._pending, return_exceptions=True)
async def run(self, operation: str, **kwargs) -> dict:
"""Validate before dispatch, and finish owned Git work before propagating cancellation."""
if not self.is_started:
raise RuntimeError("Start the Home application before managing candidate code")
if operation not in OPERATIONS:
raise ValueError(f"Unknown code operation: {operation}")
arguments = ARGUMENTS[operation].model_validate(kwargs).model_dump()
task = asyncio.create_task(asyncio.to_thread(self._run, operation, arguments))
self._pending.add(task)
cancelled = False
try:
while True:
try:
result = await asyncio.shield(task)
if cancelled:
raise asyncio.CancelledError
return result
except asyncio.CancelledError:
if task.done():
raise
cancelled = True
except Exception:
if cancelled:
raise asyncio.CancelledError from None
raise
finally:
self._pending.discard(task)
def _run(self, operation: str, arguments: dict) -> dict:
"""Hold the same cross-process lock used when evaluations snapshot HEAD."""
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
with _initialization_lock(self.workspace.state_path / "workspace.lock"):
repo, record = repository.validate_repository(self.workspace.home_path)
return OPERATIONS[operation](repo, record, **arguments)

View file

@ -0,0 +1,68 @@
"""Strict input contracts for candidate version and code-navigation Jobs."""
from pydantic import Field
from ..preparation.spec import SpecModel
class NoArguments(SpecModel):
"""An operation with no public inputs."""
class BranchArguments(SpecModel):
"""One existing or new local candidate branch."""
branch_name: str = Field(min_length=1, max_length=100)
class CommitArguments(SpecModel):
"""Commit the complete candidate change with an explicit message."""
message: str = Field(min_length=1, max_length=10_000, pattern=r"\S")
class DiffArguments(SpecModel):
"""Omit revisions for tracked working changes against HEAD."""
base_revision: str | None = None
revision: str | None = None
class TreeArguments(SpecModel):
"""Resolve HEAD by default; hosts can pin revision with injected_job_kwargs."""
revision: str = Field(default="HEAD", min_length=1)
path: str | None = None
max_results: int = Field(default=500, ge=1, le=2000)
class ReadArguments(SpecModel):
"""A bounded line range from a committed regular text file."""
path: str = Field(min_length=1)
revision: str = Field(default="HEAD", min_length=1)
start_line: int = Field(default=1, ge=1)
end_line: int | None = Field(default=None, ge=1)
class SearchArguments(SpecModel):
"""Python regex search with multiline matching and bounded result blocks."""
pattern: str = Field(min_length=1, max_length=500)
revision: str = Field(default="HEAD", min_length=1)
path: str | None = None
max_results: int = Field(default=30, ge=1, le=30)
ARGUMENTS = {
"status": NoArguments,
"list_branches": NoArguments,
"create_branch": BranchArguments,
"switch_branch": BranchArguments,
"merge_branch": BranchArguments,
"commit": CommitArguments,
"diff": DiffArguments,
"list_code_tree": TreeArguments,
"read_code_file": ReadArguments,
"search_code": SearchArguments,
}

View file

@ -0,0 +1,84 @@
"""Stdlib-only, subprocess-isolated Python regex matching for committed text."""
from bisect import bisect_right
from itertools import accumulate
import json
import re
import sys
def search(files: dict[str, str], pattern: str, max_results: int) -> dict:
"""Merge adjacent contexts, retaining at most five lines per displayed block."""
compiled = re.compile(pattern, re.MULTILINE)
blocks = []
occurrences = 0
for path, text in files.items():
lines = text.splitlines()
starts = [0, *accumulate(map(len, text.splitlines(keepends=True)))]
for match in compiled.finditer(text):
if match.start() == match.end():
raise ValueError("pattern must consume at least one character")
occurrences += 1
if occurrences > 100_000:
raise ValueError("Too many regex matches; narrow pattern or path")
first = bisect_right(starts, match.start())
last = bisect_right(starts, match.end() - 1)
start, end = max(1, first - 1), min(len(lines), last + 1)
if (
blocks
and blocks[-1]["path"] == path
and start <= blocks[-1]["end"] + 1
and (max(end, blocks[-1]["end"]) - blocks[-1]["start"] < 5)
):
block = blocks[-1]
block["end"] = max(end, block["end"])
block["ranges"].append((first, last))
else:
blocks.append({"path": path, "start": start, "end": end, "ranges": [(first, last)]})
matches = []
characters = 0
for block in blocks[:max_results]:
lines = files[block["path"]].splitlines()
displayed_end = min(block["end"], block["start"] + 4)
content = "\n".join(
f"{'>' if any(first <= index <= last for first, last in block['ranges']) else ' '} "
f"{index} | {lines[index - 1]}"
for index in range(block["start"], displayed_end + 1)
)
remaining = 100_000 - characters
if remaining <= 0:
break
content = content[:remaining]
characters += len(content)
matches.append(
{
"path": block["path"],
"start_line": block["start"],
"end_line": block["end"],
"displayed_end_line": displayed_end,
"occurrence_count": len(block["ranges"]),
"content": content,
"content_truncated": displayed_end < block["end"] or characters >= 100_000,
},
)
return {
"occurrence_count": occurrences,
"merged_match_count": len(blocks),
"returned_match_count": len(matches),
"matches": matches,
"truncated": len(blocks) > len(matches) or any(match["content_truncated"] for match in matches),
}
def main():
"""Emit structured errors without candidate text or traceback noise."""
arguments = json.load(sys.stdin)
try:
result = search(**arguments)
except (ValueError, re.error) as exc:
result = {"error": str(exc)}
json.dump(result, sys.stdout)
if __name__ == "__main__":
main()

View file

@ -0,0 +1,213 @@
"""Local candidate Git operations shared by Home Jobs and evaluation snapshots."""
from pathlib import Path
from ..evaluation.artifacts import contained, read_json
from ..preparation.code import git
from ..preparation.spec import relative_path
INITIAL_BRANCH = "init"
MAX_OUTPUT = 100_000
def resolve_revision(repository: Path, revision: str) -> str:
"""Resolve a revision once, without accepting command-line options."""
if not isinstance(revision, str) or not revision.strip() or revision.startswith("-"):
raise ValueError("revision must be a nonempty Git revision without a leading '-'")
return git(repository, "rev-parse", "--verify", "--end-of-options", revision + "^{commit}")
def validate_repository(home: Path, record: dict | None = None) -> tuple[Path, dict]:
"""Require the prepared standalone repository and its unchanged baseline ref."""
record = record if record is not None else read_json(contained(home, "metadata/preparation.json"))
repository = contained(home, "code/repo", directory=True)
metadata = contained(repository, ".git", directory=True)
if not metadata.is_dir() or git(repository, "rev-parse", "--show-toplevel") != str(repository):
raise ValueError("Candidate must be a standalone Git repository")
if resolve_revision(repository, "refs/heads/init") != record["code"]["initial_commit"]:
raise ValueError("The protected init branch has changed since preparation")
git(repository, "merge-base", "--is-ancestor", record["code"]["initial_commit"], "HEAD")
return repository, record
def current_branch(repository: Path) -> str:
"""Reject detached HEAD so every candidate commit remains branch-reachable."""
branch = git(repository, "branch", "--show-current")
validate_branch(repository, branch)
return branch
def validate_branch(repository: Path, branch: str) -> str:
"""Branch names also serve as single evaluation-directory names."""
if not branch or branch.startswith("-") or any(char in branch for char in "/\\\0\n\r"):
raise ValueError("branch_name must be one path-safe local branch name")
if git(repository, "check-ref-format", "--branch", branch) != branch:
raise ValueError("Use a literal local branch name")
return branch
def require_clean(repository: Path) -> None:
"""Do not carry uncommitted or untracked files across branch operations."""
if git(repository, "status", "--porcelain=v2", "--untracked-files=all", "--ignore-submodules=none"):
raise ValueError("Candidate repository must be clean before changing branches")
if (repository / ".git/MERGE_HEAD").exists():
raise ValueError("Finish the pending merge first")
def branches(repository: Path) -> dict:
"""Describe local refs and the current immutable commit."""
local = sorted(git(repository, "for-each-ref", "--format=%(refname:short)", "refs/heads").splitlines())
candidates = [branch for branch in local if branch != INITIAL_BRANCH]
return {
"current_branch": current_branch(repository),
"commit_sha": resolve_revision(repository, "HEAD"),
"local_branches": local,
"candidate_branches": candidates,
"candidate_branch_count": len(candidates),
}
def status(repository: Path, _record: dict) -> dict:
"""Report exact paths, including names containing whitespace."""
result = branches(repository)
output = git(repository, "status", "--porcelain=v1", "-z", "--untracked-files=all", strip=False)
entries = iter(output.split("\0") if output else [])
changes = []
for entry in entries:
if not entry:
continue
change = {"status": entry[:2], "path": entry[3:]}
if "R" in entry[:2] or "C" in entry[:2]:
change["original_path"] = next(entries)
changes.append(change)
result.update(changes=changes, clean=not changes, merge_pending=(repository / ".git/MERGE_HEAD").exists())
return result
def create_branch(repository: Path, record: dict, *, branch_name: str) -> dict:
"""Fork from the current candidate or baseline and switch to the new branch."""
validate_branch(repository, branch_name)
if branch_name in branches(repository)["local_branches"]:
raise ValueError("Local branch already exists")
require_clean(repository)
validate_tree(repository, record, "HEAD")
git(repository, "switch", "--no-track", "--create", branch_name)
return {"status": "created", **branches(repository)}
def switch_branch(repository: Path, record: dict, *, branch_name: str) -> dict:
"""Switch only to an existing local ref, including the baseline."""
validate_branch(repository, branch_name)
if branch_name not in branches(repository)["local_branches"]:
raise ValueError("Unknown local branch")
require_clean(repository)
validate_tree(repository, record, resolve_revision(repository, "refs/heads/" + branch_name))
git(repository, "switch", "--no-guess", branch_name)
return {"status": "switched", **branches(repository)}
def validate_tree(repository: Path, record: dict, revision: str) -> None:
"""Only declared editable roots may differ from the prepared baseline."""
roots = [Path(relative_path(path)) for path in record["code"]["candidate_paths"]]
changed = git(
repository,
"diff",
"--no-ext-diff",
"--no-textconv",
"--name-only",
"--no-renames",
"-z",
record["code"]["initial_commit"],
revision,
"--",
strip=False,
)
for name in filter(None, changed.split("\0")):
path = Path(relative_path(name))
if ".git" in path.parts or not any(path.is_relative_to(root) for root in roots):
raise ValueError(f"Change outside candidate_paths: {name}")
for entry in filter(None, git(repository, "ls-tree", "-r", "-z", revision, strip=False).split("\0")):
header, name = entry.split("\t", 1)
mode, kind, _oid = header.split()
if mode not in {"100644", "100755"} or kind != "blob" or ".git" in Path(relative_path(name)).parts:
raise ValueError(f"Candidate must contain regular files only: {name}")
def merge_branch(repository: Path, record: dict, *, branch_name: str) -> dict:
"""Stage compatible candidates; automatically abort conflicts."""
validate_branch(repository, branch_name)
state = branches(repository)
if state["current_branch"] == INITIAL_BRANCH:
raise ValueError("Cannot merge into init")
if branch_name in (INITIAL_BRANCH, state["current_branch"]):
raise ValueError("Select another candidate branch")
if branch_name not in state["candidate_branches"]:
raise ValueError("Unknown candidate branch")
require_clean(repository)
validate_tree(repository, record, "HEAD")
validate_tree(repository, record, resolve_revision(repository, "refs/heads/" + branch_name))
try:
git(
repository,
"-c",
"user.name=Home",
"-c",
"user.email=home@localhost",
"merge",
"--no-commit",
"--no-ff",
branch_name,
)
except Exception:
if (repository / ".git/MERGE_HEAD").exists():
git(repository, "merge", "--abort")
raise
merge_status = "merged_not_committed" if (repository / ".git/MERGE_HEAD").exists() else "already_up_to_date"
return {"status": merge_status, "source_branch": branch_name, **branches(repository)}
def commit(repository: Path, record: dict, *, message: str) -> dict:
"""Commit all candidate changes without rewriting existing history."""
if current_branch(repository) == INITIAL_BRANCH:
raise ValueError("Create a candidate branch before committing; init is protected")
# Validate existing staged changes before staging anything else, then the complete index.
validate_tree(repository, record, git(repository, "write-tree"))
changes = status(repository, record)["changes"]
roots = [Path(value) for value in record["code"]["candidate_paths"]]
for change in changes:
for name in (change["path"], change.get("original_path", change["path"])):
path = Path(relative_path(name))
if ".git" in path.parts or not any(path.is_relative_to(root) for root in roots):
raise ValueError(f"Change outside candidate_paths: {name}")
target = repository / path
if target.is_symlink():
raise ValueError(f"Candidate must contain regular files only: {name}")
git(repository, "add", "--all", "--", ".")
validate_tree(repository, record, git(repository, "write-tree"))
git(repository, "-c", "user.name=Home", "-c", "user.email=home@localhost", "commit", "--message", message)
return {"status": "committed", **branches(repository)}
def diff(repository: Path, _record: dict, *, base_revision: str | None = None, revision: str | None = None) -> dict:
"""Compare commits or all tracked changes against HEAD; never run external diff helpers."""
if revision is not None and base_revision is None:
raise ValueError("revision requires base_revision")
base = resolve_revision(repository, base_revision or "HEAD")
target = resolve_revision(repository, revision) if revision is not None else None
output = git(
repository,
"diff",
"--no-ext-diff",
"--no-textconv",
"--no-renames",
"--ignore-submodules=all",
base,
*([target] if target else []),
"--",
)
return {
"base_revision": base,
"revision": target,
"content": output[:MAX_OUTPUT],
"truncated": len(output) > MAX_OUTPUT,
}

View file

@ -0,0 +1,143 @@
"""Bounded navigation of immutable Git objects, independent of the checkout."""
import json
from pathlib import Path
import subprocess
import sys
from ..preparation.code import git, git_bytes
from ..preparation.spec import relative_path
from .repository import MAX_OUTPUT, resolve_revision
MAX_FILE_BYTES = 5_000_000
MAX_SEARCH_BYTES = 100_000_000
MAX_READ_LINES = 300
def code_path(path: str) -> str:
"""Accept literal repository-relative paths, never Git pathspec expressions."""
value = relative_path(path)
if ".git" in Path(value).parts or "\0" in value:
raise ValueError("Invalid code path")
return value
def tree(repository: Path, revision: str, path: str | None = None) -> list[tuple[str, str]]:
"""Read regular-file object IDs with NUL-delimited names and no archive filters."""
# ls-tree does not support pathspec magic. Filter exact names/prefixes ourselves.
entries = git(repository, "ls-tree", "-r", "-z", "--full-tree", revision, strip=False)
prefix = code_path(path) if path is not None else None
result = []
for entry in filter(None, entries.split("\0")):
header, name = entry.split("\t", 1)
if prefix is not None and name != prefix and not name.startswith(prefix + "/"):
continue
mode, kind, oid = header.split()
code_path(name)
if mode not in {"100644", "100755"} or kind != "blob":
raise ValueError(f"Code snapshot must contain regular files only: {name}")
result.append((name, oid))
return result
def blob(repository: Path, oid: str) -> bytes:
"""Check blob size before loading its bytes; never follow tracked symlinks."""
if int(git(repository, "cat-file", "-s", oid)) > MAX_FILE_BYTES:
raise ValueError("Code file exceeds 5 MB; narrow the path")
return git_bytes(repository, "cat-file", "blob", oid)
def list_code_tree(
repository: Path,
_record: dict,
*,
revision: str = "HEAD",
path: str | None = None,
max_results: int = 500,
) -> dict:
"""List tracked paths at a resolved commit."""
revision = resolve_revision(repository, revision)
entries = tree(repository, revision, path)
return {
"revision": revision,
"path": path,
"paths": [name for name, _ in entries[:max_results]],
"truncated": len(entries) > max_results,
}
def read_code_file(
repository: Path,
_record: dict,
*,
path: str,
revision: str = "HEAD",
start_line: int = 1,
end_line: int | None = None,
) -> dict:
"""Return at most 300 lines and 100,000 characters from one committed text file."""
path = code_path(path)
revision = resolve_revision(repository, revision)
entries = dict(tree(repository, revision, path))
if path not in entries:
raise ValueError(f"No tracked regular file at {path}")
data = blob(repository, entries[path])
if b"\0" in data:
raise ValueError("Code file is binary")
lines = data.decode("utf-8", errors="replace").splitlines()
requested_end = end_line if end_line is not None else start_line + MAX_READ_LINES - 1
if requested_end < start_line:
raise ValueError("end_line must be greater than or equal to start_line")
displayed_end = min(requested_end, start_line + MAX_READ_LINES - 1, len(lines))
content = "\n".join(lines[start_line - 1 : displayed_end])
if content:
content += "\n"
truncated = len(content) > MAX_OUTPUT or (bool(content) and (start_line > 1 or displayed_end < len(lines)))
content = content[:MAX_OUTPUT]
if truncated:
content += "\n<output truncated; request another line range to continue>"
return {
"revision": revision,
"path": path,
"start_line": start_line,
"end_line": displayed_end,
"total_lines": len(lines),
"truncated": truncated,
"content": content,
}
def search_code(
repository: Path,
_record: dict,
*,
pattern: str,
revision: str = "HEAD",
path: str | None = None,
max_results: int = 30,
) -> dict:
"""Search with Python multiline regex; isolate regex execution under a hard subprocess timeout."""
revision = resolve_revision(repository, revision)
files = {}
total = 0
for name, oid in tree(repository, revision, path):
size = int(git(repository, "cat-file", "-s", oid))
total += size
if total > MAX_SEARCH_BYTES:
raise ValueError("Code search exceeds 100 MB; narrow path")
data = blob(repository, oid)
if b"\0" not in data:
files[name] = data.decode("utf-8", errors="replace")
# Arbitrary Python regex can backtrack indefinitely. Run only the text matcher
# in a short-lived stdlib-only process; it never imports or executes candidate code.
result = subprocess.run(
[sys.executable, "-I", str(Path(__file__).with_name("regex_worker.py"))],
input=json.dumps({"files": files, "pattern": pattern, "max_results": max_results}).encode(),
capture_output=True,
check=True,
timeout=10,
)
payload = json.loads(result.stdout)
if "error" in payload:
raise ValueError(payload["error"])
return {"revision": revision, "pattern": pattern, "path": path, **payload}

View file

@ -0,0 +1,22 @@
"""Thin Job adapters; operations are fixed by Step configuration, never model input."""
from reme.steps.base_step import BaseStep, Ref
from .component import HomeCode
from .models import ARGUMENTS
class HomeCodeStep(BaseStep):
"""Expose one configured candidate operation using the normal ReMe Job lifecycle."""
code: HomeCode = Ref(HomeCode, "home.code")
async def execute(self):
"""Leave repository location and operation selection under host control."""
if self.context.get("_allowed_paths") is not None:
raise ValueError("Home code Jobs require unrestricted access to their managed workspace")
operation = self.kwargs["operation"]
fields = ARGUMENTS[operation].model_fields
arguments = {name: self.context.get(name) for name in fields if name in self.context}
self.context.response.answer = await self.code.run(operation, **arguments)
return self.context.response

View file

@ -4,6 +4,7 @@ import asyncio
import os
import shutil
import traceback
from contextlib import nullcontext
from deploy.benchmark_docker import BenchmarkDockerPool, CasePlan, DockerReMeSandboxFactory, SourceCandidate
@ -22,10 +23,12 @@ from .artifacts import (
answer_exception,
failure_kind,
)
from .inputs import EvaluationSpec, load_inputs, reuse_inputs, snapshot_code
from .inputs import EvaluationSpec, load_inputs, reuse_inputs, snapshot_code, replay_sampling_eligibility
from .compaction import compact_construction, compact_query
from ..judge import HomeJudge
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from ..sampling import SamplingPolicy, create_sampling_plan
from .history import HomeEvaluationHistory
async def blocking(function, *args, **kwargs):
@ -43,11 +46,15 @@ class HomeEvaluation(BaseComponent):
component_type = "home.evaluation"
def __init__(self, **kwargs):
def __init__(self, sampling_policy: dict | None = None, **kwargs):
if "sampling" in kwargs:
raise ValueError("Use sampling_policy for Component defaults; sampling is a Job boolean")
super().__init__(**kwargs)
self.workspace = self.bind("default", HomeWorkspace, optional=False)
self.judge = self.bind("default", HomeJudge, optional=False)
self.history = self.bind("default", HomeEvaluationHistory, optional=False)
self._pending: set[asyncio.Task] = set()
self.sampling_policy = SamplingPolicy.model_validate(sampling_policy if sampling_policy is not None else {})
async def _close(self):
if self._pending:
@ -58,7 +65,10 @@ class HomeEvaluation(BaseComponent):
if not self.is_started:
raise RuntimeError("Start the Home application before evaluation")
spec = EvaluationSpec.model_validate(kwargs)
task = asyncio.create_task(self._evaluate(spec))
override = spec.sampling_policy.model_dump(exclude_unset=True) if spec.sampling_policy is not None else {}
policy = SamplingPolicy.model_validate({**self.sampling_policy.model_dump(), **override})
spec = spec.model_copy(update={"sampling_policy": policy})
task = asyncio.create_task(self._serialized_evaluate(spec))
self._pending.add(task)
try:
return await asyncio.shield(task)
@ -69,11 +79,40 @@ class HomeEvaluation(BaseComponent):
finally:
self._pending.discard(task)
async def _serialized_evaluate(self, spec):
"""Match dev_meta's serialized formal validations; debug never owns sampling state."""
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock")
with lock:
return await self._evaluate(spec)
def _prepare(self, spec, environment):
home = self.workspace.home_path
sampling_state = self.history.sampling_state() if spec.sampling and not spec.debug else None
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
with _initialization_lock(self.workspace.state_path / "workspace.lock"):
record, dataset, cases = load_inputs(home, spec)
plan = None
if spec.sampling and not spec.debug:
eligibility = replay_sampling_eligibility(home, spec, cases)
plan = create_sampling_plan(
cases,
sampling_state,
spec.sampling_policy,
eligible_query_ids=eligibility,
)
selected = {case.case_id: case for case in cases}
cases = []
for key in plan["case_ids"]:
case = selected[key]
queries = {query.query_id: query for query in case.queries}
cases.append(
case.model_copy(
update={
"queries": [queries[query_id] for query_id in plan["query_ids"][key]],
},
),
)
judge_identity = self.judge.identity(dataset.dataset)
snapshot, commit, branch = snapshot_code(home, record)
fingerprints = {
@ -102,7 +141,22 @@ class HomeEvaluation(BaseComponent):
if not isinstance(rubric, list) or not rubric:
raise ValueError(f"BEAM query {query.query_id} has no scoring rubric")
evaluation_query(dataset.dataset, query)
return record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs
return record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs, plan
def _sampling_started(self, run, manifest, plan, policy=None, *, record_history=True):
"""The factory invokes this only after a real sandbox successfully initializes."""
if not record_history:
manifest["sandbox_started_at"] = utc_now().isoformat()
if plan is not None:
# Keep draw evidence local without claiming a started-history round.
manifest["selection_plan"] = plan
write_json(run / "manifest.json", manifest)
return
self.history.record_started(run, policy if policy is not None else self.sampling_policy, plan)
manifest.update(read_json(run / "manifest.json"))
def _finish_sampling(self, run):
self.history.record(run)
def _allocate(self, branch, commit, manifest):
"""Allocate ascending IDs under a collision-safe commit prefix."""
@ -130,7 +184,7 @@ class HomeEvaluation(BaseComponent):
async def _evaluate(self, spec):
environment = {**os.environ, **self.app_context.app_config.environment}
prepared = await blocking(self._prepare, spec, environment)
record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs = prepared
record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs, sampling_plan = prepared
judge = None if spec.debug else self.judge
run = None
try:
@ -199,6 +253,17 @@ class HomeEvaluation(BaseComponent):
plugins=tuple(record["code"]["plugins"]),
command_timeout=spec.timeout_seconds,
env=environment,
on_sandbox_started=(
None
if spec.debug
else lambda: self._sampling_started(
run,
manifest,
sampling_plan,
spec.sampling_policy,
record_history=spec.record_history,
)
),
)
semaphore = asyncio.Semaphore(spec.concurrency)
by_id = {case.case_id: case for case in cases}
@ -242,20 +307,34 @@ class HomeEvaluation(BaseComponent):
previous.update(cleanup_error=result.error, failure_kind="infra_error")
write_json(failure, previous)
summary = summarize(run, cases, scoring_required=not spec.debug)
manifest.update(status="completed", finished_at=utc_now().isoformat())
manifest.update(
status="completed",
success=summary["case_status_counts"]["infra_error"] == 0,
finished_at=utc_now().isoformat(),
)
write_json(run / "manifest.json", manifest)
write_json(run / "summary.json", summary, compact_leaf_containers=True)
if not spec.debug and spec.record_history:
await blocking(self._finish_sampling, run)
return {
"result_dir": str(run),
"commit_sha": commit,
"execution_mode": manifest["execution_mode"],
"record_history": spec.record_history and not spec.debug,
"status": "completed",
**{key: value for key, value in summary.items() if key != "cases"},
}
except BaseException as error:
# A full summary is the durable completion boundary. Cancellation
# during drained history bookkeeping cancels the caller, not the
# already completed evaluation or its successful history entry.
registration_failed = run is not None and manifest.get("status") == "completed"
if isinstance(error, asyncio.CancelledError) and registration_failed:
raise
if run is not None:
manifest.update(
status="cancelled" if isinstance(error, asyncio.CancelledError) else "failed",
success=False,
finished_at=utc_now().isoformat(),
)
write_json(run / "manifest.json", manifest)
@ -274,6 +353,11 @@ class HomeEvaluation(BaseComponent):
scoring_required=not spec.debug,
)
write_json(run / "partial_summary.json", partial, compact_leaf_containers=True)
if registration_failed and not spec.debug and spec.record_history:
try:
await blocking(self.history.invalidate)
except Exception as cache_error:
error.add_note(f"History cache invalidation failed: {cache_error}")
if isinstance(error, asyncio.CancelledError) or run is None:
raise
raise RuntimeError(f"Evaluation failed; artifacts: {run}: {error}") from error

View file

@ -0,0 +1,345 @@
"""Successful evaluation history and rebuildable sampling evidence owned by Home."""
from pathlib import Path
from typing import Mapping, Literal
import math
from pydantic import Field
from reme.components.base_component import BaseComponent
from ..preparation.data.models import utc_now
from ..preparation.spec import SpecModel
from ..sampling import SamplingPolicy, _empty_record, _observe, _query_key
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from .artifacts import contained, read_json, write_json
class EvaluationRecord(SpecModel):
"""A compact catalog entry backed by an immutable successful result directory."""
run_key: str
branch_name: str
commit_sha: str
validation_id: str
round: int
case_ids: list[str]
query_ids: dict[str, list[str]]
execution_mode: str
finished_at: str
case_count: int
candidate_failure_count: int
scored_query_count: int
mean_query_score: float | None
class HistoryIndex(SpecModel):
"""A disposable index; manifests, summaries and plans remain authoritative."""
schema_version: Literal[1] = 1
results: list[EvaluationRecord] = Field(default_factory=list)
def successful_result(home: Path, run: Path) -> tuple[dict, dict] | None:
"""Match the non-debug Response.success contract, including valid low scores."""
manifest = read_json(contained(home, run / "manifest.json"))
if manifest.get("options", {}).get("record_history") is False:
return None
if (
manifest.get("mode") != "scored"
or manifest.get("dataset_kind") != "training-set"
or manifest.get("status") != "completed"
or manifest.get("success") is False
or not manifest.get("sandbox_started_at")
):
return None
path = contained(home, run / "summary.json")
if not path.is_file():
return None
summary = read_json(path)
if summary.get("case_status_counts", {}).get("infra_error") != 0 or not summary.get("case_count"):
return None
return manifest, summary
def result_record(home: Path, run: Path) -> EvaluationRecord | None:
"""Exclude failed, cancelled, debug and incomplete directories from the catalog."""
result = successful_result(home, run)
if result is None:
return None
manifest, summary = result
return EvaluationRecord(
run_key=run.relative_to(home / "evaluations").as_posix(),
branch_name=manifest["branch_name"],
commit_sha=manifest["commit_sha"],
validation_id=manifest["validation_id"],
round=manifest["sampling"]["round"],
case_ids=manifest["case_ids"],
query_ids=manifest["query_ids"],
execution_mode=manifest["execution_mode"],
finished_at=manifest["finished_at"],
case_count=summary["case_count"],
candidate_failure_count=summary["case_status_counts"]["candidate_failure"],
scored_query_count=summary["scored_query_count"],
mean_query_score=summary["mean_query_score"],
)
def read_state(home: Path) -> dict:
"""Read a derived cache; started plans and scored artifacts retain its source evidence."""
path = contained(home, "sampling/state.json")
if not path.exists():
return rebuild_sampling_state(home, persist=False)
payload = read_json(path)
if not isinstance(payload, dict) or payload.get("schema_version") not in {1, 2}:
raise ValueError("Unsupported sampling state schema")
if not isinstance(payload.get("round"), int) or isinstance(payload["round"], bool) or payload["round"] < 0:
raise ValueError("Invalid sampling round")
for name in ("cases", "queries", "started_validations"):
if not isinstance(payload.get(name, {}), dict):
raise ValueError(f"Invalid sampling {name}")
payload.setdefault(name, {})
if not isinstance(payload.get("processed_validations"), list):
raise ValueError("Invalid processed validations")
if payload["schema_version"] == 1 or "successful_round" not in payload:
return rebuild_sampling_state(home, persist=False)
if (
not isinstance(payload["successful_round"], int)
or isinstance(payload["successful_round"], bool)
or (payload["successful_round"] < 0)
):
raise ValueError("Invalid successful-history round")
return payload
def start_validation(home: Path, run: Path, policy: SamplingPolicy, plan: dict | None = None) -> dict:
"""Record once after the first successfully initialized sandbox, never before."""
manifest = read_json(contained(home, run / "manifest.json"))
if (
manifest.get("mode") == "debug"
or manifest.get("dataset_kind") != "training-set"
or manifest.get("options", {}).get("record_history") is False
):
return {}
key = run.relative_to(home / "evaluations").as_posix()
state = read_state(home)
if key in state["started_validations"]:
return read_json(contained(home, manifest["sampling"]["plan_path"]))
plan = (
dict(plan)
if plan is not None
else {
"schema_version": 1,
"mode": "explicit",
"case_ids": manifest["case_ids"],
"query_ids": manifest["query_ids"],
"policy": policy.model_dump(),
}
)
# Preserve the draw's state snapshot separately from the committed start round.
if "round" in plan:
plan["selection_round"] = plan["round"]
plan.update(
round=state["round"] + 1,
validation_id=manifest["validation_id"],
validation_key=key,
sandbox_started_at=utc_now().isoformat(),
)
relative = f"sampling/plans/{key}.json"
write_json(contained(home, relative), plan)
manifest["sampling"] = {"round": plan["round"], "mode": plan["mode"], "plan_path": relative}
manifest["sandbox_started_at"] = plan["sandbox_started_at"]
write_json(run / "manifest.json", manifest)
state["round"] = plan["round"]
state["started_validations"][key] = plan["round"]
write_json(contained(home, "sampling/state.json"), state)
return plan
def update_sampling_state(home: Path, run: Path, *, _state: dict | None = None) -> None:
"""Update observations only from successful completed formal evaluations."""
result = successful_result(home, run)
if result is None:
return
manifest, summary = result
plan = read_json(contained(home, manifest["sampling"]["plan_path"]))
policy = SamplingPolicy.model_validate(plan["policy"])
state = read_state(home) if _state is None else _state
key = plan["validation_key"]
if key in state["processed_validations"]:
return
round_id = state["successful_round"] + 1
cases, queries = state["cases"], state["queries"]
summary_cases = summary.get("cases", {})
items = (
summary_cases.items()
if isinstance(summary_cases, Mapping)
else ((case.get("case_id"), case) for case in summary_cases)
)
for case_id, case_summary in items:
if not isinstance(case_id, str) or not isinstance(case_summary, Mapping):
continue
folder = contained(run, f"cases/{case_id}", directory=True)
query_path = contained(folder, "queries/result.json")
query_rows = read_json(query_path).get("queries", []) if query_path.is_file() else []
failure_path = contained(folder, "failure.json")
failure = read_json(failure_path) if failure_path.is_file() else {}
# Home summaries omit the redundant case status; reconstruct it using
# the same authoritative failure/query evidence as summarize().
status = case_summary.get("status") or (
"infra_error"
if failure.get("failure_kind") == "infra_error"
or any(query.get("infrastructure_error") for query in query_rows)
else "candidate_failure" if failure or any(query.get("error") for query in query_rows) else "completed"
)
case_record = cases.setdefault(case_id, _empty_record())
case_record["last_selected_round"] = max(case_record.get("last_selected_round") or 0, round_id)
if status == "infra_error":
case_record["infra_error_count"] += 1
continue
if status == "candidate_failure":
case_record["candidate_failure_count"] += 1
scores = []
for query in query_rows:
query_id = query.get("query_id")
if not isinstance(query_id, str):
continue
query_record = queries.setdefault(_query_key(case_id, query_id), _empty_record())
query_record["last_selected_round"] = max(query_record.get("last_selected_round") or 0, round_id)
if query.get("infrastructure_error"):
query_record["infra_error_count"] += 1
continue
if query.get("failure_kind") == "candidate_failure":
query_record["candidate_failure_count"] += 1
score = query.get("score")
if isinstance(score, (int, float)) and not isinstance(score, bool) and math.isfinite(score):
scores.append(float(score))
_observe(query_record, 1.0 - score, round_id, policy.ewma_alpha)
if scores:
_observe(case_record, 1.0 - sum(scores) / len(scores), round_id, policy.ewma_alpha)
state["successful_round"] = round_id
state["processed_validations"].append(key)
if _state is None:
write_json(contained(home, "sampling/state.json"), state)
def rebuild_sampling_state(home: Path, *, persist: bool = True) -> dict:
"""Rebuild the disposable cache from immutable started plans and scored artifacts."""
state = {
"schema_version": 2,
"round": 0,
"successful_round": 0,
"cases": {},
"queries": {},
"started_validations": {},
"processed_validations": [],
}
runs = []
for path in contained(home, "evaluations", directory=True).glob("*/*/*/manifest.json"):
manifest = read_json(contained(home, path))
if (
manifest.get("mode") == "debug"
or manifest.get("dataset_kind") != "training-set"
or manifest.get("options", {}).get("record_history") is False
or not (manifest.get("sandbox_started_at"))
):
continue
plan = read_json(contained(home, manifest["sampling"]["plan_path"]))
runs.append((plan["round"], path.parent, plan["validation_key"]))
for round_id, run, key in sorted(runs):
state["round"] = max(state["round"], round_id)
state["started_validations"][key] = round_id
update_sampling_state(home, run, _state=state)
if persist:
write_json(contained(home, "sampling/state.json"), state)
return state
class HomeEvaluationHistory(BaseComponent):
"""Own successful-result lookup, registration and sampling caches for one workspace."""
component_type = "home.evaluation_history"
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.workspace = self.bind("default", HomeWorkspace, optional=False)
def _history_lock(self):
if not self.is_started:
raise RuntimeError("Start the Home application before reading evaluation history")
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
return _initialization_lock(self.workspace.state_path / "workspace.lock")
def _run_path(self, run: Path) -> Path:
home = self.workspace.home_path
run = contained(home, run, directory=True)
relative = run.relative_to(home / "evaluations")
if len(relative.parts) != 3:
raise ValueError("Expected one evaluation result directory")
return run
def _index(self, *, rebuild: bool = False) -> HistoryIndex:
home = self.workspace.home_path
path = contained(home, "evaluations/history.json")
if path.exists() and not rebuild:
return HistoryIndex.model_validate(read_json(path))
records = []
for manifest in contained(home, "evaluations", directory=True).glob("*/*/*/manifest.json"):
record = result_record(home, contained(home, manifest.parent, directory=True))
if record is not None:
records.append(record)
return HistoryIndex(results=sorted(records, key=lambda record: (record.round, record.run_key)))
def sampling_state(self) -> dict:
"""Return owned, reconstructed-on-demand observations for the next draw."""
with self._history_lock():
state = read_state(self.workspace.home_path)
return {"round": state["successful_round"], "cases": state["cases"], "queries": state["queries"]}
def record_started(self, run: Path, policy: SamplingPolicy, plan: dict | None = None) -> dict:
"""Record a started round, independently of whether it later enters successful history."""
with self._history_lock():
return start_validation(self.workspace.home_path, self._run_path(run), policy, plan)
def record(self, run: Path) -> EvaluationRecord | None:
"""Register a successful result once; unsuccessful runs never train the sampler."""
with self._history_lock():
home = self.workspace.home_path
run = self._run_path(run)
record = result_record(home, run)
if record is None:
return None
index = self._index()
by_key = {item.run_key: item for item in index.results}
by_key[record.run_key] = record
update_sampling_state(home, run)
index = HistoryIndex(results=sorted(by_key.values(), key=lambda item: (item.round, item.run_key)))
write_json(contained(home, "evaluations/history.json"), index.model_dump(mode="json"))
return record
def list_results(self) -> list[EvaluationRecord]:
"""Return successful results in chronological order, including low-scoring candidates."""
with self._history_lock():
return self._index().results
def get_result(self, run_key: str) -> EvaluationRecord:
"""Look up a successful result by its stable workspace-relative identity."""
for result in self.list_results():
if result.run_key == run_key:
return result
raise KeyError(f"No successful evaluation: {run_key}")
def rebuild(self) -> list[EvaluationRecord]:
"""Rebuild only derived indexes and sampling observations; never rewrite source evidence."""
with self._history_lock():
home = self.workspace.home_path
index = self._index(rebuild=True)
rebuild_sampling_state(home)
write_json(contained(home, "evaluations/history.json"), index.model_dump(mode="json"))
return index.results
def invalidate(self) -> None:
"""Discard only derived caches after an unsuccessful registration; retain all source evidence."""
with self._history_lock():
for name in ("evaluations/history.json", "sampling/state.json"):
contained(self.workspace.home_path, name).unlink(missing_ok=True)

View file

@ -16,6 +16,8 @@ from ..preparation.code import git, tree_fingerprint
from ..preparation.data.models import CaseSpec, DatasetManifest, fingerprint
from .artifacts import contained, file_sha256, inspect_archive, read_json
from ..preparation.spec import SpecModel
from ..code.repository import validate_repository
from ..sampling import SamplingPolicy
class EvaluationSpec(SpecModel):
@ -25,6 +27,9 @@ class EvaluationSpec(SpecModel):
query_ids: dict[str, list[str]] | None = None
reuse_construction_from: str | None = Field(default=None, min_length=1)
debug: bool = False
sampling: bool = False
record_history: bool = True
sampling_policy: SamplingPolicy | None = None
concurrency: int = Field(default=4, ge=1)
timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False)
base_image: str = Field(default="reme-benchmark-base:agentscope-2.0.10", min_length=1)
@ -64,6 +69,8 @@ class EvaluationSpec(SpecModel):
"""Debug stops on failure by default and never reuses scored evidence."""
if self.debug and self.reuse_construction_from:
raise ValueError("Construction reuse requires training-set")
if self.sampling and not self.debug and (self.case_ids is not None or self.query_ids is not None):
raise ValueError("sampling=true selects case/query IDs automatically; omit explicit selections")
if "fail_fast" not in self.model_fields_set:
self.fail_fast = self.debug
return self
@ -119,7 +126,7 @@ def load_inputs(home: Path, spec: EvaluationSpec):
def snapshot_code(home: Path, record: dict):
"""Combine fixed runtime with clean current HEAD; reject all uncommitted changes."""
repository = contained(home, "code/repo", directory=True)
repository, _ = validate_repository(home, record)
runtime = contained(home, "code/runtime", directory=True)
if tree_fingerprint(runtime) != record["code"]["runtime_fingerprint"]:
raise ValueError("Fixed runtime has changed since preparation")
@ -235,3 +242,49 @@ def reuse_inputs(home: Path, spec: EvaluationSpec, cases: list, fingerprints: di
if errors:
raise ValueError("Invalid construction snapshots: " + "; ".join(errors))
return {"result_dir": str(source), "commit_sha": manifest["commit_sha"]}, archives
def replay_sampling_eligibility(home: Path, spec: EvaluationSpec, cases: list[CaseSpec]) -> dict | None:
"""Port dev_meta's replay eligibility without restricting future full rebuilds."""
if spec.reuse_construction_from is None:
return None
source = contained(home, spec.reuse_construction_from, directory=True)
evaluations = contained(home, "evaluations", directory=True)
if not source.is_relative_to(evaluations) or source == evaluations:
raise ValueError("reuse_construction_from must be inside home/evaluations")
manifest = read_json(contained(source, "manifest.json"))
summary = read_json(contained(source, "summary.json"))
if (
manifest.get("dataset_kind") != "training-set"
or manifest.get("mode") == "debug"
or (manifest.get("status", summary.get("status")) != "completed")
):
raise ValueError("Replay sampling requires a completed scored training-set run")
case_ids = manifest.get("case_ids")
if not isinstance(case_ids, list) or not case_ids or len(set(case_ids)) != len(case_ids):
raise ValueError("Replay source must select unique nonempty cases")
reusable = set()
for case_id in case_ids:
entity_id(case_id)
folder = contained(source, f"cases/{case_id}", directory=True)
result_path = contained(folder, "memory_construction/result.json")
archive = contained(folder, "memory_construction/reme_workspace.tar.gz")
if not result_path.is_file() or not archive.is_file() or read_json(result_path).get("success") is not True:
continue
reusable.add(case_id)
if (len(case_ids) - len(reusable)) / len(case_ids) > 0.05:
raise ValueError("Replay source construction failure ratio exceeds 5%")
for case_id in reusable:
payload = read_json(contained(source, f"cases/{case_id}/queries/result.json"))
queries = payload.get("queries")
if not queries or not all(
isinstance(query, dict)
and query.get("error") is None
and isinstance(query.get("answer"), str)
and query["answer"].strip()
for query in queries
):
raise ValueError(f"Replay source has no completed successful answers for {case_id}")
# A construction snapshot can answer new query subsets from the same case;
# dev_meta intentionally considers every installed query of a reusable case.
return {case.case_id: [query.query_id for query in case.queries] for case in cases if case.case_id in reusable}

View file

@ -0,0 +1,358 @@
"""Read-only evaluation navigation Jobs, independent of optimization history."""
import asyncio
import json
from pydantic import Field, field_validator, model_validator
from reme.steps.base_step import BaseStep, Ref
from ..code.repository import current_branch, resolve_revision, validate_branch, validate_repository
from ..preparation.code import git
from ..preparation.spec import SpecModel
from ..workspace import HomeWorkspace, _initialization_lock
from .artifacts import contained, read_json
from .inputs import entity_id
TOTAL_FIELDS = (
"schema_version",
"scoring_policy",
"case_count",
"case_status_counts",
"construction_status_counts",
"failure_stage_case_counts",
"expected_query_count",
"scored_query_count",
"query_status_counts",
"mean_query_score",
"observed_mean_query_score",
)
class ListValidationResultsInput(SpecModel):
"""Filters affect catalog navigation, never which evaluations enter history."""
branch_name: str | None = None
commit_sha: str | None = Field(default=None, pattern=r"^[0-9a-fA-F]+$")
offset: int = Field(default=0, ge=0)
limit: int = Field(default=20, ge=1, le=100)
@field_validator("branch_name")
@classmethod
def validate_branch_name(cls, value):
"""Keep historical branch filters within one directory segment."""
return entity_id(value) if value is not None else None
class InspectValidationInput(SpecModel):
"""A query belongs to a case and every inspection belongs to one run."""
validation_path: str = Field(min_length=1)
case_id: str | None = None
query_id: str | None = None
offset: int = Field(default=0, ge=0)
limit: int = Field(default=20, ge=1, le=100)
@field_validator("case_id", "query_id")
@classmethod
def validate_entity(cls, value):
"""Reject path traversal before resolving any artifact."""
return entity_id(value) if value is not None else None
@model_validator(mode="after")
def validate_selection(self):
"""Require a case when selecting one query."""
if self.query_id is not None and self.case_id is None:
raise ValueError("query_id requires case_id")
return self
def _preview(value, limit=600):
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
return value if len(text) <= limit else text[:limit] + "… [truncated; read source field]"
def _pointer(value):
return str(value).replace("~", "~0").replace("/", "~1")
def _page(rows, spec):
return {
"row_count": len(rows),
"rows": rows[spec.offset : spec.offset + spec.limit],
"next_offset": spec.offset + spec.limit if spec.offset + spec.limit < len(rows) else None,
}
def _object(root, relative, *, required=False):
path = contained(root, relative)
if not path.exists() and not required:
return None
payload = read_json(path)
if not isinstance(payload, dict):
raise ValueError(f"Expected JSON object: {path}")
return payload
def _summary(run):
for name in ("summary.json", "partial_summary.json"):
value = _object(run, name)
if value is not None:
return value, str(run / name)
return None, None
def _run_path(home, value):
run = contained(home, value, directory=True)
relative = run.relative_to(home / "evaluations")
if len(relative.parts) != 3 or not run.is_dir():
raise ValueError("validation_path must identify one directory inside home/evaluations")
return run
def _commit_filter(home, spec):
if spec.commit_sha is None:
return spec.branch_name, None
# Resolve a local branch first, then match ALL of its reachable commits.
# Git's global abbreviation rules could otherwise reject a branch-unique prefix.
with _initialization_lock(contained(home, "metadata/workspace.lock")):
repo, _ = validate_repository(home)
branch = spec.branch_name if spec.branch_name is not None else current_branch(repo)
validate_branch(repo, branch)
tip = resolve_revision(repo, "refs/heads/" + branch)
prefix = spec.commit_sha.lower()
matches = [sha for sha in git(repo, "rev-list", tip, "--").splitlines() if sha.startswith(prefix)]
if len(matches) != 1:
raise ValueError(f"commit_sha prefix {prefix!r} has {len(matches)} matches on branch {branch!r}; require one")
return branch, matches[0]
def _catalog_row(run, manifest):
summary, summary_path = _summary(run)
options = manifest.get("options", {})
row = {
"validation_path": str(run),
"manifest_path": str(run / "manifest.json"),
**{
key: manifest.get(key)
for key in (
"branch_name",
"commit_sha",
"validation_id",
"status",
"success",
"started_at",
"finished_at",
"dataset",
"dataset_kind",
"mode",
"execution_mode",
"construction_source",
"fingerprints",
)
},
"sampling": options.get("sampling", False),
"record_history": options.get("record_history", True) and manifest.get("mode") != "debug",
"selected_case_count": len(manifest.get("case_ids", [])),
"selected_query_count": sum(len(ids) for ids in manifest.get("query_ids", {}).values()),
"summary_path": summary_path,
"summary_missing": summary is None,
"totals": {key: summary[key] for key in TOTAL_FIELDS if key in summary} if summary else {},
}
failure = _object(run, "failure.json")
row["failure"] = _preview(failure)
row["failure_path"] = str(run / "failure.json") if failure is not None else None
return row
def _list_results(home, spec):
branch, commit = _commit_filter(home, spec)
evaluations = contained(home, "evaluations", directory=True)
rows = []
for path in evaluations.glob("*/*/*/manifest.json"):
if branch is not None and path.parent.parent.parent.name != branch:
continue
row = {"validation_path": str(path.parent), "manifest_path": str(path)}
try:
run = _run_path(home, path.parent)
manifest = _object(run, "manifest.json", required=True)
row.update({key: manifest.get(key) for key in ("branch_name", "commit_sha", "started_at")})
if commit is not None and row["commit_sha"] != commit:
continue
row = _catalog_row(run, manifest)
except (ValueError, OSError, TypeError, KeyError, AttributeError) as error:
row["read_error"] = _preview(str(error))
if commit is None or row.get("commit_sha") == commit:
rows.append(row)
rows.sort(key=lambda row: (str(row.get("started_at") or ""), row["validation_path"]), reverse=True)
return {"branch_name": branch, "commit_sha": commit, **_page(rows, spec)}
def _case_rows(manifest, summary):
cases = (summary or {}).get("cases", {})
indexed = (
{key: (value, f"/cases/{_pointer(key)}") for key, value in cases.items()}
if isinstance(cases, dict)
else {value["case_id"]: (value, f"/cases/{index}") for index, value in enumerate(cases)}
)
rows = []
for case_id in manifest.get("case_ids", []):
case, pointer = indexed.get(case_id, ({}, None))
rows.append(
{
"case_id": case_id,
"mean_score": case.get("mean_score"),
"construction": case.get("construction"),
"construction_error_count": case.get("construction_error_count", 0),
"query_error_count": case.get("query_error_count", 0),
"query_tokens": case.get("average_query_total_token_usage"),
"missing": pointer is None,
"pointer": pointer,
},
)
rows.sort(
key=lambda row: (
row["mean_score"] is not None,
row["mean_score"] if row["mean_score"] is not None else -1,
-row["construction_error_count"],
row["case_id"],
),
)
return rows
def _case_inspection(run, manifest, spec):
folder = contained(run, f"cases/{spec.case_id}", directory=True)
build_path, query_path = "memory_construction/result.json", "queries/result.json"
build, payload = _object(folder, build_path), _object(folder, query_path)
failures = []
jobs = (build or {}).get("jobs", [])
for index, job in enumerate(jobs):
outcome = job.get("result") or {}
if outcome.get("success") is False:
failures.append(
{
"job": job.get("job"),
"error": _preview(outcome.get("error")),
"pointer": f"/jobs/{index}/result",
},
)
queries = (payload or {}).get("queries", [])
by_id = {query["query_id"]: (query, index) for index, query in enumerate(queries)}
selected = manifest["query_ids"][spec.case_id]
if spec.query_id is not None and spec.query_id not in selected:
raise ValueError("query_id is not selected by this evaluation manifest")
rows = []
for query_id in ([spec.query_id] if spec.query_id is not None else selected):
query, index = by_id.get(query_id, ({}, None))
row = {
"query_id": query_id,
"score": query.get("score"),
"question_type": query.get("question_type"),
"failure_kind": query.get("failure_kind"),
"error": _preview(query.get("error")),
"answer_success": (query.get("answer_result") or {}).get("success"),
"judge_success": (query.get("judge_result") or {}).get("success"),
"missing": index is None,
"pointer": f"/queries/{index}" if index is not None else None,
}
if spec.query_id is not None:
row["fields"] = {
key: _preview(query[key], 2000)
for key in (
"question",
"golden_answer",
"answer",
"score",
"question_type",
"answer_result",
"judge_result",
"error",
"exception",
"failure_kind",
"infrastructure_error",
)
if key in query
}
rows.append(row)
result = {
"case_id": spec.case_id,
"missing_files": [name for name, value in ((build_path, build), (query_path, payload)) if value is None],
"case_failure": _preview(_object(folder, "failure.json")),
"construction": {
"source": str(folder / build_path),
"success": (build or {}).get("success"),
"status": (build or {}).get("status"),
"job_count": len(jobs),
"failed_jobs": _page(failures, spec),
},
"build_log": str(contained(folder, "memory_construction/build.log")),
"memory_workspace": str(contained(folder, "memory_construction/reme_workspace", directory=True)),
"rows_source": str(folder / query_path),
**_page(rows, spec),
}
if spec.query_id is not None:
query_folder = contained(folder, f"queries/{spec.query_id}", directory=True)
result.update(
query_missing=rows[0]["missing"],
answer_log=str(contained(query_folder, "answer.log")),
query_failure=_preview(_object(query_folder, "failure.json")),
)
return result
def _inspect_result(home, spec):
run = _run_path(home, spec.validation_path)
manifest = _object(run, "manifest.json", required=True)
if spec.case_id is not None and spec.case_id not in manifest.get("case_ids", []):
raise ValueError("case_id is not selected by this evaluation manifest")
result = {"validation_path": str(run), "manifest_path": str(run / "manifest.json")}
if spec.case_id is not None:
return {**result, **_case_inspection(run, manifest, spec)}
summary, summary_path = _summary(run)
return {
**_catalog_row(run, manifest),
"rows_source": summary_path,
**_page(_case_rows(manifest, summary), spec),
}
async def _execute(step, model, function):
if step.context.get("_allowed_paths") is not None:
raise ValueError("Home inspection Jobs require unrestricted access to their managed evaluation artifacts")
spec = model.model_validate({name: step.context.get(name) for name in model.model_fields if name in step.context})
if not step.workspace.is_started:
raise RuntimeError("Start the Home workspace before inspecting evaluations")
task = asyncio.create_task(asyncio.to_thread(function, step.workspace.home_path, spec))
cancelled = False
while True:
try:
result = await asyncio.shield(task)
if cancelled:
raise asyncio.CancelledError
step.context.response.answer = result
return step.context.response
except asyncio.CancelledError:
if task.done():
raise
cancelled = True
class HomeListValidationResultsStep(BaseStep):
"""List all evaluation artifacts, including failures, debug and standalone runs."""
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
async def execute(self):
"""Navigate runs without reading or rebuilding optimization history."""
return await _execute(self, ListValidationResultsInput, _list_results)
class HomeInspectValidationStep(BaseStep):
"""Inspect a run, a case or one query with bounded result previews."""
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
async def execute(self):
"""Read only the requested result scope and return artifact coordinates."""
return await _execute(self, InspectValidationInput, _inspect_result)

View file

@ -0,0 +1,284 @@
"""Read-only navigation and comparison of exported memory Git checkpoints."""
import difflib
import hashlib
import re
import subprocess
from pydantic import Field, field_validator, model_validator
from reme.steps.base_step import BaseStep, Ref
from ..code.repository import MAX_OUTPUT, resolve_revision
from ..code.snapshot import blob, code_path, tree
from ..preparation.code import git as safe_git
from ..preparation.spec import SpecModel
from ..workspace import HomeWorkspace
from .inputs import entity_id
from .artifacts import contained
from .inspection_step import _execute, _object, _page, _run_path
MAX_TREE_FILES = 2000
MAX_TREE_BYTES = 20_000_000
def git(repository, *arguments):
"""Ignore replace refs while reusing the shared noninteractive Git executor."""
return safe_git(repository, "--no-replace-objects", *arguments)
class MemoryScopeInput(SpecModel):
"""An existing evaluation and one manifest-selected case."""
validation_path: str = Field(min_length=1)
case_id: str = Field(min_length=1)
path: str | None = None
@field_validator("case_id")
@classmethod
def validate_case(cls, value):
"""Accept a single case directory segment."""
return entity_id(value)
@field_validator("path")
@classmethod
def validate_path(cls, value):
"""Accept literal repository-relative paths only."""
return code_path(value) if value is not None else None
class MemoryHistoryInput(MemoryScopeInput):
"""Page through checkpoints, newest first."""
offset: int = Field(default=0, ge=0)
limit: int = Field(default=20, ge=1, le=100)
class CompareMemoryInput(MemoryScopeInput):
"""Compare two commits in one or two exported repositories."""
base_ref: str | None = Field(default=None, min_length=1)
target_ref: str = Field(default="HEAD", min_length=1)
target_validation_path: str | None = Field(default=None, min_length=1)
@model_validator(mode="after")
def validate_base(self):
"""Require an explicit earlier checkpoint for a same-repository comparison."""
if self.target_validation_path is None and self.base_ref is None:
raise ValueError("base_ref is required when target_validation_path is omitted")
return self
def _repository(home, validation_path, case_id):
run = _run_path(home, validation_path)
manifest = _object(run, "manifest.json", required=True)
if case_id not in manifest.get("case_ids", []):
raise ValueError("case_id is not selected by this evaluation manifest")
repository = contained(run, f"cases/{case_id}/memory_construction/reme_workspace", directory=True)
return run, manifest, repository
def _head(repository):
"""Reject indirect Git storage and distinguish absent/empty/corrupt repositories."""
if not repository.exists():
return "workspace_missing", None
metadata = contained(repository, ".git")
if not metadata.exists():
return "history_missing", None
if not metadata.is_dir():
raise ValueError("Memory history requires a standalone .git directory")
# Exported artifacts must not redirect reads to another repository or host path.
if any(path.is_symlink() for path in metadata.rglob("*")):
raise ValueError("Memory Git metadata must not contain symbolic links")
for name in ("commondir", "objects/info/alternates", "objects/info/http-alternates", "shallow"):
if (metadata / name).exists():
raise ValueError(f"Unsupported indirect or incomplete memory history: {name}")
if any(metadata.glob("objects/pack/*.promisor")):
raise ValueError("Partial memory repositories are unsupported")
if git(repository, "for-each-ref", "--format=%(refname)", "refs/replace"):
raise ValueError("Memory replace refs are unsupported")
configuration = git(repository, "config", "--local", "--list")
if any(line.lower().startswith(("include.", "includeif.")) for line in configuration.splitlines()):
raise ValueError("Memory Git configuration must not include external files")
if any(
"promisor=" in line.lower() or line.lower().startswith("extensions.partialclone=")
for line in configuration.splitlines()
):
raise ValueError("Partial memory repositories are unsupported")
if git(repository, "rev-parse", "--show-toplevel") != str(repository):
raise ValueError("Memory history must be a standalone repository")
if git(repository, "rev-parse", "--is-shallow-repository") != "false":
raise ValueError("Memory history must not be shallow")
try:
return "available", resolve_revision(repository, "HEAD")
except subprocess.CalledProcessError:
# Only an unborn symbolic HEAD is an empty repository, not a broken ref.
reference = git(repository, "symbolic-ref", "-q", "HEAD")
if git(repository, "for-each-ref", "--format=%(refname)", reference):
raise ValueError("Memory HEAD cannot be resolved") from None
if not (metadata / "objects").is_dir():
raise ValueError("Memory object database is missing") from None
return "history_empty", None
def _resolve(repository, reference, head):
if re.fullmatch(r"[0-9a-fA-F]+", reference or ""):
matches = [
sha for sha in git(repository, "rev-list", head, "--").splitlines() if sha.startswith(reference.lower())
]
if len(matches) != 1:
raise ValueError(f"Memory commit prefix has {len(matches)} matches reachable from HEAD; require one")
return matches[0]
revision = resolve_revision(repository, reference)
git(repository, "merge-base", "--is-ancestor", revision, head)
return revision
def inspect_history(home, spec):
"""Return saved Git history without refreshing the worktree or index."""
run, _manifest, repository = _repository(home, spec.validation_path, spec.case_id)
status, head = _head(repository)
result = {
"validation_path": str(run),
"case_id": spec.case_id,
"repository": str(repository),
"workspace_exists": repository.exists(),
"has_git_history": head is not None,
"status": status,
"head": head,
"path": spec.path,
}
if head is None:
return {**result, **_page([], spec)}
# Literal pathspecs prevent user-controlled wildcard/pathspec expansion.
path_args = ["--", ":(literal)" + spec.path] if spec.path is not None else ["--"]
count = int(git(repository, "rev-list", "--count", head, *path_args))
raw = git(
repository,
"log",
"--no-show-signature",
f"--skip={spec.offset}",
f"--max-count={spec.limit}",
"--format=%H%x00%P%x00%aI%x00%s",
head,
*path_args,
)
rows = []
for line in raw.splitlines():
sha, parents, timestamp, subject = line.split("\0", 3)
rows.append(
{
"commit_sha": sha,
"parents": parents.split(),
"timestamp": timestamp,
"subject": subject[:2000],
"subject_truncated": len(subject) > 2000,
"session_id": subject[9:2009] if subject.startswith("session: ") else None,
},
)
return {
**result,
"row_count": count,
"rows": rows,
"next_offset": spec.offset + len(rows) if spec.offset + len(rows) < count else None,
}
def _files(repository, revision, path):
entries = tree(repository, revision, path)
if len(entries) > MAX_TREE_FILES:
raise ValueError("Memory comparison exceeds 2000 files; narrow path")
result = {}
total = 0
for name, oid in entries:
total += int(git(repository, "cat-file", "-s", oid))
if total > MAX_TREE_BYTES:
raise ValueError("Memory comparison exceeds 20 MB per snapshot; narrow path")
result[name] = blob(repository, oid)
return result
def _hash(files):
digest = hashlib.sha256()
for name, data in sorted(files.items()):
digest.update(name.encode() + b"\0" + hashlib.sha256(data).digest())
return digest.hexdigest()
def compare_memory(home, spec):
"""Compare committed regular-file contents, independent of dirty checkout files."""
base_run, base_manifest, base_repo = _repository(home, spec.validation_path, spec.case_id)
target_run, target_manifest, target_repo = _repository(
home,
spec.target_validation_path or spec.validation_path,
spec.case_id,
)
base_status, base_head = _head(base_repo)
target_status, target_head = _head(target_repo)
if base_head is None or target_head is None:
raise ValueError(f"Comparison requires memory commits on both sides: {base_status}, {target_status}")
base_sha = _resolve(base_repo, spec.base_ref or "HEAD", base_head)
target_sha = _resolve(target_repo, spec.target_ref, target_head)
before = _files(base_repo, base_sha, spec.path)
after = _files(target_repo, target_sha, spec.path)
added = sorted(after.keys() - before.keys())
removed = sorted(before.keys() - after.keys())
modified = sorted(name for name in before.keys() & after.keys() if before[name] != after[name])
parts = []
length = 0
truncated = False
for name in sorted(added + removed + modified):
left, right = before.get(name, b""), after.get(name, b"")
try:
if b"\0" in left or b"\0" in right:
raise UnicodeDecodeError("utf-8", b"\0", 0, 1, "binary")
lines = difflib.unified_diff(
left.decode("utf-8").splitlines(keepends=True),
right.decode("utf-8").splitlines(keepends=True),
fromfile="base/" + name,
tofile="target/" + name,
)
except UnicodeDecodeError:
lines = iter([f"Binary files differ: {name}\n"])
for line in lines:
if not line.endswith("\n"):
line += "\n\\ No newline at end of file\n"
available = MAX_OUTPUT - length
parts.append(line[:available])
length += min(len(line), available)
if len(line) > available:
truncated = True
break
if truncated:
break
return {
"base_validation_path": str(base_run),
"target_validation_path": str(target_run),
"case_id": spec.case_id,
"path": spec.path,
"base_commit": base_sha,
"target_commit": target_sha,
"base_candidate_commit": base_manifest.get("commit_sha"),
"target_candidate_commit": target_manifest.get("commit_sha"),
"base_fingerprints": base_manifest.get("fingerprints"),
"target_fingerprints": target_manifest.get("fingerprints"),
"added": added,
"removed": removed,
"modified": modified,
"base_tree_sha256": _hash(before),
"target_tree_sha256": _hash(after),
"diff": "".join(parts),
"truncated": truncated,
}
class HomeMemoryInspectionStep(BaseStep):
"""Two read-only Jobs sharing one invocation-scoped Step implementation."""
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
async def execute(self):
"""Select the operation from host configuration, never model input."""
operations = {"history": (MemoryHistoryInput, inspect_history), "compare": (CompareMemoryInput, compare_memory)}
model, function = operations[self.kwargs["operation"]]
return await _execute(self, model, function)

View file

@ -7,14 +7,14 @@ from .inputs import EvaluationSpec
class HomeEvaluateStep(BaseStep):
"""Case IDs come from runtime inputs; no selection means all prepared cases."""
"""One evaluation Job with optional host-configured adaptive sampling."""
evaluation: HomeEvaluation = Ref(HomeEvaluation, "home.evaluation")
async def execute(self):
"""Evaluate via the Docker pool and persist host-side scores and evidence."""
if self.context.get("_allowed_paths") is not None:
raise ValueError("home_evaluate requires unrestricted access to its workspace output directories")
raise ValueError("Home evaluation Jobs require unrestricted access to their workspace output directories")
arguments = {name: self.context.get(name) for name in EvaluationSpec.model_fields if name in self.context}
# Do not silently accept a removed toggle that conflicts with reuse paths.
if "eval_only" in self.context:

View file

@ -1,20 +1,39 @@
backends:
home_memory_inspection_step: reme_home.evaluation.memory_inspection_step:HomeMemoryInspectionStep
home_code: reme_home.code.component:HomeCode
home_code_step: reme_home.code.step:HomeCodeStep
home_bounded: reme_home.agent:BoundedAgentWrapper
home_preparation: reme_home.preparation.component:HomePreparation
home_prepare_step: reme_home.preparation.component:HomePrepareStep
home_workspace: reme_home.workspace:HomeWorkspace
home_judge: reme_home.judge:HomeJudge
home_evaluation: reme_home.evaluation.component:HomeEvaluation
home_evaluation_history: reme_home.evaluation.history:HomeEvaluationHistory
home_evaluate_step: reme_home.evaluation.step:HomeEvaluateStep
home_list_validation_results_step: reme_home.evaluation.inspection_step:HomeListValidationResultsStep
home_inspect_validation_step: reme_home.evaluation.inspection_step:HomeInspectValidationStep
auto_bash_step: reme_home.auto_bash:AutoBashStep
application_defaults:
components:
home.code:
default:
backend: home_code
home.judge:
default:
backend: home_judge
home.evaluation_history:
default:
backend: home_evaluation_history
home.evaluation:
default:
backend: home_evaluation
sampling_policy:
max_cases: 5
max_queries: null
temperature: 1.0
ewma_alpha: 0.3
uniform_exploration: 0.2
home.preparation:
default:
backend: home_preparation
@ -22,12 +41,105 @@ application_defaults:
default:
backend: home_workspace
jobs:
home_evaluate:
home_inspect_memory_history:
backend: base
description: "Evaluate committed candidate code on prepared cases; optionally restore prior construction tar files"
description: "List a case's exported memory Git checkpoints, newest first; use home_inspect_validation to locate the case"
parameters:
type: object
additionalProperties: false
required: [validation_path, case_id]
properties:
validation_path: {type: string, minLength: 1}
case_id: {type: string, minLength: 1}
path:
type: [string, "null"]
description: "Literal memory-workspace-relative file or directory to follow"
offset: {type: integer, minimum: 0, default: 0}
limit: {type: integer, minimum: 1, maximum: 100, default: 20}
steps:
- backend: home_memory_inspection_step
operation: history
home_compare_memory:
backend: base
description: "Compare a case's memory checkpoints or committed final memory across evaluations; narrow path when output is truncated"
parameters:
type: object
additionalProperties: false
required: [validation_path, case_id]
properties:
validation_path: {type: string, minLength: 1}
case_id: {type: string, minLength: 1}
base_ref:
type: [string, "null"]
description: "Required within one evaluation; defaults to HEAD for cross-evaluation comparisons"
target_ref:
type: string
default: HEAD
description: "Target memory commit or revision; hexadecimal prefixes must be unique among HEAD-reachable commits"
target_validation_path:
type: [string, "null"]
description: "Optional other evaluation containing the same case"
path:
type: [string, "null"]
description: "Literal memory-workspace-relative file or directory"
steps:
- backend: home_memory_inspection_step
operation: compare
home_list_validation_results:
backend: base
description: "List evaluation runs including debug, failed and standalone runs; inspect a returned validation_path with home_inspect_validation"
parameters:
type: object
additionalProperties: false
properties:
branch_name:
type: [string, "null"]
description: "Filter by branch; with commit_sha defaults to the current candidate branch"
commit_sha:
type: [string, "null"]
pattern: "^[0-9a-fA-F]+$"
description: "Full commit SHA or hexadecimal prefix matching exactly one reachable commit on the selected branch"
offset: {type: integer, minimum: 0, default: 0}
limit: {type: integer, minimum: 1, maximum: 100, default: 20}
steps:
- backend: home_list_validation_results_step
home_inspect_validation:
backend: base
description: "Read an evaluation's case summaries, a case's query summaries, or one query's answer/judge previews; follow next_offset and source paths for details"
parameters:
type: object
additionalProperties: false
required: [validation_path]
properties:
validation_path:
type: string
minLength: 1
description: "Result directory returned by home_evaluate or home_list_validation_results; absolute or relative to home/"
case_id:
type: [string, "null"]
description: "Selected case to inspect; omit to list cases"
query_id:
type: [string, "null"]
description: "Selected query to inspect in detail; requires case_id"
offset: {type: integer, minimum: 0, default: 0}
limit: {type: integer, minimum: 1, maximum: 100, default: 20}
steps:
- backend: home_inspect_validation_step
home_evaluate:
backend: base
description: "Evaluate committed code; optionally sample cases/queries or restore prior construction tar files"
parameters:
type: object
additionalProperties: false
properties:
sampling:
type: boolean
default: false
description: "Adaptively select case/query IDs using the host sampling_policy; debug ignores this flag"
record_history:
type: boolean
default: true
description: "Register successful scored results and started rounds for optimization; false saves results without history or sampling updates; debug always skips history"
# case_ids/query_ids are internal runtime inputs, supplied by the host
# through injected_job_kwargs or run_job(), never model-selected tools.
reuse_construction_from:
@ -101,3 +213,191 @@ application_defaults:
- command
steps:
- backend: auto_bash_step
home_code_status:
backend: base
description: Inspect candidate branch, commit and uncommitted files
parameters:
additionalProperties: false
properties: {}
type: object
steps:
- backend: home_code_step
operation: status
home_list_candidate_branches:
backend: base
description: List the protected init branch and local candidate branches
parameters:
additionalProperties: false
properties: {}
type: object
steps:
- backend: home_code_step
operation: list_branches
home_create_candidate_branch:
backend: base
description: Create and switch to a new candidate branch; requires clean code
parameters:
additionalProperties: false
properties:
branch_name:
maxLength: 100
minLength: 1
type: string
required:
- branch_name
type: object
steps:
- backend: home_code_step
operation: create_branch
home_switch_candidate_branch:
backend: base
description: Switch to an existing local branch; requires clean code
parameters:
additionalProperties: false
properties:
branch_name:
maxLength: 100
minLength: 1
type: string
required:
- branch_name
type: object
steps:
- backend: home_code_step
operation: switch_branch
home_merge_candidate_branch:
backend: base
description: Stage another candidate branch without committing; conflicts are aborted
parameters:
additionalProperties: false
properties:
branch_name:
maxLength: 100
minLength: 1
type: string
required:
- branch_name
type: object
steps:
- backend: home_code_step
operation: merge_branch
home_commit_candidate:
backend: base
description: Commit all changes within candidate_paths; init and existing history are protected
parameters:
additionalProperties: false
properties:
message:
maxLength: 10000
minLength: 1
pattern: \S
type: string
required:
- message
type: object
steps:
- backend: home_code_step
operation: commit
home_code_diff:
backend: base
description: Inspect tracked working changes against HEAD or compare immutable commits
parameters:
additionalProperties: false
properties:
base_revision:
anyOf:
- type: string
- type: 'null'
default: null
revision:
anyOf:
- type: string
- type: 'null'
default: null
type: object
steps:
- backend: home_code_step
operation: diff
home_list_code_tree:
backend: base
description: List tracked regular files at a committed revision
parameters:
additionalProperties: false
properties:
revision:
default: HEAD
minLength: 1
type: string
path:
anyOf:
- type: string
- type: 'null'
default: null
max_results:
default: 500
maximum: 2000
minimum: 1
type: integer
type: object
steps:
- backend: home_code_step
operation: list_code_tree
home_read_code_file:
backend: base
description: Read up to 300 lines from a committed code file
parameters:
additionalProperties: false
properties:
path:
minLength: 1
type: string
revision:
default: HEAD
minLength: 1
type: string
start_line:
default: 1
minimum: 1
type: integer
end_line:
anyOf:
- minimum: 1
type: integer
- type: 'null'
default: null
required:
- path
type: object
steps:
- backend: home_code_step
operation: read_code_file
home_search_code:
backend: base
description: Search committed code with a bounded Python multiline regular expression
parameters:
additionalProperties: false
properties:
pattern:
maxLength: 500
minLength: 1
type: string
revision:
default: HEAD
minLength: 1
type: string
path:
anyOf:
- type: string
- type: 'null'
default: null
max_results:
default: 30
maximum: 30
minimum: 1
type: integer
required:
- pattern
type: object
steps:
- backend: home_code_step
operation: search_code

View file

@ -10,18 +10,35 @@ import tomllib
from .spec import PreparationSpec, relative_path
def git(repository: Path, *arguments: str) -> str:
def git_bytes(repository: Path, *arguments: str) -> bytes:
"""Run bounded, noninteractive Git commands without a shell or user hooks."""
environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
environment.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0")
result = subprocess.run(
["git", "-c", "core.hooksPath=" + os.devnull, "-c", "commit.gpgSign=false", "-C", str(repository), *arguments],
[
"git",
"-c",
"core.hooksPath=" + os.devnull,
"-c",
"commit.gpgSign=false",
"-c",
"core.fsmonitor=false",
"-C",
str(repository),
*arguments,
],
check=True,
capture_output=True,
env=environment,
timeout=120,
)
return result.stdout.decode().strip()
return result.stdout
def git(repository: Path, *arguments: str, strip: bool = True) -> str:
"""Decode Git output, optionally preserving porcelain whitespace and path bytes."""
output = git_bytes(repository, *arguments).decode("utf-8")
return output.strip() if strip else output
def tree_fingerprint(root: Path) -> str:

View file

@ -76,3 +76,19 @@ flag. Host-installed benchmark judge plugins score answers outside Docker and
outside the candidate repository.
See `validation-artifact-guide.md` for saved evidence and score interpretation.
## Candidate versions
Use the Home code Jobs to list, create, switch, merge and commit candidates in
`home/code/repo`. `init` is the protected preparation baseline. Branch operations
require clean code; merges stay uncommitted for inspection. Commit all intended
changes with `home_commit_candidate` before debug or scored evaluation. Mutations
are limited to prepared `candidate_paths`; keep repository metadata and runtime
outside the optimizer's write permissions.
`home_list_code_tree`, `home_read_code_file` and `home_search_code` inspect raw
committed files. Their `revision` defaults to current HEAD and returns a resolved
SHA. For diagnosis or version comparisons, the host should inject a fixed SHA
through `injected_job_kwargs` for each agent/Job alias, so subsequent checkout
changes cannot affect the evidence. See the Home plugin README for inputs and
output limits.

View file

@ -120,3 +120,84 @@ structured exception details are treated as infrastructure errors. Construction
failure-rate tolerance applies only to classified candidate errors.
Construction reuse answers and judges again with the current candidate; it does
not reuse donor answers or scores.
## Evaluation history and adaptive sampling evidence
`home_evaluate sampling=true` enables adaptive selection; false preserves full or
explicit evaluation. Debug always bypasses history. `sampling_policy` is the
host policy input, distinct from the boolean toggle. The effective policy is
captured in `manifest.options` and the started plan.
`record_history=true` (the default) includes a successful full baseline even when
`sampling=false`. `record_history=false` runs the same scoring and artifact pipeline
without registering results or advancing any optimization/sampling counters. The
choice is stored in `manifest.options.record_history` and respected by history
registration and rebuilding; a missing legacy setting means true. Debug always
skips history. The response reports effective participation as `record_history`.
An untracked adaptive run may read existing observations and saves its draw locally
in `manifest.selection_plan`, without a shared started plan or ledger round.
`home.evaluation_history` owns the successful-result catalog at
`home/evaluations/history.json`. Entries require completed scored training runs
with `success=true` (or equivalent legacy summary counts) and no infrastructure
errors. Low scores remain useful. Failed/cancelled/debug/partial runs are excluded;
a mixed run with an infrastructure failure contributes no quality observations.
Source manifests, summaries and query results remain authoritative.
`sandbox_started_at` marks the first successfully initialized sandbox. The
history-enabled manifest's `sampling.plan_path` points to its policy, seed, weights, selection
and started round under `home/sampling/plans/`. Pre-start failures have no marker
or plan, and additional workers do not create extra rounds. The derived sampling
cache separates started `round` and `successful_round`: only successful history
advances staleness/recheck timing and trains EWMA. Adaptive plans preserve their
probability clock in `selection_round`. Actual numeric scores alone count as
quality observations; missing/unjudged answers never become zero scores.
Missing caches and `HomeEvaluationHistory.rebuild()` reconstruct the catalog and
sampling observations from saved evidence. Rebuilding never modifies datasets,
candidate code or memory files. See the Home README for configuration, host
per-call policy overrides and enforced agent injection examples.
## Read-only evaluation inspection
Use `home_list_validation_results` to find runs and pass a returned
`validation_path` to `home_inspect_validation`. The catalog includes failed,
debug, running and untracked runs, not only successful optimization history.
Both Jobs page with `offset` and `limit` (default 20, maximum 100), returning
`row_count`, `rows`, `next_offset`. Catalog filters are `branch_name` and
`commit_sha`; a hexadecimal commit prefix must uniquely match among all commits
reachable from the specified branch, or the current branch when omitted.
Inspection without `case_id` returns case summaries; add `case_id` for query
summaries and `query_id` for bounded answer/judge fields. Each row's JSON pointer
is relative to `rows_source`. Failed construction Jobs have separate pagination
and pointers relative to `construction.source`. Query fields are limited to
2,000-character previews and errors to 600 characters. Follow returned source
paths for complete evidence. Missing files/results remain explicit, with null
scores, and partial summaries are used when full summaries are absent. Reads do
not rebuild summaries or modify evaluation history, sampling state or artifacts.
## Memory checkpoint tools
After locating one case, call `home_inspect_memory_history` with its
`validation_path` and `case_id` to page through exported memory checkpoints.
Optional `path` follows a literal memory-workspace-relative file or directory.
The response distinguishes a missing workspace, absent history and an empty
repository; invalid history fails explicitly. Session IDs are extracted only
from the established `session: <id>` commit-subject convention.
Use `home_compare_memory` with `base_ref` and optional `target_ref` (default
`HEAD`) to inspect how memory changed within that evaluation. Add
`target_validation_path` to compare the same case across evaluations; both
references then default to `HEAD`. References must resolve to commits reachable
from each memory HEAD, and hexadecimal prefixes must be unique within that
reachable history. Compare candidate identities and fingerprints before drawing
cross-evaluation conclusions.
Diffs describe committed tracked regular-file contents rather than dirty checkout
files, untracked files or mode-only changes. Binary changes are identified without
printing bytes. Narrow `path` when the 2,000-file, 20-MB snapshot, 5-MB file or
100,000-character diff limit is reached. Hashes cover the selected path scope.
These tools neither reconstruct missing exports nor modify Git/evaluation/history
artifacts. Raw logs and source JSON remain available through the existing result
inspection paths.

View file

@ -0,0 +1,169 @@
"""Adaptive formal-validation sampling ported from dev_meta b53048e4.
Keep the source weighting, weighted ordering and fair-share query allocation.
EvaluationHistory owns persisted observations and started-validation plans.
"""
import json
import math
import random
import time
from typing import Any, Mapping
from pydantic import ConfigDict, Field
from .preparation.data.models import CaseSpec
from .preparation.spec import SpecModel
class SamplingPolicy(SpecModel):
"""Host-owned limits with the same defaults as dev_meta's validation.sampling."""
model_config = ConfigDict(frozen=True)
max_cases: int = Field(default=5, ge=1)
max_queries: int | None = Field(default=None, ge=1)
temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
ewma_alpha: float = Field(default=0.3, gt=0, le=1, allow_inf_nan=False)
uniform_exploration: float = Field(default=0.2, ge=0, le=1, allow_inf_nan=False)
def create_sampling_plan(
cases: list[CaseSpec],
state: Mapping[str, Any],
policy: SamplingPolicy,
*,
eligible_query_ids: Mapping[str, list[str]] | None = None,
) -> dict:
"""Draw without replacement, without advancing state or writing an attempted plan."""
if eligible_query_ids is not None:
cases = [case for case in cases if case.case_id in eligible_query_ids]
if not cases:
raise ValueError("training-set contains no cases to sample")
round_id = int(state.get("round", 0)) + 1
seed = time.time_ns()
rng = random.Random(seed)
case_records = state.get("cases", {})
case_weights = _mixed_weights(
{case.case_id: case_records.get(case.case_id, {}) for case in cases},
round_id,
policy,
)
ordered_cases = _weighted_order(cases, case_weights, rng, lambda case: case.case_id)
query_limited = policy.max_queries is not None
remaining_queries = policy.max_queries
selected_cases: list[str] = []
selected_queries: dict[str, list[str]] = {}
query_weights: dict[str, dict[str, float]] = {}
query_records = state.get("queries", {})
for case in ordered_cases:
if len(selected_cases) >= policy.max_cases:
break
if remaining_queries is not None and remaining_queries <= 0:
break
query_ids = [query.query_id for query in case.queries]
if eligible_query_ids is not None:
allowed = set(eligible_query_ids.get(case.case_id, []))
query_ids = [query_id for query_id in query_ids if query_id in allowed]
if not query_ids:
continue
if query_limited:
weights = _mixed_weights(
{query_id: query_records.get(_query_key(case.case_id, query_id), {}) for query_id in query_ids},
round_id,
policy,
)
query_weights[case.case_id] = weights
ordered_queries = _weighted_order(query_ids, weights, rng, lambda value: value)
open_case_slots = max(policy.max_cases - len(selected_cases), 1)
fair_share = max(math.ceil(int(remaining_queries or 0) / open_case_slots), 1)
take = min(len(ordered_queries), fair_share, int(remaining_queries or 0))
query_ids = ordered_queries[:take]
if not query_ids:
continue
remaining_queries = int(remaining_queries or 0) - len(query_ids)
selected_cases.append(case.case_id)
selected_queries[case.case_id] = query_ids
if not selected_cases:
raise ValueError("sampling budget selected no executable case/query pairs")
return {
"schema_version": 1,
"mode": "sampled",
"round": round_id,
"seed": seed,
"policy": policy.model_dump(),
"case_ids": selected_cases,
"query_ids": selected_queries,
"case_draw_weights": case_weights,
"query_draw_weights": query_weights,
"eligibility": dict(eligible_query_ids) if eligible_query_ids is not None else None,
}
def _observe(record: dict[str, Any], loss: float, round_id: int, alpha: float) -> None:
previous = record.get("ewma_loss")
record["ewma_loss"] = loss if previous is None else alpha * loss + (1.0 - alpha) * float(previous)
record["observed_count"] += 1
record["last_valid_score_round"] = round_id
if loss > 0:
record["last_error_round"] = round_id
record["successful_observations_after_last_error"] = 0
elif record.get("last_error_round") is not None:
record["successful_observations_after_last_error"] += 1
def _adaptive_weight(record: Mapping[str, Any], round_id: int, policy: SamplingPolicy) -> float:
observed = int(record.get("observed_count", 0))
raw_risk = record.get("ewma_loss")
risk = float(raw_risk) if isinstance(raw_risk, (int, float)) and not isinstance(raw_risk, bool) else 0.5
last_selected = record.get("last_selected_round")
staleness = round_id if last_selected is None else max(round_id - int(last_selected), 0)
uncertainty = 1.0 / math.sqrt(observed + 1.0)
last_error = record.get("last_error_round")
successes = int(record.get("successful_observations_after_last_error", 0))
unrechecked_error = 0.0
if last_error is not None:
unrechecked_error = math.log1p(max(round_id - int(last_error), 0)) / (successes + 1.0)
priority = risk + 0.2 * math.log1p(staleness) + 0.2 * uncertainty + 0.1 * unrechecked_error
return math.exp(min(priority / policy.temperature, 50.0))
def _mixed_weights(
records: Mapping[str, Mapping[str, Any]],
round_id: int,
policy: SamplingPolicy,
) -> dict[str, float]:
"""Mix normalized adaptive weights with a true uniform exploration mass."""
adaptive = {key: _adaptive_weight(record, round_id, policy) for key, record in records.items()}
total = sum(adaptive.values())
uniform = 1.0 / len(adaptive)
return {
key: policy.uniform_exploration * uniform + (1.0 - policy.uniform_exploration) * value / total
for key, value in adaptive.items()
}
def _weighted_order(values: list[Any], weights: Mapping[str, float], rng: random.Random, key: Any) -> list[Any]:
return sorted(
values,
key=lambda value: rng.random() ** (1.0 / max(weights[key(value)], 1e-12)),
reverse=True,
)
def _empty_record() -> dict[str, Any]:
return {
"observed_count": 0,
"ewma_loss": None,
"last_selected_round": None,
"last_valid_score_round": None,
"last_error_round": None,
"successful_observations_after_last_error": 0,
"candidate_failure_count": 0,
"infra_error_count": 0,
}
def _query_key(case_id: str, query_id: str) -> str:
return json.dumps([case_id, query_id], ensure_ascii=False, separators=(",", ":"))

View file

@ -6,6 +6,7 @@ from pathlib import Path
from unittest.mock import AsyncMock
import pytest
import yaml
from reme.schema import Response
@ -64,6 +65,13 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
assert observed["plugins"] == ["home", "beam-judge" if benchmark == "beam" else "lme-judge"]
assert observed["service"] == {"backend": "cli"}
assert observed["workspace_dir"] == str(tmp_path / "experiment")
assert observed["components"]["home.evaluation"]["default"]["sampling_policy"] == {
"max_cases": 5,
"max_queries": None,
"temperature": 1.0,
"ewma_alpha": 0.3,
"uniform_exploration": 0.2,
}
application.start.assert_awaited_once()
application.close.assert_awaited_once()
assert application.run_job.await_count == 1
@ -71,6 +79,36 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
assert result["workspace_dir"] == str(tmp_path / "experiment")
@pytest.mark.asyncio
async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch):
"""Partial policies merge typed defaults and never leak into preparation inputs."""
config = yaml.safe_load((ROOT / "benchmark/beam/config_home.yaml").read_text())
config["sampling_policy"] = {"max_cases": 2, "max_queries": 7}
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
application = AsyncMock()
application.run_job.return_value = Response(answer={})
observed = {}
def factory(**kwargs):
observed.update(kwargs)
return application
monkeypatch.setattr(home_runner, "Application", factory)
await home_runner.prepare(path)
policy = observed["components"]["home.evaluation"]["default"]["sampling_policy"]
assert policy["max_cases"] == 2 and policy["max_queries"] == 7 and policy["ewma_alpha"] == 0.3
assert "sampling_policy" not in application.run_job.call_args.kwargs["spec"]
for invalid in ({"max_cases": 0}, {"unknown": 1}, {"max_queries": "7"}):
config["sampling_policy"] = invalid
path.write_text(yaml.safe_dump(config))
with pytest.raises(ValueError):
await home_runner.prepare(path)
assert application.start.await_count == 1
assert application.run_job.await_count == 1
assert application.run_job.call_args.args == ("home_prepare",)
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["startup", "preparation"])
async def test_runner_propagates_failure_and_closes(monkeypatch, failure):

View file

@ -0,0 +1,351 @@
"""Home budgets exercised through AgentScope's real reply and event loop."""
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access,unused-argument
import asyncio
from types import SimpleNamespace
import pytest
from agentscope.agent import Agent, ReActConfig
from agentscope.agent._config import ModelConfig
from agentscope.event import ModelCallEndEvent, ReplyEndEvent, ReplyEndReason
from agentscope.message import AssistantMsg, HintBlock, TextBlock, ThinkingBlock, ToolCallBlock, UserMsg
from agentscope.model import ChatResponse, ChatUsage, FinishedReason
from agentscope.tool import FunctionTool, Toolkit
from reme_home.agent import BoundedReActAgent, RunTimeBudgetTimer
class RecordingModel:
model = "isolated-test"
context_size = 10000
formatter = SimpleNamespace(supported_input_media_types=[])
def __init__(self, *, final=None, stream=False):
self.calls = []
self.final = final
self.stream = stream
async def count_tokens(self, messages, tools):
return 10
async def __call__(self, **kwargs):
self.calls.append(kwargs)
if kwargs["tool_choice"] is not None and kwargs["tool_choice"].mode == "none":
if isinstance(self.final, BaseException):
raise self.final
response = self.final or ChatResponse(content=[TextBlock(text="final report")], is_last=True)
if self.stream:
async def chunks():
yield ChatResponse(content=[TextBlock(text="final ")], is_last=False)
yield response
return chunks()
return response
return ChatResponse(
content=[ThinkingBlock(thinking="investigating")],
is_last=True,
usage=ChatUsage(input_tokens=2, output_tokens=3, time=0),
)
def hints(agent, source):
return [
block
for message in agent.state.context
for block in message.content
if isinstance(block, HintBlock) and block.source == source
]
def make_agent(model=None, timer=None, max_iters=2):
return BoundedReActAgent(
name="investigator",
system_prompt="Investigate the evidence.",
model=model or RecordingModel(),
react_config=ReActConfig(max_iters=max_iters, interruption_raise_cancelled_error=True),
model_config=ModelConfig(max_retries=0),
run_time_budget_timer=timer,
)
@pytest.mark.parametrize("budget", [True, 0, -1, float("nan"), float("inf"), "10"])
def test_timer_rejects_invalid_budget(budget):
with pytest.raises(ValueError, match="positive"):
RunTimeBudgetTimer(budget, agent_label="diagnostic")
def test_timer_thresholds_restart_and_skipped_thresholds():
now = [0.0]
timer = RunTimeBudgetTimer(100, agent_label="diagnostic", clock=lambda: now[0])
assert timer.next_reminder() is None
timer.start()
for percentage, persistent in [(59, None), (80, False), (90, True), (100, False), (110, True)]:
now[0] = percentage * 60
reminder = timer.next_reminder()
if persistent is None:
assert reminder is None
else:
assert reminder.persistent is persistent
assert "does not terminate" in reminder.content
assert timer.next_reminder() is None
timer.record_context_compression()
timer.stop()
assert timer.next_reminder() is None
timer.start()
assert timer.context_compression_count == 0
assert timer.next_reminder() is None
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_native_loop_has_exactly_one_tool_free_finalization(stream):
usage = ChatUsage(input_tokens=5, output_tokens=7, time=0, cache_input_tokens=2)
model = RecordingModel(
final=ChatResponse(content=[TextBlock(text="report")], is_last=True, usage=usage),
stream=stream,
)
agent = make_agent(model)
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
agent.run_time_budget_timer = timer
items = [
item
async for item in agent.reply_stream(UserMsg(name="caller", content="Find the cause"), yield_final_msg=True)
]
assert len(model.calls) == 3
assert agent.state.cur_iter == 3
request = model.calls[-1]
assert request["tools"] == []
assert request["tool_choice"].mode == "none"
assert "Find the cause" in request["messages"][-1].get_text_content()
assert items[-1].get_text_content() == "report"
# Reminders must preserve cumulative reply usage as well as per-call events.
assert items[-1].usage.input_tokens == 9
assert items[-1].usage.output_tokens == 13
assert items[-1].usage.cache_input_tokens == 2
calls = [item for item in items if isinstance(item, ModelCallEndEvent)]
assert len(calls) == 3
assert sum(item.input_tokens for item in calls) == 9
assert sum(item.output_tokens for item in calls) == 13
ends = [item for item in items if isinstance(item, ReplyEndEvent)]
assert len(ends) == 1 and ends[0].finished_reason == ReplyEndReason.EXCEED_MAX_ITERS
assert not timer.active
@pytest.mark.asyncio
async def test_finalization_discards_unsolicited_tools():
executed = []
def forbidden():
executed.append(True)
model = RecordingModel(
final=ChatResponse(
content=[TextBlock(text="report"), ToolCallBlock(id="call", name="forbidden", input="{}")],
is_last=True,
),
)
agent = make_agent(model)
agent.toolkit = Toolkit(tools=[FunctionTool(forbidden)])
result = await agent.reply(UserMsg(name="caller", content="Investigate"))
assert result.get_text_content() == "report"
assert not executed
assert not agent.state.get_unfinished_tool_calls(agent.name)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failure",
[
RuntimeError("provider unavailable"),
ChatResponse(content=[], is_last=True),
ChatResponse(content=[], is_last=True, usage=ChatUsage(input_tokens=5, output_tokens=7, time=0)),
],
)
async def test_finalization_failure_returns_explicit_incomplete_output(failure):
model = RecordingModel(final=failure)
agent = make_agent(model)
result = await agent.reply(UserMsg(name="caller", content="Investigate"))
assert "incomplete prior model output" in result.get_text_content()
assert len(model.calls) == 3
final_usage = failure.usage if isinstance(failure, ChatResponse) else None
assert result.usage.input_tokens == 4 + (final_usage.input_tokens if final_usage else 0)
assert result.usage.output_tokens == 6 + (final_usage.output_tokens if final_usage else 0)
@pytest.mark.asyncio
async def test_cancellation_stops_timer_without_fallback():
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
agent = make_agent(RecordingModel(final=asyncio.CancelledError()), timer)
with pytest.raises(asyncio.CancelledError):
await agent.reply(UserMsg(name="caller", content="Investigate"))
assert not timer.active
assert not any("incomplete prior model output" in (msg.get_text_content() or "") for msg in agent.state.context)
@pytest.mark.asyncio
async def test_interrupted_response_preserves_native_interruption():
model = RecordingModel(final=ChatResponse(content=[], is_last=True, finished_reason=FinishedReason.INTERRUPTED))
with pytest.raises(asyncio.CancelledError):
await make_agent(model).reply(UserMsg(name="caller", content="Investigate"))
@pytest.mark.asyncio
async def test_new_reply_resets_iteration_reminders():
model = RecordingModel()
agent = make_agent(model)
for task in ("first", "second"):
result = await agent.reply(UserMsg(name="caller", content=task))
assert result.get_text_content() == "final report"
assert agent._original_task_prompt == task
assert len(model.calls) == 6
reminders = hints(agent, "home_iteration_guard")
assert len(reminders) == 4
@pytest.mark.asyncio
async def test_normal_reply_does_not_force_finalization():
class ImmediateModel(RecordingModel):
async def __call__(self, **kwargs):
self.calls.append(kwargs)
return ChatResponse(content=[TextBlock(text="already sufficient")], is_last=True)
model = ImmediateModel()
result = await make_agent(model).reply(UserMsg(name="caller", content="Investigate"))
assert result.get_text_content() == "already sufficient"
assert len(model.calls) == 1
@pytest.mark.asyncio
async def test_final_context_trims_old_evidence_but_preserves_system_and_task():
class SmallContextModel(RecordingModel):
async def count_tokens(self, messages, tools):
return 10000 if len(messages) > 2 else 10
model = SmallContextModel()
agent = make_agent(model)
agent.state.cur_iter = 2
agent._original_task_prompt = "original task"
messages = [UserMsg(name="system", content="system prompt"), UserMsg(name="caller", content="old evidence")]
await agent._call_model(messages, tools=[{"name": "unused"}])
assert len(model.calls[-1]["messages"]) == 2
assert model.calls[-1]["messages"][0] is messages[0]
assert "original task" in model.calls[-1]["messages"][-1].get_text_content()
assert len(messages) == 2
@pytest.mark.asyncio
async def test_missing_stream_completion_returns_previous_text_without_retry():
class IncompleteStreamModel(RecordingModel):
async def __call__(self, **kwargs):
self.calls.append(kwargs)
async def chunks():
yield ChatResponse(content=[TextBlock(text="partial")], is_last=False)
return chunks()
model = IncompleteStreamModel()
agent = make_agent(model)
agent.state.cur_iter = 2
agent.state.context.append(AssistantMsg(name=agent.name, content="previous finding"))
response = await agent._call_model([], tools=[])
assert "incomplete prior model output" in response.content[0].text
assert "previous finding" in response.content[0].text
assert len(model.calls) == 1
@pytest.mark.asyncio
async def test_time_reminders_alternate_and_do_not_stop_investigation():
now = [0.0]
timer = RunTimeBudgetTimer(100, agent_label="diagnostic", clock=lambda: now[0])
model = RecordingModel()
agent = make_agent(model, timer, max_iters=100)
timer.start()
for percentage in (60, 75, 90, 100, 110):
now[0] = percentage * 60
_ = [item async for item in agent._reasoning()]
persistent = hints(agent, "home_run_time_guard")
assert len(persistent) == 2
assert "(75%" in persistent[0].hint
assert "(100%" in persistent[1].hint
assert len(model.calls) == 5
assert timer.active
@pytest.mark.asyncio
async def test_ephemeral_reminder_removed_when_reasoning_fails(monkeypatch):
now = [0.0]
timer = RunTimeBudgetTimer(1, agent_label="diagnostic", clock=lambda: now[0])
agent = make_agent(timer=timer)
timer.start()
now[0] = 40
async def fail(*args, **kwargs):
raise RuntimeError("failed")
yield # pylint: disable=unreachable
monkeypatch.setattr(Agent, "_reasoning", fail)
with pytest.raises(RuntimeError):
_ = [item async for item in agent._reasoning()]
assert not hints(agent, "home_run_time_guard")
@pytest.mark.asyncio
async def test_only_successful_context_replacement_counts_as_compression(monkeypatch):
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
agent = make_agent(timer=timer)
async def no_change(self):
pass
monkeypatch.setattr(Agent, "compress_context", no_change)
await agent.compress_context()
assert timer.context_compression_count == 0
async def change(self):
self.state.context = []
monkeypatch.setattr(Agent, "compress_context", change)
await agent.compress_context()
assert timer.context_compression_count == 1
class CompressionFailureModel(RecordingModel):
def __init__(self, failure, fail_after=2):
super().__init__()
self.failure = failure
self.fail_after = fail_after
async def count_tokens(self, messages, tools):
return 9000 if len(self.calls) >= self.fail_after and len(messages) > 2 else 10
async def generate_structured_output(self, **kwargs):
raise self.failure
@pytest.mark.asyncio
async def test_final_report_survives_real_compression_failure():
model = CompressionFailureModel(RuntimeError("compression provider unavailable"))
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
agent = make_agent(model, timer)
agent.context_config.compression_fallback_to_truncation = False
result = await agent.reply(UserMsg(name="caller", content="Investigate"))
assert result.get_text_content() == "final report"
assert len(model.calls) == 3
assert model.calls[-1]["tools"] == []
assert len(model.calls[-1]["messages"]) == 2
assert timer.context_compression_count == 0
assert not timer.active
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", [RuntimeError("compression failed"), asyncio.CancelledError()])
async def test_nonfinal_compression_errors_and_cancellation_propagate(failure):
model = CompressionFailureModel(failure, fail_after=1 if isinstance(failure, RuntimeError) else 2)
agent = make_agent(model)
agent.context_config.compression_fallback_to_truncation = False
with pytest.raises(type(failure)):
await agent.reply(UserMsg(name="caller", content="Investigate"))
assert len(model.calls) == model.fail_after

View file

@ -0,0 +1,210 @@
"""Bounded ReMe agents use real Job tools, injection and persisted sessions."""
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access
import json
from importlib.metadata import EntryPoint
from types import SimpleNamespace
import pytest
from agentscope.message import HintBlock, TextBlock, ToolCallBlock
from agentscope.model import ChatResponse, ChatUsage
from agentscope.tool import FunctionTool, Toolkit
from reme_home.agent import BoundedAgentWrapper, BoundedReActAgent, RunTimeBudgetTimer
from test_bounded_agent import RecordingModel
from reme.components.agent_wrapper import AsAgentWrapper
from reme.components.application_context import ApplicationContext
from reme.components.job import BaseJob
from reme.enumeration import ComponentEnum
from reme.plugin import resolve_plugin_runtime
from reme.steps.base_step import BaseStep
from reme.utils import global_counter_get
class CaptureStep(BaseStep):
async def execute(self):
self.app_context.metadata.setdefault("test.calls", []).append(dict(self.context.data))
self.context.response.answer = "captured"
class JobModel(RecordingModel):
def __init__(self, arguments):
super().__init__()
self.arguments = arguments
async def __call__(self, **kwargs):
self.calls.append(kwargs)
content = (
[ToolCallBlock(id="capture-call", name="capture", input=json.dumps(self.arguments))]
if len(self.calls) == 1
else [TextBlock(text="finished")]
)
return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0))
@pytest.fixture(name="wrapper")
def bounded_wrapper(tmp_path):
context = ApplicationContext(workspace_dir=str(tmp_path))
result = BoundedAgentWrapper(
app_context=context,
name="investigator",
as_llm="",
time_budget_minutes=30,
react_config={"max_iters": 2},
)
result.as_llm = SimpleNamespace(model=RecordingModel())
return result
def test_plugin_registers_bounded_agent_only_when_enabled(monkeypatch):
entry = EntryPoint(name="home", value="reme_home", group="reme.plugins")
monkeypatch.setattr("reme.plugin.find_entry_points", lambda group, name: [entry])
enabled = resolve_plugin_runtime({"plugins": ["home"]})
assert enabled.registry.get(ComponentEnum.AGENT_WRAPPER, "home_bounded") is BoundedAgentWrapper
assert resolve_plugin_runtime({}).registry.get(ComponentEnum.AGENT_WRAPPER, "home_bounded") is None
assert issubclass(BoundedAgentWrapper, AsAgentWrapper)
@pytest.mark.asyncio
async def test_each_build_owns_timer_and_honors_call_overrides(wrapper):
first, _ = await wrapper._build_agent("first", **wrapper._merged_kwargs({}))
second, _ = await wrapper._build_agent("second", **wrapper._merged_kwargs({"time_budget_minutes": 5}))
assert isinstance(first, BoundedReActAgent)
assert first.run_time_budget_timer.time_budget_minutes == 30
assert second.run_time_budget_timer.time_budget_minutes == 5
assert first.run_time_budget_timer is not second.run_time_budget_timer
assert first.react_config.max_iters == 2
no_timer, _ = await wrapper._build_agent("third", **wrapper._merged_kwargs({"time_budget_minutes": None}))
assert no_timer.run_time_budget_timer is None
@pytest.mark.asyncio
@pytest.mark.parametrize("conflict", [False, True])
async def test_real_job_tools_enforce_hidden_injected_arguments(wrapper, conflict):
context = wrapper.app_context
context.registry.register(CaptureStep, "test_capture")
job = BaseJob(
name="capture",
app_context=context,
parameters={
"type": "object",
"properties": {"visible": {"type": "string"}, "case_ids": {"type": "array"}},
"required": ["visible", "case_ids"],
},
steps=[{"backend": "test_capture"}],
)
context.jobs["capture"] = job
arguments = {"visible": "model value"}
if conflict:
arguments["case_ids"] = ["model-chosen"]
model = JobModel(arguments)
wrapper.as_llm.model = model
await job.start()
await wrapper.start()
try:
result = await wrapper.reply(
"Run the capture job",
job_tools=["capture"],
injected_job_kwargs={"case_ids": ["host-chosen"]},
tool_context_id="investigation-1",
)
schemas = json.dumps(model.calls[0]["tools"])
assert "case_ids" not in schemas
assert "tool_context_id" not in schemas
if conflict:
assert "test.calls" not in context.metadata
else:
assert context.metadata["test.calls"] == [
{
"visible": "model value",
"case_ids": ["host-chosen"],
"tool_context_id": "investigation-1",
},
]
assert result["result"] == "finished"
assert (wrapper.session_path / f"{result['session_id']}.jsonl").is_file()
resumed = await wrapper.reply("Continue", resume=result["session_id"])
assert resumed["session_id"] == result["session_id"]
finally:
await wrapper.close()
await job.close()
@pytest.mark.asyncio
async def test_wrapper_preserves_observed_task_in_forced_finalization(wrapper):
result = await wrapper.reply("Identify the original failure")
assert result["result"] == "final report"
messages = wrapper.as_llm.model.calls[-1]["messages"]
assert "Original task: Identify the original failure" in messages[-1].get_text_content()
assert result["session_id"]
@pytest.mark.asyncio
async def test_wrapper_streams_reme_chunks_and_persists_session(wrapper):
chunks = [chunk async for chunk in wrapper.reply_stream("Investigate")]
assert any(chunk.chunk == "final report" for chunk in chunks)
session_id = chunks[-1].session_id
assert session_id
assert (wrapper.session_path / f"{session_id}.jsonl").is_file()
@pytest.mark.asyncio
@pytest.mark.parametrize("timed", [False, True])
async def test_multiround_usage_and_tools_continue_past_soft_time_limit(wrapper, monkeypatch, timed):
now = [0.0]
timer = RunTimeBudgetTimer(1, agent_label="investigator", clock=lambda: now[0])
monkeypatch.setattr("reme_home.agent.wrapper.RunTimeBudgetTimer", lambda *args, **kwargs: timer)
executions = []
async def inspect_evidence():
executions.append(now[0])
return "evidence"
class MultiRoundModel(RecordingModel):
def __init__(self):
super().__init__()
self.time_reminders = []
async def __call__(self, **kwargs):
self.calls.append(kwargs)
self.time_reminders.extend(
block.hint
for message in kwargs["messages"]
for block in message.content
if isinstance(block, HintBlock) and block.source == "home_run_time_guard"
)
now[0] += 65
count = len(self.calls)
content = (
[ToolCallBlock(id=f"inspect-{count}", name="inspect_evidence", input="{}")]
if count < 3
else [TextBlock(text="finished")]
)
return ChatResponse(
content=content,
is_last=True,
usage=ChatUsage(input_tokens=10, output_tokens=5, time=0),
)
model = MultiRoundModel()
wrapper.as_llm.model = model
options = {
"react_config": {"max_iters": 4},
"time_budget_minutes": 1 if timed else None,
"toolkit": Toolkit(tools=[FunctionTool(inspect_evidence)]),
}
result = await wrapper.reply("Investigate", **options)
assert result["usage"] == {"input_tokens": 30, "output_tokens": 15, "total_tokens": 45}
assert result["last_message"]["finished_reason"] == "completed"
assert len(executions) == 2
assert executions[-1] > 60 # A tool really executes after the soft deadline.
assert len(model.calls) == 3
assert all(call["tools"] for call in model.calls)
assert bool(model.time_reminders) is timed
assert not timer.active
metadata = wrapper.app_context.metadata
assert global_counter_get(metadata, ["__token_counter", wrapper.name, "total_tokens"]) == 45
resumed = await wrapper.reply("Continue", resume=result["session_id"], **options)
assert resumed["usage"]["total_tokens"] == 15
assert global_counter_get(metadata, ["__token_counter", wrapper.name, "total_tokens"]) == 60

View file

@ -0,0 +1,316 @@
"""Candidate version Jobs exercised against real isolated Git repositories."""
# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name,unused-argument
import asyncio
from pathlib import Path
import subprocess
from threading import Event
import pytest
import pytest_asyncio
from reme_home.code import snapshot
from reme_home.code.models import ARGUMENTS
from reme_home.workspace import _initialization_lock
from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
command,
)
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
@pytest_asyncio.fixture
async def prepared(app, inputs):
spec, _ = inputs
response = await app.run_job("home_prepare", spec=spec)
assert response.success, response.answer
repo = Path(app.config.workspace_dir) / "home/code/repo"
return app, repo, spec
async def run(app, job, **kwargs):
response = await app.run_job(job, **kwargs)
assert response.success, response.answer
return response.answer
async def create(app, name="candidate"):
return await run(app, "home_create_candidate_branch", branch_name=name)
async def commit(app, repo, text="value = 2\n"):
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text(text)
return await run(app, "home_commit_candidate", message="Improve candidate")
@pytest.mark.asyncio
async def test_branch_commit_diff_and_immutable_read(prepared):
app, repo, _ = prepared
initial = await run(app, "home_list_candidate_branches")
assert initial["current_branch"] == "init"
assert initial["candidate_branches"] == []
created = await create(app)
assert created["candidate_branch_count"] == 1
result = await commit(app, repo)
assert result["commit_sha"] != initial["commit_sha"]
assert (await run(app, "home_code_status"))["clean"]
diff = await run(app, "home_code_diff", base_revision="init", revision=result["commit_sha"])
assert "+value = 2" in diff["content"]
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text("dirty = 3\n")
status = await run(app, "home_code_status")
assert status["changes"] == [{"status": " M", "path": "plugins/beam/src/reme_beam/__init__.py"}]
assert not status["clean"]
assert "+dirty = 3" in (await run(app, "home_code_diff"))["content"]
read = await run(
app,
"home_read_code_file",
path="plugins/beam/src/reme_beam/__init__.py",
revision=initial["commit_sha"],
)
assert read["content"] == "value = 1\n"
with pytest.raises(ValueError, match="clean"):
await app.context.components["home.code"]["default"].run("switch_branch", branch_name="init")
@pytest.mark.asyncio
async def test_host_pinned_job_tool_and_schema(prepared):
app, repo, _ = prepared
initial = command(repo, "rev-parse", "HEAD")
await create(app)
await commit(app, repo)
job = app.context.jobs["home_read_code_file"]
tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"revision": initial})
assert "revision" not in tool.input_schema["properties"]
# Test the actual wrapper's merge contract through its executable FunctionTool.
response = await tool(path="plugins/beam/src/reme_beam/__init__.py")
assert "value = 1" in response.content[0].text
assert "value = 2" not in response.content[0].text
with pytest.raises(ValueError, match="injected"):
await tool(path="plugins/beam/src/reme_beam/__init__.py", revision="HEAD")
assert all(model.model_json_schema()["additionalProperties"] is False for model in ARGUMENTS.values())
@pytest.mark.asyncio
@pytest.mark.parametrize("name", ["init", "a/b", "../escape", "-bad", "a\\b", "@{-1}"])
async def test_invalid_candidate_branch(prepared, name):
app, _, _ = prepared
response = await app.run_job("home_create_candidate_branch", branch_name=name)
assert not response.success
assert (await run(app, "home_code_status"))["current_branch"] == "init"
@pytest.mark.asyncio
async def test_protected_init_and_scope(prepared):
app, repo, _ = prepared
response = await app.run_job("home_commit_candidate", message="not allowed")
assert not response.success
await create(app)
baseline = command(repo, "rev-parse", "HEAD")
readme = repo / "plugins/beam/README.md"
readme.write_text("outside declared editable roots")
response = await app.run_job("home_commit_candidate", message="outside scope")
assert not response.success
assert command(repo, "rev-parse", "HEAD") == baseline
assert not command(repo, "diff", "--cached", "--name-only")
readme.write_text("Plugin installation instructions\n")
target = repo / "plugins/beam/src/link"
target.symlink_to("/etc/passwd")
response = await app.run_job("home_commit_candidate", message="link")
assert not response.success
target.unlink()
result = await commit(app, repo)
command(repo, "update-ref", "refs/heads/init", result["commit_sha"])
response = await app.run_job("home_code_status")
assert not response.success
@pytest.mark.asyncio
async def test_merge_then_explicit_commit(prepared):
app, repo, _ = prepared
await create(app, "left")
left = await commit(app, repo)
await run(app, "home_switch_candidate_branch", branch_name="init")
await create(app, "right")
(repo / "plugins/beam/src/new.py").write_text("another mechanism\n")
right = await run(app, "home_commit_candidate", message="right")
merged = await run(app, "home_merge_candidate_branch", branch_name="left")
assert merged["status"] == "merged_not_committed"
assert merged["commit_sha"] == right["commit_sha"]
assert (await run(app, "home_code_status"))["merge_pending"]
response = await app.run_job("home_switch_candidate_branch", branch_name="init")
assert not response.success
merged = await run(app, "home_commit_candidate", message="Combine compatible improvements")
assert command(repo, "rev-list", "--parents", "-n", "1", "HEAD").split() == [
merged["commit_sha"],
right["commit_sha"],
left["commit_sha"],
]
@pytest.mark.asyncio
async def test_conflicting_merge_aborts_cleanly(prepared):
app, repo, _ = prepared
await create(app, "left")
await commit(app, repo, "left = 2\n")
await run(app, "home_switch_candidate_branch", branch_name="init")
await create(app, "right")
right = await commit(app, repo, "right = 2\n")
response = await app.run_job("home_merge_candidate_branch", branch_name="left")
assert not response.success
assert not (repo / ".git/MERGE_HEAD").exists()
state = await run(app, "home_code_status")
assert state["clean"] and state["commit_sha"] == right["commit_sha"]
@pytest.mark.asyncio
async def test_raw_snapshot_regex_paths_and_truncation(prepared):
app, repo, _ = prepared
await create(app)
folder = repo / "plugins/beam/src"
(folder / "space name.py").write_text("prefix\nhello\nworld\nsuffix\n" + "x\n" * 400)
(folder / ".gitattributes").write_text('"space name.py" export-ignore\n')
(folder / "binary").write_bytes(b"a\0b")
(folder / "carriage.py").write_bytes(b"first\rsecond\rthird")
(folder / "colon:name.py").write_text("hello")
# Non-portable paths fail closed instead of becoming Git pathspecs.
rejected = await app.run_job("home_commit_candidate", message="bad path")
assert not rejected.success
(folder / "colon:name.py").unlink()
result = await run(app, "home_commit_candidate", message="Navigation fixtures")
tree = await run(app, "home_list_code_tree", path="plugins/beam/src", max_results=1)
assert tree["revision"] == result["commit_sha"] and tree["truncated"]
found = await run(app, "home_search_code", pattern="hello\nworld", path="plugins/beam/src")
assert found["occurrence_count"] == 1
assert found["matches"][0]["start_line"] == 1
assert "hello" in found["matches"][0]["content"]
carriage = await run(app, "home_search_code", pattern="second", path="plugins/beam/src/carriage.py")
assert "> 2 | second" in carriage["matches"][0]["content"]
read = await run(app, "home_read_code_file", path="plugins/beam/src/space name.py")
assert read["end_line"] == 300 and read["truncated"]
for kwargs in [
{"path": "../escape"},
{"path": ".git/HEAD"},
{"path": "/etc/passwd"},
{"path": "plugins/beam/src/binary"},
{"path": "plugins/beam/src", "revision": "--help"},
]:
assert not (await app.run_job("home_read_code_file", **kwargs)).success
for pattern in ("[", "^", ""):
assert not (await app.run_job("home_search_code", pattern=pattern)).success
assert not (await app.run_job("home_list_code_tree", path=":(glob)**")).success
@pytest.mark.asyncio
async def test_lock_restrictions_and_unprepared(app):
assert not (await app.run_job("home_code_status")).success
assert not (await app.run_job("home_code_status", _allowed_paths=[])).success
component = app.context.components["home.code"]["default"]
with pytest.raises(ValueError, match="Unknown"):
await component.run("reset")
with pytest.raises(ValueError):
await component.run("create_branch", branch_name=1)
with _initialization_lock(component.workspace.state_path / "workspace.lock"):
assert not (await app.run_job("home_list_code_tree")).success
@pytest.mark.asyncio
async def test_cancel_waits_for_owned_git_thread(prepared, monkeypatch):
app, _, _ = prepared
component = app.context.components["home.code"]["default"]
entered, release, finished = Event(), Event(), Event()
original = component._run
def blocked(*args):
entered.set()
assert release.wait(5)
try:
return original(*args)
finally:
finished.set()
monkeypatch.setattr(component, "_run", blocked)
task = asyncio.create_task(component.run("status"))
assert await asyncio.to_thread(entered.wait, 5)
task.cancel()
await asyncio.sleep(0)
assert not task.done()
release.set()
with pytest.raises(asyncio.CancelledError):
await task
assert finished.is_set() and not component._pending
@pytest.mark.asyncio
async def test_snapshot_size_and_regex_timeout(prepared, monkeypatch):
app, repo, _ = prepared
await create(app)
await commit(app, repo, 'x = "' + "a" * 10000 + '!"\n')
original = snapshot.subprocess.run
def short_timeout(*args, **kwargs):
if args[0][0] == snapshot.sys.executable:
kwargs["timeout"] = 0.05
return original(*args, **kwargs) # pylint: disable=subprocess-run-check
monkeypatch.setattr(snapshot.subprocess, "run", short_timeout)
with pytest.raises(subprocess.TimeoutExpired):
snapshot.search_code(repo, {}, pattern="(a+)+$", path="plugins/beam/src")
monkeypatch.setattr(snapshot.subprocess, "run", original)
monkeypatch.setattr(snapshot, "MAX_FILE_BYTES", 3)
response = await app.run_job("home_read_code_file", path="plugins/beam/src/reme_beam/__init__.py")
assert not response.success
@pytest.mark.asyncio
@pytest.mark.parametrize("mutation", ["staged", "untracked"])
async def test_branch_operations_refuse_dirty_inputs(prepared, mutation):
app, repo, _ = prepared
if mutation == "staged":
target = repo / "plugins/beam/src/reme_beam/__init__.py"
target.write_text("staged = True\n")
command(repo, "add", ".")
else:
target = repo / "plugins/beam/src/untracked file.py"
target.write_text("untracked = True\n")
before = target.read_bytes()
assert not (await app.run_job("home_create_candidate_branch", branch_name="dirty")).success
assert target.read_bytes() == before
assert command(repo, "branch", "--show-current") == "init"
@pytest.mark.asyncio
async def test_exact_paths_rename_deletion_and_revision_pinning(prepared):
app, repo, _ = prepared
await create(app)
original = repo / "plugins/beam/src/reme_beam/__init__.py"
name = "plugins/beam/src/space name\nand newline.py"
target = repo / name
original.rename(target)
first = await run(app, "home_commit_candidate", message="Rename a file")
assert (await run(app, "home_read_code_file", path=name))["content"] == "value = 1\n"
target.write_text("value = 9\n")
pinned = await run(app, "home_commit_candidate", message="Improve renamed code")
target.unlink()
await run(app, "home_commit_candidate", message="Remove obsolete file")
await run(app, "home_switch_candidate_branch", branch_name="init")
read = await run(app, "home_read_code_file", path=name, revision=pinned["commit_sha"])
assert read["content"] == "value = 9\n"
assert name in (await run(app, "home_list_code_tree", revision=first["commit_sha"]))["paths"]
@pytest.mark.asyncio
async def test_reject_symlink_repository_and_detached_candidate(prepared):
app, repo, _ = prepared
await create(app)
commit_sha = (await commit(app, repo))["commit_sha"]
command(repo, "checkout", "--detach", commit_sha)
assert not (await app.run_job("home_commit_candidate", message="detached")).success
assert (await run(app, "home_read_code_file", path="plugins/beam/src/reme_beam/__init__.py"))["content"]
moved = repo.with_name("moved")
repo.rename(moved)
repo.symlink_to(moved, target_is_directory=True)
assert not (await app.run_job("home_code_status")).success

View file

@ -126,6 +126,8 @@ class FakeFactory:
async def create_cases(self, ids, **kwargs):
self.workers = [FakeWorker(self) for _ in ids]
if self.workers and self.kwargs.get("on_sandbox_started"):
self.kwargs["on_sandbox_started"]()
return self.workers
@ -209,6 +211,7 @@ async def test_reuse_tar_runs_current_candidate_without_building(app, prepared):
baseline = await app.run_job("home_evaluate")
assert baseline.success, baseline.answer
repo = prepared / "code/repo"
assert (await app.run_job("home_create_candidate_branch", branch_name="new-answer")).success
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text("value = 2\n")
command(repo, "add", ".")
command(repo, "-c", "user.name=Fixture", "-c", "user.email=test@localhost", "commit", "-m", "new answer")
@ -414,6 +417,7 @@ async def test_debug_empty_answers_fail_without_scoring(app, prepared, monkeypat
@pytest.mark.asyncio
async def test_current_head_is_immutable_and_does_not_ship_judge(app, prepared):
repo = prepared / "code/repo"
assert (await app.run_job("home_create_candidate_branch", branch_name="changed")).success
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text("value = 3\n", encoding="utf-8")
command(repo, "add", ".")
command(repo, "-c", "user.name=Fixture", "-c", "user.email=test@localhost", "commit", "-m", "change")

View file

@ -0,0 +1,360 @@
"""Successful-history ownership and evaluation parameter precedence through real Jobs."""
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
import json
from pathlib import Path
import pytest
from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state
from reme_home.sampling import SamplingPolicy, create_sampling_plan
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
home_application,
preparation_inputs,
prepared_home,
)
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
def history(app):
return app.context.components["home.evaluation_history"]["default"]
def manifest(response):
return json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
@pytest.mark.asyncio
async def test_history_catalog_keeps_successful_low_scores_and_deduplicates(app, prepared):
manager = history(app)
assert isinstance(manager, HomeEvaluationHistory) and manager.is_started
assert manager.list_results() == []
app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": '{"score": 0}'}
response = await app.run_job("home_evaluate")
assert response.success, response.answer
results = manager.list_results()
assert len(results) == 1 and isinstance(results[0], EvaluationRecord)
assert results[0].mean_query_score == 0 and results[0].scored_query_count == 2
assert manager.get_result(results[0].run_key) == results[0]
run = Path(response.answer["result_dir"])
before = manager.sampling_state()
assert manager.record(run) == results[0]
assert len(manager.list_results()) == 1 and manager.sampling_state() == before
payload = json.loads((prepared / "evaluations/history.json").read_text())
assert payload["results"][0]["run_key"] == results[0].run_key
assert "golden_answer" not in json.dumps(payload)
debug = await app.run_job("home_evaluate", sampling=True, debug=True)
assert debug.success and manager.record(Path(debug.answer["result_dir"])) is None
assert manager.list_results() == results
with pytest.raises(KeyError):
manager.get_result("unknown")
@pytest.mark.asyncio
async def test_failed_run_does_not_change_next_probabilities_or_successful_history(app, prepared, monkeypatch):
baseline = await app.run_job("home_evaluate")
assert baseline.success
manager = history(app)
records, observations = manager.list_results(), manager.sampling_state()
policy = SamplingPolicy()
from reme_home.preparation.data.models import CaseSpec
cases = [
CaseSpec.model_validate_json(path.read_text())
for path in sorted(
(prepared / "datasets/training-set/cases").glob("*.json"),
)
]
monkeypatch.setattr("reme_home.sampling.time.time_ns", lambda: 5678)
before = create_sampling_plan(cases, observations, policy)
original = FakeFactory.create_cases
async def create(factory, *args, **kwargs):
workers = await original(factory, *args, **kwargs)
for worker in workers:
build = worker.run_build
async def fail_one(jobs, worker=worker, build=build):
if worker.case_id == "2":
raise ConnectionError("only one case failed")
return await build(jobs)
worker.run_build = fail_one
return workers
monkeypatch.setattr(FakeFactory, "create_cases", create)
failed = await app.run_job("home_evaluate", concurrency=1)
assert not failed.success
assert failed.answer["scored_query_count"] == 1 # Its healthy case is also excluded from feedback.
assert manager.record(Path(failed.answer["result_dir"])) is None
assert manager.list_results() == records and manager.sampling_state() == observations
assert create_sampling_plan(cases, manager.sampling_state(), policy) == before
assert read_state(prepared)["round"] == 2
assert read_state(prepared)["successful_round"] == 1
assert manager.rebuild() == records
assert manager.sampling_state() == observations
@pytest.mark.asyncio
async def test_history_and_observations_rebuild_without_touching_artifacts(app, prepared):
baseline = await app.run_job("home_evaluate")
sampled = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_cases": 1})
assert baseline.success and sampled.success
manager = history(app)
expected, observations = manager.list_results(), manager.sampling_state()
source = Path(sampled.answer["result_dir"]) / "summary.json"
original = source.read_bytes()
(prepared / "evaluations/history.json").unlink()
(prepared / "sampling/state.json").unlink()
assert manager.list_results() == expected
assert manager.sampling_state() == observations
assert manager.rebuild() == expected
assert source.read_bytes() == original
assert (prepared / "evaluations/history.json").is_file()
assert (prepared / "sampling/state.json").is_file()
@pytest.mark.asyncio
async def test_call_policy_merges_component_defaults_without_mutating_them(app, prepared):
evaluator = app.context.components["home.evaluation"]["default"]
evaluator.sampling_policy = SamplingPolicy(max_cases=2, max_queries=2, temperature=0.7, ewma_alpha=0.6)
response = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_cases": 1})
assert response.success, response.answer
options = manifest(response)["options"]
assert options["sampling"] is True
assert options["sampling_policy"] == {
"max_cases": 1,
"max_queries": 2,
"temperature": 0.7,
"ewma_alpha": 0.6,
"uniform_exploration": 0.2,
}
assert response.answer["case_count"] == 1
following = await app.run_job("home_evaluate", sampling=True)
assert following.success and following.answer["case_count"] == 2
assert evaluator.sampling_policy.max_cases == 2
assert manifest(following)["options"]["sampling_policy"]["max_cases"] == 2
for kwargs in (
{"sampling": "true"},
{"sampling": {"max_cases": 1}},
{"record_history": "false"},
{"sampling_policy": {"max_cases": 0}},
{"sampling_policy": {"unknown": 1}},
):
assert not (await app.run_job("home_evaluate", **kwargs)).success
assert len(history(app).list_results()) == 2
@pytest.mark.asyncio
async def test_job_defaults_and_hidden_agent_injection_use_one_job(app, prepared):
job = app.context.jobs["home_evaluate"]
assert "home_validate" not in app.context.jobs
job.kwargs.update(sampling=True, sampling_policy={"max_cases": 1})
default = await app.run_job("home_evaluate")
assert default.success and default.answer["case_count"] == 1
explicit = await app.run_job("home_evaluate", sampling=False)
assert explicit.success and explicit.answer["case_count"] == 2
tool = AsAgentWrapper._make_tool(
job,
injected_job_kwargs={
"sampling": True,
"sampling_policy": {"max_cases": 1, "max_queries": 1},
},
)
assert "sampling" not in tool.input_schema["properties"]
await tool.call()
records = history(app).list_results()
assert records[-1].case_count == records[-1].scored_query_count == 1
assert len(records) == 3
with pytest.raises(ValueError, match="injected"):
await tool.call(sampling=False)
with pytest.raises(ValueError, match="injected"):
await tool.call(sampling_policy={"max_cases": 100})
assert len(history(app).list_results()) == 3
@pytest.mark.asyncio
@pytest.mark.parametrize("baseline", [False, True])
async def test_standalone_full_evaluation_preserves_history_and_rebuild_excludes_it(app, prepared, baseline):
manager = history(app)
if baseline:
assert (await app.run_job("home_evaluate")).success
records, observations = manager.list_results(), manager.sampling_state()
cache_paths = [prepared / name for name in ("evaluations/history.json", "sampling/state.json")]
caches = [path.read_bytes() if path.exists() else None for path in cache_paths]
response = await app.run_job("home_evaluate", sampling=False, record_history=False)
assert response.success, response.answer
assert response.answer["case_count"] == response.answer["scored_query_count"] == 2
assert response.answer["mean_query_score"] == 0.5 and response.answer["record_history"] is False
run = Path(response.answer["result_dir"])
payload = manifest(response)
assert payload["status"] == "completed" and payload["success"] is True
assert payload["options"]["record_history"] is False and payload["sandbox_started_at"]
assert "sampling" not in payload and "selection_plan" not in payload
assert (run / "summary.json").is_file() and (run / "cases/1/queries/result.json").is_file()
assert manager.record_started(run, SamplingPolicy()) == {}
assert manager.record(run) is None
assert [path.read_bytes() if path.exists() else None for path in cache_paths] == caches
assert manager.list_results() == records and manager.sampling_state() == observations
assert manager.rebuild() == records and manager.sampling_state() == observations
following = await app.run_job("home_evaluate")
assert following.success and following.answer["record_history"] is True
state = read_state(prepared)
assert state["round"] == state["successful_round"] == 1 + int(baseline)
# Legacy manifests without the new flag keep the previous opt-in behavior.
tracked_run = Path(following.answer["result_dir"])
tracked_manifest = manifest(following)
tracked_manifest["options"].pop("record_history")
(tracked_run / "manifest.json").write_text(json.dumps(tracked_manifest))
assert len(manager.rebuild()) == 1 + int(baseline)
@pytest.mark.asyncio
async def test_untracked_full_evaluation_does_not_consult_history(app, prepared, monkeypatch):
manager = history(app)
def fail(*args, **kwargs):
raise AssertionError("standalone full evaluation must not access history")
for name in ("sampling_state", "record_started", "record", "invalidate"):
monkeypatch.setattr(manager, name, fail)
response = await app.run_job("home_evaluate", record_history=False)
assert response.success, response.answer
assert response.answer["case_count"] == response.answer["scored_query_count"] == 2
@pytest.mark.asyncio
async def test_sampling_and_history_are_independent_and_selection_evidence_is_local(app, prepared):
assert (await app.run_job("home_evaluate")).success
manager = history(app)
records, observations = manager.list_results(), manager.sampling_state()
response = await app.run_job(
"home_evaluate",
sampling=True,
record_history=False,
sampling_policy={"max_cases": 1},
)
assert response.success and response.answer["case_count"] == 1
payload = manifest(response)
assert "sampling" not in payload
assert payload["selection_plan"]["case_ids"] == payload["case_ids"]
assert payload["selection_plan"]["seed"] and payload["selection_plan"]["policy"]["max_cases"] == 1
assert manager.rebuild() == records and manager.sampling_state() == observations
assert read_state(prepared)["round"] == read_state(prepared)["successful_round"] == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["infra", "exception"])
async def test_untracked_failures_preserve_existing_history(app, prepared, monkeypatch, failure):
assert (await app.run_job("home_evaluate")).success
manager = history(app)
records, observations = manager.list_results(), manager.sampling_state()
paths = [prepared / name for name in ("evaluations/history.json", "sampling/state.json")]
caches = [path.read_bytes() for path in paths]
if failure == "infra":
monkeypatch.setattr(FakeFactory, "bad_export", True)
else:
original = FakeFactory.create_cases
async def fail(factory, *args, **kwargs):
await original(factory, *args, **kwargs)
raise ConnectionError("pool failed after first initialized sandbox")
monkeypatch.setattr(FakeFactory, "create_cases", fail)
response = await app.run_job("home_evaluate", record_history=False)
assert not response.success
assert [path.read_bytes() for path in paths] == caches
assert manager.rebuild() == records and manager.sampling_state() == observations
assert read_state(prepared)["round"] == read_state(prepared)["successful_round"] == 1
@pytest.mark.asyncio
async def test_history_job_defaults_and_agent_injection_are_enforced(app, prepared):
job = app.context.jobs["home_evaluate"]
assert job.parameters["properties"]["record_history"]["default"] is True
job.kwargs["record_history"] = False
standalone = await app.run_job("home_evaluate")
assert standalone.success and not history(app).list_results()
baseline = await app.run_job("home_evaluate", sampling=False, record_history=True)
assert baseline.success and baseline.answer["case_count"] == 2
tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"record_history": True})
assert "record_history" not in tool.input_schema["properties"]
await tool.call()
assert len(history(app).list_results()) == 2
with pytest.raises(ValueError, match="injected"):
await tool.call(record_history=False)
@pytest.mark.asyncio
async def test_legacy_cache_rebuild_and_explicit_null_query_budget(app, prepared):
evaluator = app.context.components["home.evaluation"]["default"]
evaluator.sampling_policy = SamplingPolicy(max_cases=2, max_queries=1)
result = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_queries": None})
assert result.success and result.answer["scored_query_count"] == 2
assert manifest(result)["options"]["sampling_policy"]["max_queries"] is None
assert evaluator.sampling_policy.max_queries == 1
manager = history(app)
observations = manager.sampling_state()
legacy = read_state(prepared)
legacy["schema_version"] = 1
legacy.pop("successful_round")
legacy["cases"]["1"]["ewma_loss"] = 999
(prepared / "sampling/state.json").write_text(json.dumps(legacy))
assert manager.sampling_state() == observations
manager.rebuild()
assert read_state(prepared)["schema_version"] == 2
assert read_state(prepared)["successful_round"] == 1
@pytest.mark.asyncio
async def test_cancellation_after_durable_completion_keeps_history_consistent(app, prepared, monkeypatch):
import asyncio
from threading import Event
manager = history(app)
original = manager.record
entered, release = Event(), Event()
def blocked(run):
entered.set()
assert release.wait(5)
return original(run)
monkeypatch.setattr(manager, "record", blocked)
task = asyncio.create_task(app.run_job("home_evaluate"))
assert await asyncio.to_thread(entered.wait, 5)
task.cancel()
release.set()
with pytest.raises(asyncio.CancelledError):
await task
records = manager.list_results()
assert len(records) == 1
run = prepared / "evaluations" / records[0].run_key
completed = json.loads((run / "manifest.json").read_text())
assert completed["status"] == "completed" and completed["success"] is True
assert manager.rebuild() == records
@pytest.mark.asyncio
async def test_registration_failure_invalidates_only_caches(app, prepared, monkeypatch):
manager = history(app)
original = manager._index
def fail():
raise OSError("history index could not be loaded")
monkeypatch.setattr(manager, "_index", fail)
response = await app.run_job("home_evaluate")
assert not response.success
runs = list((prepared / "evaluations").glob("*/*/*/manifest.json"))
assert len(runs) == 1
run = runs[0].parent
failed = json.loads(runs[0].read_text())
assert failed["status"] == "failed" and failed["success"] is False
assert (run / "summary.json").is_file() and (run / "cases/1/queries/result.json").is_file()
assert not (prepared / "sampling/state.json").exists()
assert read_state(prepared)["round"] == 1 and read_state(prepared)["successful_round"] == 0
monkeypatch.setattr(manager, "_index", original)
assert manager.list_results() == [] and not manager.sampling_state()["cases"]

View file

@ -0,0 +1,312 @@
"""Read-only result Jobs against real Home applications and isolated Git history."""
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
import asyncio
from pathlib import Path
from threading import Event
import pytest
from reme_home.evaluation import inspection_step as inspection
from reme_home.evaluation.artifacts import write_json
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
home_application,
preparation_inputs,
prepared_home,
)
from test_preparation import command
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
async def call(app, name, **kwargs):
response = await app.run_job(name, **kwargs)
assert response.success, response.answer
return response.answer
def snapshot(home):
return {path.relative_to(home): path.read_bytes() for path in home.rglob("*") if path.is_file()}
def write_run(home, run_id="1", *, branch="candidate", commit="a" * 40, **updates):
run = home / "evaluations" / branch / commit[:7] / run_id
manifest = {
"branch_name": branch,
"commit_sha": commit,
"validation_id": run_id,
"case_ids": ["case~a", "missing-case"],
"query_ids": {"case~a": ["q1", "q2"], "missing-case": ["q3"]},
"mode": "scored",
"dataset_kind": "training-set",
"status": "failed",
"started_at": f"2026-01-01T00:00:{int(run_id):02d}+00:00",
"options": {"record_history": False},
**updates,
}
write_json(run / "manifest.json", manifest)
return run
@pytest.mark.asyncio
async def test_real_evaluation_catalog_and_case_query_navigation_are_read_only(app, prepared):
scored = await call(app, "home_evaluate", record_history=False)
debug = await call(app, "home_evaluate", debug=True)
before = snapshot(prepared)
page = await call(app, "home_list_validation_results", limit=1)
assert page["row_count"] == 2 and page["next_offset"] == 1
assert page["rows"][0]["mode"] == "debug" and page["rows"][0]["record_history"] is False
last = await call(app, "home_list_validation_results", offset=1, limit=1)
assert last["next_offset"] is None and last["rows"][0]["validation_path"] == scored["result_dir"]
assert last["rows"][0]["totals"]["scored_query_count"] == 2
run = Path(scored["result_dir"])
result = await call(app, "home_inspect_validation", validation_path=run.relative_to(prepared).as_posix(), limit=1)
assert result["row_count"] == 2 and result["next_offset"] == 1
case = result["rows"][0]
assert case["mean_score"] == 0.5 and case["pointer"].startswith("/cases/")
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id=case["case_id"])
query = result["rows"][0]
assert query["score"] == 0.5 and query["pointer"] == "/queries/0"
assert "fields" not in query and result["construction"]["failed_jobs"]["rows"] == []
result = await call(
app,
"home_inspect_validation",
validation_path=str(run),
case_id=case["case_id"],
query_id=query["query_id"],
)
fields = result["rows"][0]["fields"]
assert fields["answer"] == "hello" and fields["score"] == 0.5 and "judge_result" in fields
assert result["query_missing"] is False and result["answer_log"].endswith("answer.log")
assert snapshot(prepared) == before
assert app.context.components["home.evaluation_history"]["default"].list_results() == []
assert debug["result_dir"] != scored["result_dir"]
@pytest.mark.asyncio
async def test_partial_missing_and_corrupt_runs_are_visible_without_rebuilding(app, prepared):
run = write_run(prepared)
write_json(
run / "partial_summary.json",
{
"case_count": 1,
"cases": {
"case~a": {
"mean_score": 0.0,
"construction": {"status": "partial_success"},
"construction_error_count": 2,
},
},
},
)
write_json(run / "failure.json", {"error": "failure" * 300})
missing = write_run(prepared, "2", mode="debug", status="running")
corrupt = write_run(prepared, "3")
(corrupt / "manifest.json").write_text("invalid JSON")
before = snapshot(prepared)
result = await call(app, "home_list_validation_results")
assert result["row_count"] == 3
assert sum("read_error" in row for row in result["rows"]) == 1
row = next(row for row in result["rows"] if row["validation_path"] == str(run))
assert row["summary_path"].endswith("partial_summary.json") and "truncated" in row["failure"]
result = await call(app, "home_inspect_validation", validation_path=str(run))
assert result["rows"][0]["case_id"] == "missing-case" and result["rows"][0]["mean_score"] is None
assert result["rows"][1]["pointer"] == "/cases/case~0a"
result = await call(app, "home_inspect_validation", validation_path=str(missing), case_id="case~a", query_id="q1")
assert result["query_missing"] and result["rows"][0]["score"] is None
assert len(result["missing_files"]) == 2
result = await call(app, "home_inspect_validation", validation_path=str(missing))
assert result["summary_missing"] and result["rows_source"] is None and result["row_count"] == 2
assert snapshot(prepared) == before and not (missing / "partial_summary.json").exists()
@pytest.mark.asyncio
async def test_failure_and_query_paging_previews_and_source_coordinates(app, prepared):
run = write_run(prepared)
folder = run / "cases/case~a"
write_json(
folder / "memory_construction/result.json",
{
"success": False,
"jobs": [{"job": f"job-{index}", "result": {"success": False, "error": "x" * 900}} for index in range(3)],
},
)
write_json(
folder / "queries/result.json",
{
"queries": [
{
"query_id": "q2",
"score": 0,
"answer": "a" * 3000,
"judge_result": {"success": True},
"golden_answer": ["rubric"],
"answer_result": None,
},
{"query_id": "q1", "score": None, "error": "e" * 900},
],
},
)
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", limit=1)
assert result["rows"][0]["query_id"] == "q1" and result["rows"][0]["pointer"] == "/queries/1"
assert result["next_offset"] == 1 and "truncated" in result["rows"][0]["error"]
failed_jobs = result["construction"]["failed_jobs"]
assert failed_jobs["row_count"] == 3 and failed_jobs["next_offset"] == 1
assert failed_jobs["rows"][0]["pointer"] == "/jobs/0/result"
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", offset=2, limit=1)
assert result["rows"] == [] and result["construction"]["failed_jobs"]["rows"][0]["job"] == "job-2"
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", query_id="q2")
assert result["rows"][0]["pointer"] == "/queries/0"
assert "truncated" in result["rows"][0]["fields"]["answer"]
assert result["rows"][0]["fields"]["golden_answer"] == ["rubric"]
assert result["rows"][0]["score"] == 0
@pytest.mark.asyncio
async def test_prefix_resolves_on_branch_history_not_only_evaluated_commits(app, prepared):
repo = prepared / "code/repo"
initial = command(repo, "rev-parse", "HEAD")
await call(app, "home_create_candidate_branch", branch_name="candidate")
write_run(prepared, commit=initial)
write_run(prepared, branch="other", commit=initial)
result = await call(app, "home_list_validation_results", commit_sha=initial[:7].upper())
assert result["branch_name"] == "candidate" and result["commit_sha"] == initial
assert result["row_count"] == 1
# An unevaluated reachable commit can make a short prefix ambiguous.
tree = command(repo, "rev-parse", "HEAD^{tree}")
for index in range(500):
sha = command(
repo,
"-c",
"user.name=Home",
"-c",
"user.email=home@local",
"commit-tree",
tree,
"-p",
initial,
"-m",
f"unevaluated {index}",
)
if sha[0] == initial[0]:
break
else:
pytest.fail("Could not construct a prefix collision")
command(repo, "update-ref", "refs/heads/candidate", sha)
response = await app.run_job("home_list_validation_results", commit_sha=initial[0])
assert not response.success and "2 matches" in response.answer
# The very same prefix remains unique on init, despite the other Git object.
result = await call(app, "home_list_validation_results", branch_name="init", commit_sha=initial[0])
assert result["commit_sha"] == initial and result["row_count"] == 0
result = await call(app, "home_list_validation_results", commit_sha=sha)
assert result["commit_sha"] == sha and result["rows"] == []
absent = "f" * len(initial)
assert not (await app.run_job("home_list_validation_results", commit_sha=absent)).success
@pytest.mark.asyncio
async def test_branch_filters_and_deleted_historical_branch_do_not_require_git(app, prepared):
write_run(prepared, branch="removed")
write_run(prepared, "2", branch="candidate")
result = await call(app, "home_list_validation_results", branch_name="removed")
assert result["row_count"] == 1 and result["rows"][0]["branch_name"] == "removed"
response = await app.run_job("home_list_validation_results", branch_name="removed", commit_sha="a")
assert not response.success
@pytest.mark.asyncio
@pytest.mark.parametrize("corruption", ["summary", "options"])
async def test_commit_filtered_catalog_reports_corrupt_run_with_identity(app, prepared, corruption):
sha = command(prepared / "code/repo", "rev-parse", "HEAD")
run = write_run(prepared, branch="init", commit=sha, options=None if corruption == "options" else {})
if corruption == "summary":
write_json(run / "summary.json", [])
result = await call(app, "home_list_validation_results", commit_sha=sha[:7])
assert result["row_count"] == 1
assert result["rows"][0]["commit_sha"] == sha and "read_error" in result["rows"][0]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"job,kwargs",
[
("home_list_validation_results", {"offset": -1}),
("home_list_validation_results", {"limit": 0}),
("home_list_validation_results", {"limit": 101}),
("home_list_validation_results", {"limit": True}),
("home_list_validation_results", {"offset": "1"}),
("home_list_validation_results", {"commit_sha": "HEAD"}),
("home_list_validation_results", {"commit_sha": ""}),
("home_list_validation_results", {"branch_name": "../init"}),
("home_inspect_validation", {}),
("home_inspect_validation", {"validation_path": "evaluations"}),
("home_inspect_validation", {"validation_path": "../outside"}),
("home_inspect_validation", {"validation_path": "/tmp/outside"}),
("home_inspect_validation", {"validation_path": "evaluations/a/b/c", "query_id": "q"}),
("home_inspect_validation", {"validation_path": "evaluations/a/b/c", "case_id": "a/b"}),
("home_list_validation_results", {"_allowed_paths": []}),
],
)
async def test_invalid_input_fails_closed(app, prepared, job, kwargs):
assert not (await app.run_job(job, **kwargs)).success
@pytest.mark.asyncio
async def test_unknown_ids_and_symlink_artifacts_are_rejected(app, prepared, tmp_path):
run = write_run(prepared)
assert not (await app.run_job("home_inspect_validation", validation_path=str(run), case_id="other")).success
assert not (
await app.run_job(
"home_inspect_validation",
validation_path=str(run),
case_id="case~a",
query_id="other",
)
).success
secret = tmp_path / "secret.json"
secret.write_text('{"secret": "must not be read"}')
(run / "summary.json").symlink_to(secret)
response = await app.run_job("home_inspect_validation", validation_path=str(run))
assert not response.success and "must not be read" not in response.answer
result = await call(app, "home_list_validation_results")
assert "read_error" in result["rows"][0] and "secret" not in str(result["rows"])
(run / "summary.json").unlink()
(run / "cases").symlink_to(tmp_path, target_is_directory=True)
assert not (await app.run_job("home_inspect_validation", validation_path=str(run), case_id="case~a")).success
@pytest.mark.asyncio
async def test_hidden_case_scope_is_enforced_by_agent_job_injection(app, prepared):
run = write_run(prepared)
job = app.context.jobs["home_inspect_validation"]
tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"validation_path": str(run), "case_id": "case~a"})
assert "validation_path" not in tool.input_schema["properties"] and "case_id" not in tool.input_schema["properties"]
await tool.call(query_id="q1")
with pytest.raises(ValueError, match="injected"):
await tool.call(case_id="missing-case")
with pytest.raises(ValueError, match="injected"):
await tool.call(validation_path="evaluations/other/commit/1")
@pytest.mark.asyncio
async def test_cancelled_job_drains_read_thread(app, prepared, monkeypatch):
entered, release, finished = Event(), Event(), Event()
def read(*args):
entered.set()
assert release.wait(5)
finished.set()
return {}
monkeypatch.setattr(inspection, "_list_results", read)
task = asyncio.create_task(app.run_job("home_list_validation_results"))
assert await asyncio.to_thread(entered.wait, 5)
task.cancel()
await asyncio.sleep(0)
assert not task.done()
release.set()
with pytest.raises(asyncio.CancelledError):
await task
assert finished.is_set()

View file

@ -0,0 +1,230 @@
"""Memory evidence Jobs against isolated real Git repositories."""
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
import shutil
import pytest
from pydantic import ValidationError
from reme_home.evaluation import memory_inspection_step as memory
from reme_home.preparation.code import git
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
home_application,
preparation_inputs,
prepared_home,
)
from test_evaluation_inspection import call, snapshot, write_run
@pytest.fixture
def evidence(tmp_path):
home = tmp_path / "home"
run = write_run(home, case_ids=["case1"])
repo = run / "cases/case1/memory_construction/reme_workspace"
repo.mkdir(parents=True)
git(repo, "init")
git(repo, "config", "user.name", "test")
git(repo, "config", "user.email", "test@localhost")
(repo / "memory").mkdir()
(repo / "memory/note.md").write_text("old fact\n")
(repo / "removed.txt").write_text("removed\n")
first = commit(repo, "session: session1")
(repo / "memory/note.md").write_text("new fact\n")
(repo / "removed.txt").unlink()
(repo / "added.bin").write_bytes(b"\x00\xff")
second = commit(repo, "session: session2")
return home, run, repo, first, second
def commit(repo, subject):
git(repo, "add", "--all")
git(repo, "commit", "-m", subject)
return git(repo, "rev-parse", "HEAD")
def history(run, **kwargs):
return memory.MemoryHistoryInput(validation_path=str(run), case_id="case1", **kwargs)
def comparison(run, **kwargs):
return memory.CompareMemoryInput(validation_path=str(run), case_id="case1", **kwargs)
def test_history_paging_filter_and_no_writes(evidence):
home, run, repo, first, second = evidence
(repo / "memory/note.md").write_text("dirty checkout ignored\n")
before = snapshot(home)
result = memory.inspect_history(home, history(run, limit=1))
assert result["head"] == second and result["row_count"] == 2
assert result["rows"][0]["session_id"] == "session2" and result["next_offset"] == 1
result = memory.inspect_history(home, history(run, offset=1, limit=1))
assert result["rows"][0]["commit_sha"] == first and result["next_offset"] is None
assert memory.inspect_history(home, history(run, path="added.bin"))["row_count"] == 1
assert memory.inspect_history(home, history(run, path="memory/*"))["rows"] == []
assert snapshot(home) == before
def test_same_repository_diff_and_unique_prefix(evidence):
home, run, repo, first, second = evidence
(repo / "memory/note.md").write_text("dirty checkout ignored\n")
before = snapshot(home)
result = memory.compare_memory(home, comparison(run, base_ref=first[:7].upper()))
assert result["base_commit"] == first and result["target_commit"] == second
assert result["added"] == ["added.bin"] and result["removed"] == ["removed.txt"]
assert result["modified"] == ["memory/note.md"]
assert "-old fact" in result["diff"] and "+new fact" in result["diff"]
assert "Binary files differ" in result["diff"] and not result["truncated"]
assert "dirty checkout" not in result["diff"]
assert memory.compare_memory(home, comparison(run, base_ref="HEAD~1", path="memory"))["added"] == []
assert snapshot(home) == before
def test_cross_evaluation_final_snapshots(evidence):
home, run, _repo, _first, second = evidence
other = write_run(home, "2", case_ids=["case1"], commit="b" * 40)
target = other / "cases/case1/memory_construction/reme_workspace"
target.mkdir(parents=True)
git(target, "init")
git(target, "config", "user.name", "test")
git(target, "config", "user.email", "test@localhost")
(target / "target.txt").write_text("target\n")
target_head = commit(target, "new history")
before = snapshot(home)
result = memory.compare_memory(home, comparison(run, target_validation_path=str(other)))
assert result["base_commit"] == second and result["target_commit"] == target_head
assert result["added"] == ["target.txt"]
assert result["removed"] == ["added.bin", "memory/note.md"]
assert result["base_tree_sha256"] != result["target_tree_sha256"]
assert snapshot(home) == before
@pytest.mark.parametrize("state", ["workspace_missing", "history_missing", "history_empty"])
def test_missing_and_empty_history(tmp_path, state):
home = tmp_path / "home"
run = write_run(home, case_ids=["case1"])
repo = run / "cases/case1/memory_construction/reme_workspace"
if state != "workspace_missing":
repo.mkdir(parents=True)
if state == "history_empty":
git(repo, "init")
result = memory.inspect_history(home, history(run))
assert result["status"] == state and result["rows"] == [] and not result["has_git_history"]
with pytest.raises(ValueError, match="requires memory commits"):
memory.compare_memory(home, comparison(run, base_ref="HEAD"))
@pytest.mark.parametrize(
"kwargs",
[
{"offset": -1},
{"limit": 101},
{"limit": True},
{"case_id": "../bad"},
{"path": "../escape"},
{"path": ".git/config"},
],
)
def test_input_validation(kwargs):
with pytest.raises(ValidationError):
memory.MemoryHistoryInput.model_validate({"validation_path": "a", "case_id": "case1", **kwargs})
def test_comparison_requires_base():
with pytest.raises(ValidationError, match="base_ref is required"):
comparison("a")
@pytest.mark.parametrize("kind", ["gitfile", "symlink", "alternates", "corrupt"])
def test_invalid_or_external_git_storage(evidence, tmp_path, kind):
home, run, repo, _first, _second = evidence
if kind == "gitfile":
(repo / ".git").rename(repo / "saved-git")
(repo / ".git").write_text("gitdir: saved-git\n")
elif kind == "symlink":
(repo / ".git/objects/external").symlink_to(tmp_path)
elif kind == "alternates":
(repo / ".git/objects/info/alternates").write_text(str(tmp_path))
else:
(repo / ".git/HEAD").write_text("broken\n")
with pytest.raises((ValueError, memory.subprocess.CalledProcessError)):
memory.inspect_history(home, history(run))
def test_scope_and_limits(evidence, monkeypatch):
home, run, _repo, first, _second = evidence
with pytest.raises(ValueError, match="case_id"):
memory.inspect_history(home, memory.MemoryHistoryInput(validation_path=str(run), case_id="other"))
with pytest.raises(ValueError):
memory.inspect_history(home, history(home))
monkeypatch.setattr(memory, "MAX_OUTPUT", 20)
result = memory.compare_memory(home, comparison(run, base_ref=first))
assert result["truncated"] and len(result["diff"]) <= 20
monkeypatch.setattr(memory, "MAX_TREE_FILES", 1)
with pytest.raises(ValueError, match="2000 files"):
memory.compare_memory(home, comparison(run, base_ref=first))
def test_ambiguous_and_unreachable_revisions(evidence):
home, run, repo, first, second = evidence
# Publish real commits until two reachable SHAs share a one-character prefix.
previous = second
seen = {first[0]: first, second[0]: second}
for index in range(100):
sha = git(repo, "commit-tree", "HEAD^{tree}", "-p", previous, "-m", str(index))
previous = sha
if sha[0] in seen:
prefix = sha[0]
break
seen[sha[0]] = sha
git(repo, "update-ref", "HEAD", previous)
with pytest.raises(ValueError, match="matches reachable"):
memory.compare_memory(home, comparison(run, base_ref=prefix))
with pytest.raises(ValueError, match="0 matches"):
memory.compare_memory(home, comparison(run, base_ref="f" * 40))
detached = git(repo, "commit-tree", "HEAD^{tree}", "-m", "unreachable")
with pytest.raises(memory.subprocess.CalledProcessError):
memory.compare_memory(home, comparison(run, base_ref=detached + "~0"))
@pytest.mark.asyncio
async def test_registered_jobs_and_restricted_scope(app, prepared, evidence):
run = write_run(prepared, case_ids=["case1"])
result = await call(app, "home_inspect_memory_history", validation_path=str(run), case_id="case1")
assert result["status"] == "workspace_missing"
response = await app.run_job("home_compare_memory", validation_path=str(run), case_id="case1")
assert not response.success and "base_ref" in str(response.answer)
response = await app.run_job(
"home_inspect_memory_history",
validation_path=str(run),
case_id="case1",
_allowed_paths=[str(run)],
)
assert not response.success and "unrestricted" in str(response.answer)
_home, _run, repository, first, second = evidence
destination = run / "cases/case1/memory_construction/reme_workspace"
shutil.copytree(repository, destination)
result = await call(app, "home_compare_memory", validation_path=str(run), case_id="case1", base_ref=first)
assert result["target_commit"] == second and result["modified"] == ["memory/note.md"]
def test_unterminated_lines_and_snapshot_byte_limit(evidence, monkeypatch):
home, run, repo, _first, second = evidence
(repo / "memory/note.md").write_text("unterminated")
commit(repo, "ordinary checkpoint")
result = memory.compare_memory(home, comparison(run, base_ref=second, path="memory"))
assert "+unterminated\n\\ No newline at end of file\n" in result["diff"]
assert memory.inspect_history(home, history(run))["rows"][0]["session_id"] is None
monkeypatch.setattr(memory, "MAX_TREE_BYTES", 1)
with pytest.raises(ValueError, match="20 MB"):
memory.compare_memory(home, comparison(run, base_ref=second))
def test_tracked_symlinks_are_not_followed(evidence):
home, run, repo, _first, second = evidence
(repo / "memory/link").symlink_to("/etc/passwd")
commit(repo, "link")
with pytest.raises(ValueError, match="regular files"):
memory.compare_memory(home, comparison(run, base_ref=second))

View file

@ -0,0 +1,412 @@
"""Adaptive sampling parity and actual-start accounting through real Home Jobs."""
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
import asyncio
import copy
import json
from pathlib import Path
import pytest
from reme_home import sampling
from reme_home.evaluation import history as history_module
from reme_home.preparation.data.models import CaseSpec, QuerySpec
from reme_home.workspace import _initialization_lock
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
home_application,
preparation_inputs,
prepared_home,
)
def cases():
return [
CaseSpec(
case_id=key,
sessions=[],
queries=[
QuerySpec(query_id=f"query-{key}-{index}", question="?", golden_answer="answer") for index in range(4)
],
)
for key in "abc"
]
def test_plan_matches_dev_meta_golden_draw(monkeypatch):
# Golden output computed with dev_meta b53048e4 sampling.py, seed=123456.
monkeypatch.setattr(sampling.time, "time_ns", lambda: 123456)
state = {
"round": 2,
"cases": {
"a": {
"observed_count": 2,
"ewma_loss": 0.9,
"last_selected_round": 2,
"last_error_round": 1,
},
},
"queries": {},
}
before = copy.deepcopy(state)
policy = sampling.SamplingPolicy(max_cases=2, max_queries=3, temperature=0.8)
plan = sampling.create_sampling_plan(cases(), state, policy)
assert plan["case_ids"] == ["a", "b"]
assert plan["query_ids"] == {"a": ["query-a-2", "query-a-0"], "b": ["query-b-3"]}
assert plan["case_draw_weights"] == pytest.approx(
{
"a": 0.40033106216673214,
"b": 0.29983446891663396,
"c": 0.29983446891663396,
},
)
assert plan["query_draw_weights"]["a"] == {f"query-a-{index}": 0.25 for index in range(4)}
assert state == before
assert sampling.create_sampling_plan(cases(), state, policy) == plan
def test_replay_eligibility_and_unrestricted_next_draw(monkeypatch):
monkeypatch.setattr(sampling.time, "time_ns", lambda: 100)
policy = sampling.SamplingPolicy(max_cases=3, max_queries=3)
eligible = {"b": ["query-b-1", "query-b-3"], "c": ["query-c-2"]}
plan = sampling.create_sampling_plan(cases(), {}, policy, eligible_query_ids=eligible)
assert set(plan["case_ids"]) == {"b", "c"}
assert all(set(values) <= set(eligible[key]) for key, values in plan["query_ids"].items())
full = sampling.create_sampling_plan(cases(), {}, policy)
assert set(full["case_draw_weights"]) == {"a", "b", "c"}
assert full["eligibility"] is None
@pytest.mark.parametrize(
"options",
[
{"max_cases": 0},
{"max_queries": 0},
{"max_cases": True},
{"max_queries": True},
{"temperature": 0},
{"temperature": float("nan")},
{"temperature": float("inf")},
{"ewma_alpha": 0},
{"ewma_alpha": 1.1},
{"uniform_exploration": -0.1},
{"uniform_exploration": 1.1},
{"unexpected": 1},
],
)
def test_policy_rejects_invalid_limits(options):
with pytest.raises(ValueError):
sampling.SamplingPolicy(**options)
def test_uniform_mass_and_unobserved_priority():
records = {
"easy": {"ewma_loss": 0.0, "observed_count": 100, "last_selected_round": 10},
"hard": {"ewma_loss": 1.0, "observed_count": 1, "last_selected_round": 1, "last_error_round": 1},
"unseen": {},
}
weights = sampling._mixed_weights(records, 11, sampling.SamplingPolicy(uniform_exploration=0.3))
assert sum(weights.values()) == pytest.approx(1)
assert all(weight >= 0.1 for weight in weights.values())
assert weights["hard"] > weights["easy"] and weights["unseen"] > weights["easy"]
uniform = sampling._mixed_weights(records, 11, sampling.SamplingPolicy(uniform_exploration=1))
assert set(uniform.values()) == {1 / 3}
def state(home):
return history_module.read_state(home)
@pytest.mark.asyncio
async def test_baseline_then_adaptive_job_updates_only_selected_cases(app, prepared):
baseline = await app.run_job("home_evaluate")
assert baseline.success, baseline.answer
assert state(prepared)["round"] == 1
assert all(record["ewma_loss"] == 0.5 for record in state(prepared)["cases"].values())
evaluation = app.context.components["home.evaluation"]["default"]
evaluation.sampling_policy = sampling.SamplingPolicy(max_cases=1, max_queries=1, ewma_alpha=0.5)
app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": '{"score": 1}'}
before = copy.deepcopy(state(prepared))
response = await app.run_job("home_evaluate", sampling=True)
assert response.success, response.answer
run = Path(response.answer["result_dir"])
manifest = json.loads((run / "manifest.json").read_text())
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
selected = manifest["case_ids"][0]
unselected = ({"1", "2"} - {selected}).pop()
assert manifest["case_ids"] == plan["case_ids"] and manifest["query_ids"] == plan["query_ids"]
assert plan["mode"] == "sampled" and plan["round"] == 2
assert response.answer["case_count"] == response.answer["scored_query_count"] == 1
after = state(prepared)
assert after["cases"][selected]["ewma_loss"] == 0.25
assert after["cases"][selected]["observed_count"] == 2
assert after["cases"][selected]["successful_observations_after_last_error"] == 1
assert after["cases"][unselected] == before["cases"][unselected]
evaluation._finish_sampling(run)
assert state(prepared) == after # A summary is consumed once, including after reopening.
assert "home_validate" not in app.context.jobs
assert app.context.jobs["home_evaluate"].parameters["properties"]["sampling"]["type"] == "boolean"
assert "sampling_policy" not in app.context.jobs["home_evaluate"].parameters["properties"]
assert "case_ids" not in app.context.jobs["home_evaluate"].parameters["properties"]
@pytest.mark.asyncio
async def test_debug_never_reads_or_changes_sampling_state(app, prepared):
before = prepared / "sampling/state.json"
for enabled in (False, True):
response = await app.run_job("home_evaluate", sampling=enabled, debug=True)
assert response.success, response.answer
assert not before.exists()
assert not list((prepared / "sampling/plans").rglob("*.json"))
# Corrupt sampling state cannot prevent an unrelated debug smoke test.
before.write_text("corrupt cached state")
response = await app.run_job("home_evaluate", sampling=True, debug=True)
assert response.success and before.read_text() == "corrupt cached state"
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
assert "sandbox_started_at" not in manifest and "sampling" not in manifest
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["dirty", "factory_constructor", "startup", "selection", "replay"])
async def test_prestart_failure_never_counts(app, prepared, monkeypatch, failure):
await app.run_job("home_evaluate")
previous = (prepared / "sampling/state.json").read_bytes()
plans = sorted((prepared / "sampling/plans").rglob("*.json"))
options = {}
if failure == "dirty":
(prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py").write_text("dirty")
elif failure == "selection":
options["case_ids"] = ["1"]
elif failure == "replay":
options["reuse_construction_from"] = "evaluations/not-a-run"
elif failure == "factory_constructor":
def fail(*args, **kwargs):
raise RuntimeError("factory never created")
monkeypatch.setattr("reme_home.evaluation.component.DockerReMeSandboxFactory", fail)
else:
async def fail(*args, **kwargs):
raise RuntimeError("no sandbox initialized")
monkeypatch.setattr(FakeFactory, "create_cases", fail)
response = await app.run_job("home_evaluate", sampling=True, **options)
assert not response.success
assert (prepared / "sampling/state.json").read_bytes() == previous
assert sorted((prepared / "sampling/plans").rglob("*.json")) == plans
@pytest.mark.asyncio
async def test_started_infrastructure_failure_is_not_a_quality_zero(app, prepared, monkeypatch):
async def fail(*args, **kwargs):
raise ConnectionError("construction provider unavailable")
monkeypatch.setattr("test_evaluation.FakeWorker.run_build", fail)
response = await app.run_job("home_evaluate", case_ids=["1"])
assert not response.success
observed = state(prepared)
assert observed["round"] == 1
assert not observed["cases"] and not observed["queries"]
assert app.context.components["home.evaluation_history"]["default"].list_results() == []
@pytest.mark.asyncio
async def test_started_candidate_failure_does_not_fabricate_a_score(app, prepared, monkeypatch):
factory = FakeFactory.create_cases
async def create(self, *args, **kwargs):
workers = await factory(self, *args, **kwargs)
async def empty(_queries):
return {"success": True, "queries": [{"query_id": "fact:1", "answer": "", "error": None}]}
for worker in workers:
worker.run_queries = empty
return workers
monkeypatch.setattr(FakeFactory, "create_cases", create)
response = await app.run_job("home_evaluate", case_ids=["1"])
assert response.success, response.answer
observed = state(prepared)
assert observed["round"] == 1
record = observed["cases"]["1"]
assert record["candidate_failure_count"] == 1
assert record["observed_count"] == 0 and record["ewma_loss"] is None
query = observed["queries"][sampling._query_key("1", "fact:1")]
assert query["candidate_failure_count"] == 1 and query["observed_count"] == 0
@pytest.mark.asyncio
async def test_started_cancellation_retains_round_but_no_partial_quality(app, prepared, monkeypatch):
monkeypatch.setattr(FakeFactory, "block", asyncio.Event())
task = asyncio.create_task(app.run_job("home_evaluate", sampling=True, concurrency=1))
for _ in range(200):
if FakeFactory.instances and FakeFactory.instances[0].entered.is_set():
break
await asyncio.sleep(0.01)
assert FakeFactory.instances[0].entered.is_set()
assert state(prepared)["round"] == 1
concurrent = await app.run_job("home_evaluate", sampling=True)
assert not concurrent.success # Formal validations follow dev_meta's serialization boundary.
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
observed = state(prepared)
assert observed["round"] == 1 and not observed["cases"] and not observed["queries"]
assert not observed["processed_validations"]
with _initialization_lock(prepared / "metadata/validation.lock"):
pass
@pytest.mark.asyncio
async def test_replay_sampling_is_limited_to_donor_cases(app, prepared):
baseline = await app.run_job("home_evaluate", case_ids=["1"])
assert baseline.success, baseline.answer
app.context.components["home.evaluation"]["default"].sampling_policy = sampling.SamplingPolicy(max_cases=1)
response = await app.run_job("home_evaluate", sampling=True, reuse_construction_from=baseline.answer["result_dir"])
assert response.success, response.answer
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
assert manifest["case_ids"] == ["1"]
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
assert plan["eligibility"] == {"1": ["fact:1"]}
# The next full rebuild must see the entire training set again.
response = await app.run_job("home_evaluate", sampling=True)
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
assert set(plan["case_draw_weights"]) == {"1", "2"} and plan["eligibility"] is None
@pytest.mark.asyncio
async def test_sampling_cache_rebuild_and_new_round_after_reopen(app, prepared):
baseline = await app.run_job("home_evaluate")
assert baseline.success
expected = state(prepared)
(prepared / "sampling/state.json").unlink()
assert state(prepared) == expected
assert not (prepared / "sampling/state.json").exists() # Rebuilding reads never claim a new round.
with _initialization_lock(prepared / "metadata/workspace.lock"):
assert history_module.rebuild_sampling_state(prepared) == expected
result = await app.run_job("home_evaluate", sampling=True)
assert result.success, result.answer
assert state(prepared)["round"] == 2
@pytest.mark.asyncio
async def test_fail_fast_started_round_without_summary_is_not_quality_feedback(app, prepared):
app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": "invalid judge output"}
response = await app.run_job("home_evaluate", sampling=True, fail_fast=True)
assert not response.success
observed = state(prepared)
assert observed["round"] == 1
assert not observed["processed_validations"] and not observed["cases"]
assert not observed["queries"]
(prepared / "sampling/state.json").unlink()
assert state(prepared) == observed
@pytest.mark.asyncio
async def test_partial_pool_start_failure_counts_started_round(app, prepared, monkeypatch):
original = FakeFactory.create_cases
async def fail_after_first_start(factory, *args, **kwargs):
await original(factory, *args, **kwargs)
raise ConnectionError("a later worker failed after first initialized")
monkeypatch.setattr(FakeFactory, "create_cases", fail_after_first_start)
result = await app.run_job("home_evaluate", sampling=True)
assert not result.success
observed = state(prepared)
assert observed["round"] == 1 and not observed["processed_validations"]
assert not observed["cases"] and not observed["queries"]
@pytest.mark.asyncio
async def test_started_callback_is_idempotent(app, prepared):
response = await app.run_job("home_evaluate", sampling=True)
assert response.success
run = Path(response.answer["result_dir"])
manifest = json.loads((run / "manifest.json").read_text())
original = state(prepared)
component = app.context.components["home.evaluation"]["default"]
component._sampling_started(run, manifest, None)
assert state(prepared) == original
@pytest.mark.parametrize("missing", [1, 2])
def test_replay_eligibility_uses_dev_meta_five_percent_threshold(tmp_path, missing):
from reme_home.evaluation.inputs import EvaluationSpec, replay_sampling_eligibility
from reme_home.evaluation.artifacts import write_json
home = tmp_path / "home"
source = home / "evaluations/init/commit/1"
selected = [
CaseSpec(
case_id=f"case-{index}",
sessions=[],
queries=[
QuerySpec(query_id="q", question="?", golden_answer="answer"),
],
)
for index in range(20)
]
write_json(
source / "manifest.json",
{
"dataset_kind": "training-set",
"mode": "scored",
"status": "completed",
"case_ids": [case.case_id for case in selected],
},
)
write_json(source / "summary.json", {})
for case in selected[missing:]:
folder = source / "cases" / case.case_id
write_json(folder / "memory_construction/result.json", {"success": True})
(folder / "memory_construction/reme_workspace.tar.gz").write_bytes(b"fixture")
write_json(folder / "queries/result.json", {"queries": [{"answer": "answer", "error": None}]})
spec = EvaluationSpec(reuse_construction_from=str(source))
if missing == 1:
eligible = replay_sampling_eligibility(home, spec, selected)
assert len(eligible) == 19 and "case-0" not in eligible
else:
with pytest.raises(ValueError, match="5%"):
replay_sampling_eligibility(home, spec, selected)
@pytest.mark.asyncio
async def test_query_budget_and_draw_order_reach_actual_execution(app, inputs, monkeypatch):
from unittest.mock import AsyncMock
source = Path(inputs[0]["benchmark"]["source"])
for path in source.glob("chats/*/*/probing_questions/probing_questions.json"):
payload = json.loads(path.read_text())
payload["fact"] = [
dict(payload["fact"][0], question=f"question {index}", rubric=["says hello"]) for index in range(4)
]
path.write_text(json.dumps(payload))
prepared = await app.run_job("home_prepare", spec=inputs[0])
assert prepared.success
monkeypatch.setattr("reme_home.evaluation.component.DockerReMeSandboxFactory", FakeFactory)
monkeypatch.setattr(
app.context.components["agent_wrapper"]["judge"],
"reply",
AsyncMock(return_value={"result": '{"score": 0.5}'}),
)
monkeypatch.setattr(sampling.time, "time_ns", lambda: 123456)
component = app.context.components["home.evaluation"]["default"]
component.sampling_policy = sampling.SamplingPolicy(max_cases=2, max_queries=3)
response = await app.run_job("home_evaluate", sampling=True, concurrency=1)
assert response.success, response.answer
run = Path(response.answer["result_dir"])
manifest = json.loads((run / "manifest.json").read_text())
home = Path(app.config.workspace_dir) / "home"
plan = json.loads((home / manifest["sampling"]["plan_path"]).read_text())
assert manifest["query_ids"] == plan["query_ids"]
assert len(manifest["case_ids"]) == 2 and sum(map(len, manifest["query_ids"].values())) == 3
worker = FakeFactory.instances[-1].workers[0]
assert [query.query_id for query in worker.queries] == manifest["query_ids"][worker.case_id]
for case_id, query_ids in manifest["query_ids"].items():
result = json.loads((run / "cases" / case_id / "queries/result.json").read_text())
assert [row["query_id"] for row in result["queries"]] == query_ids
assert response.answer["scored_query_count"] == 3

View file

@ -299,6 +299,11 @@ class AsAgentWrapper(BaseAgentWrapper):
"""Resolve configured skill names to AgentScope local skill directories."""
return [str(path) for path in self._resolve_project_skills(skills).values()]
def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> Agent:
"""Construct the SDK agent while subclasses reuse ReMe's tool and session wiring."""
del options
return Agent(**kwargs)
async def _build_agent(self, inputs: Any, **kwargs) -> tuple[Agent, Any]:
"""Build an Agent instance from kwargs. Returns (agent, processed_inputs)."""
model = self.as_llm.model if self.as_llm else None
@ -327,7 +332,8 @@ class AsAgentWrapper(BaseAgentWrapper):
perm_mode = PermissionMode(kwargs.get("permission_mode", "bypass"))
state = await self._load_state(kwargs, perm_mode)
agent = Agent(
agent = self._create_agent(
options=kwargs,
name=self.name,
system_prompt=system_prompt,
model=model,

View file

@ -0,0 +1,84 @@
"""Observe the first initialized sandbox without counting failed startup attempts."""
import asyncio
from unittest.mock import AsyncMock
import pytest
from deploy.benchmark_docker import DockerReMeSandboxFactory, ImageCandidate
from deploy.benchmark_docker import docker_sandbox
@pytest.mark.asyncio
async def test_start_callback_only_once_after_initialization(monkeypatch):
"""Initialization is the boundary, including replacement workers and partial pool failure."""
initialized, reported = [], []
async def initialize(case):
await asyncio.sleep(0)
initialized.append(case.case_id)
if case.case_id == "bad":
raise RuntimeError("startup failed")
monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", initialize)
workspace = AsyncMock()
factory = DockerReMeSandboxFactory(
ImageCandidate("fixture"),
workspace_builder=lambda *_: workspace,
on_sandbox_started=lambda: reported.append(list(initialized)),
)
with pytest.raises(RuntimeError, match="startup failed"):
await factory.create_case("bad")
assert not reported
await factory.create_case("good")
assert reported == [["bad", "good"]]
await factory.create_case("replacement")
assert len(reported) == 1
@pytest.mark.asyncio
async def test_start_callback_failure_closes_initialized_resource(monkeypatch):
"""Persisting a start marker must not orphan an already-created workspace on failure."""
monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", AsyncMock())
workspace = AsyncMock()
def fail():
raise OSError("start marker unavailable")
factory = DockerReMeSandboxFactory(
ImageCandidate("fixture"),
workspace_builder=lambda *_: workspace,
on_sandbox_started=fail,
)
with pytest.raises(OSError, match="start marker unavailable"):
await factory.create_case("created")
workspace.close.assert_awaited_once()
assert not factory._sandbox_started_reported # pylint: disable=protected-access
@pytest.mark.asyncio
async def test_partial_pool_start_reports_first_success_before_failure(monkeypatch):
"""An initialized worker counts even if a subsequent worker prevents pool startup."""
workspaces, started = [], []
def workspace_builder(*_args):
workspace = AsyncMock()
workspaces.append(workspace)
return workspace
async def initialize(case):
if case.case_id == "bad":
raise ConnectionError("second worker failed")
monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", initialize)
factory = DockerReMeSandboxFactory(
ImageCandidate("fixture"),
workspace_builder=workspace_builder,
on_sandbox_started=lambda: started.append(True),
)
with pytest.raises(ConnectionError, match="second worker failed"):
await factory.create_cases(["good", "bad"], concurrency=1)
assert started == [True]
assert len(workspaces) == 2
for workspace in workspaces:
workspace.close.assert_awaited_once()