mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-16 09:21:50 +00:00
feat(home): migrate agent, sampling and evidence inspection tooling
This commit is contained in:
parent
95f9ea7b61
commit
91b7c14df4
41 changed files with 5428 additions and 20 deletions
|
|
@ -158,6 +158,12 @@ the host application loads `home`. `source_ref: HEAD` exports committed source
|
|||
only, excluding uncommitted modifications. Preparation failure rolls back its
|
||||
outputs; after success, another call fails instead of resetting the experiment.
|
||||
|
||||
The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`,
|
||||
`max_queries` (null means unlimited), `temperature`, `ewma_alpha` and
|
||||
`uniform_exploration`. The runner validates these and passes them to the host
|
||||
`home.evaluation` Component. It still only prepares; it does not run evaluations.
|
||||
Each evaluation controls `sampling` and `record_history` separately.
|
||||
|
||||
Python callers can use the same lifecycle:
|
||||
|
||||
```python
|
||||
|
|
|
|||
|
|
@ -149,4 +149,9 @@ python benchmark/beam/run_home.py
|
|||
准备失败会回滚;成功后禁止再次准备同一工作区。
|
||||
Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。
|
||||
|
||||
顶层 `sampling_policy` 配置采样默认值:`max_cases`、`max_queries`(null 表示不限)、
|
||||
`temperature`、`ewma_alpha` 和 `uniform_exploration`。入口校验后将其传给宿主
|
||||
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
|
||||
和 `record_history`。
|
||||
|
||||
默认选择 BEAM **100K 全部 case**。
|
||||
|
|
|
|||
|
|
@ -1,7 +1,13 @@
|
|||
# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
|
||||
workspace_dir: benchmark/beam/workspaces/home-100K
|
||||
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
|
||||
sampling_policy:
|
||||
max_cases: 5
|
||||
max_queries: null
|
||||
temperature: 1.0
|
||||
ewma_alpha: 0.3
|
||||
uniform_exploration: 0.2
|
||||
preparation:
|
||||
schema_version: 1
|
||||
benchmark:
|
||||
name: beam
|
||||
source: benchmark/beam/dataset/BEAM
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from pydantic import BaseModel, ConfigDict, Field
|
|||
import yaml
|
||||
from reme_home.preparation.spec import PreparationSpec
|
||||
from reme_home.judge import JUDGE_PLUGINS
|
||||
from reme_home.sampling import SamplingPolicy
|
||||
|
||||
from reme.application import Application
|
||||
|
||||
|
|
@ -21,6 +22,7 @@ class RunHomeConfig(BaseModel):
|
|||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
workspace_dir: str = Field(min_length=1)
|
||||
preparation: dict
|
||||
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
|
|
@ -30,7 +32,7 @@ def project_path(value: str, project_root: Path) -> Path:
|
|||
return (project_root / value).resolve()
|
||||
|
||||
|
||||
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
|
||||
def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
|
||||
"""Resolve source paths and validate explicit Home preparation parameters."""
|
||||
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
config = RunHomeConfig.model_validate(raw)
|
||||
|
|
@ -40,15 +42,22 @@ def load_config(config_path: str | Path, *, workspace_dir: str | None = None, pr
|
|||
spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
|
||||
spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
|
||||
workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
|
||||
return workspace, spec.model_dump(mode="json")
|
||||
return workspace, spec.model_dump(mode="json"), config.sampling_policy
|
||||
|
||||
|
||||
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
|
||||
"""Return resolved preparation inputs, preserving the preparation-only caller contract."""
|
||||
workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
|
||||
return workspace, spec
|
||||
|
||||
|
||||
async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
|
||||
"""Start Home, prepare once, and close all resources without evaluating cases."""
|
||||
workspace, spec = load_config(config_path, workspace_dir=workspace_dir)
|
||||
workspace, spec, policy = _load_config(config_path, workspace_dir=workspace_dir)
|
||||
app = Application(
|
||||
workspace_dir=str(workspace),
|
||||
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
|
||||
components={"home.evaluation": {"default": {"sampling_policy": policy.model_dump(mode="json")}}},
|
||||
service={"backend": "cli"},
|
||||
enable_logo=False,
|
||||
log_to_file=False,
|
||||
|
|
|
|||
|
|
@ -131,6 +131,12 @@ the host application loads `home`. `source_ref: HEAD` exports committed source
|
|||
only, excluding uncommitted modifications. Preparation failure rolls back its
|
||||
outputs; after success, another call fails instead of resetting the experiment.
|
||||
|
||||
The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`,
|
||||
`max_queries` (null means unlimited), `temperature`, `ewma_alpha` and
|
||||
`uniform_exploration`. The runner validates these and passes them to the host
|
||||
`home.evaluation` Component. It still only prepares; it does not run evaluations.
|
||||
Each evaluation controls `sampling` and `record_history` separately.
|
||||
|
||||
Python callers can use the same lifecycle:
|
||||
|
||||
```python
|
||||
|
|
|
|||
|
|
@ -120,6 +120,11 @@ python benchmark/longmemeval/run_home.py
|
|||
准备失败会回滚;成功后禁止再次准备同一工作区。
|
||||
Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=...)`。
|
||||
|
||||
顶层 `sampling_policy` 配置采样默认值:`max_cases`、`max_queries`(null 表示不限)、
|
||||
`temperature`、`ewma_alpha` 和 `uniform_exploration`。入口校验后将其传给宿主
|
||||
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
|
||||
和 `record_history`。
|
||||
|
||||
默认使用 `dev_meta` 已保存的 **group_1–group_5,共 250 个 case**,
|
||||
ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchmark.case_ids`,
|
||||
运行时无需读取分组文件,group_6–group_10 保持为留出集。
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
|
||||
# Preserve dev_meta's fixed training split; groups 6-10 remain held out.
|
||||
workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5
|
||||
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
|
||||
sampling_policy:
|
||||
max_cases: 5
|
||||
max_queries: null
|
||||
temperature: 1.0
|
||||
ewma_alpha: 0.3
|
||||
uniform_exploration: 0.2
|
||||
preparation:
|
||||
schema_version: 1
|
||||
benchmark:
|
||||
name: longmemeval
|
||||
source: benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json
|
||||
|
|
|
|||
|
|
@ -272,8 +272,11 @@ class DockerReMeSandboxFactory:
|
|||
plugins: tuple[str, ...] | list[str] = (),
|
||||
command_timeout: float = 1800.0,
|
||||
workspace_builder: WorkspaceBuilder | None = None,
|
||||
on_sandbox_started: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
self.candidate = candidate
|
||||
self._on_sandbox_started = on_sandbox_started
|
||||
self._sandbox_started_reported = False
|
||||
self.env = dict(env or {})
|
||||
self.config = config
|
||||
self.plugins = tuple(plugins)
|
||||
|
|
@ -309,6 +312,9 @@ class DockerReMeSandboxFactory:
|
|||
)
|
||||
try:
|
||||
await case.initialize()
|
||||
if self._on_sandbox_started is not None and not self._sandbox_started_reported:
|
||||
self._on_sandbox_started()
|
||||
self._sandbox_started_reported = True
|
||||
except BaseException as exc:
|
||||
await cleanup_on_error([(f"workspace {container_id!r}", workspace.close())], exc)
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -315,3 +315,394 @@ processes must not write there. Recovery requires a writable, functioning
|
|||
filesystem; if rollback fails, its journal remains for the next attempt and the
|
||||
Job reports the error. Calls with `_allowed_paths` restrictions are rejected
|
||||
because preparation owns multiple workspace output locations.
|
||||
|
||||
## Bounded agent foundation
|
||||
|
||||
`reme_home.agent.BoundedAgentWrapper` inherits ReMe's `AsAgentWrapper` and is
|
||||
registered as the `home_bounded` agent-wrapper backend when Home is enabled.
|
||||
It reuses ReMe's Job tools, hidden `injected_job_kwargs`, `tool_context_id`,
|
||||
built-in tools, permissions, model configuration, skills, token accounting and
|
||||
persisted sessions. No agent component or optimization run is enabled automatically.
|
||||
Configure a component explicitly, alongside the required model and Home plugins:
|
||||
|
||||
```yaml
|
||||
components:
|
||||
agent_wrapper:
|
||||
investigator:
|
||||
backend: home_bounded
|
||||
as_llm: default
|
||||
react_config:
|
||||
max_iters: 40
|
||||
time_budget_minutes: 30 # Optional advisory budget; null disables it.
|
||||
job_tools: [home_evaluate]
|
||||
injected_job_kwargs:
|
||||
case_ids: ["1"] # Hidden from the model and enforced by the host.
|
||||
```
|
||||
|
||||
Call `reply()` or `reply_stream()` through the normal ReMe wrapper interface;
|
||||
call-time options override component defaults. Each built agent owns its own
|
||||
timer and request state. The internal `BoundedReActAgent` still subclasses the AS
|
||||
runtime to implement loop hooks; callers use `BoundedAgentWrapper` for ReMe wiring.
|
||||
The lower-level `BoundedReActAgent` and `RunTimeBudgetTimer` exports remain available.
|
||||
|
||||
Each reply invocation starts and stops its advisory timer, including on errors and
|
||||
cancellation. Iteration reminders occur at 50%, 75%, 90% and 95%; coincident or
|
||||
skipped thresholds produce one reminder. Time reminders occur at 60%, 75%, 90%,
|
||||
100%, then each additional 10%, alternating temporary and retained messages. Time
|
||||
budgets do not cancel model calls or tools. Completed context replacements are
|
||||
counted in time reminders. Reminders use native AS hint blocks, preserving the
|
||||
reply's cumulative token usage across reasoning and tool calls. Use separate
|
||||
agent instances for concurrent work.
|
||||
|
||||
The AgentScope loop owns the iteration cap and its single finalization call.
|
||||
For text reports, Home removes all tools from that call, retains the original
|
||||
task, and trims oldest messages if needed to fit the context budget. The final
|
||||
report is still attempted if its preceding context compression fails; ordinary
|
||||
compression errors outside finalization and cancellation continue to propagate.
|
||||
The final response is buffered until complete, including for streaming models; unsolicited
|
||||
tool calls are discarded. If finalization fails or returns no text, the reply
|
||||
explicitly identifies any retained prior model output as incomplete. Cancellation
|
||||
and provider interruption keep AgentScope's interruption behavior. Native
|
||||
structured-output requests retain the SDK's schema and grace-iteration behavior.
|
||||
Events and final-message token usage retain AgentScope's existing semantics.
|
||||
|
||||
## Candidate versions and code inspection
|
||||
|
||||
Home registers the `home.code` Component and ten ordinary Step-backed Jobs. They
|
||||
operate only on the prepared `home/code/repo`, sharing the preparation/evaluation
|
||||
process lock. Enable Home and prepare the workspace before calling them.
|
||||
|
||||
| Job | Inputs and behavior |
|
||||
| --- | --- |
|
||||
| `home_code_status` | Current branch, full commit SHA, exact changed paths and pending merge state. |
|
||||
| `home_list_candidate_branches` | Local branches and candidates, excluding protected `init` from the candidate count. |
|
||||
| `home_create_candidate_branch` | `branch_name`: fork current HEAD and switch to a new candidate. |
|
||||
| `home_switch_candidate_branch` | `branch_name`: switch to an existing local branch, including `init`. |
|
||||
| `home_merge_candidate_branch` | `branch_name`: merge another candidate without committing; conflicts are aborted. |
|
||||
| `home_commit_candidate` | `message`: stage and commit all visible changes within prepared `candidate_paths`. |
|
||||
| `home_code_diff` | Omit inputs for tracked changes against HEAD; supply `base_revision` and `revision` to compare commits. With only `base_revision`, compare the working tree against that commit. |
|
||||
| `home_list_code_tree` | Optional `revision`, literal `path`, and `max_results` (default 500, maximum 2000). |
|
||||
| `home_read_code_file` | Required `path`; optional `revision`, one-based `start_line` and inclusive `end_line`. |
|
||||
| `home_search_code` | Required Python regex `pattern`; optional `revision`, literal `path`, and `max_results` (maximum 30). |
|
||||
|
||||
Start the configured service in one terminal, then use a second terminal:
|
||||
|
||||
```bash
|
||||
reme start config=/absolute/path/to/home.yaml
|
||||
```
|
||||
|
||||
```bash
|
||||
reme home_code_status
|
||||
reme home_create_candidate_branch branch_name=retrieval-routing
|
||||
# Edit files under the prepared candidate_paths, then inspect and commit.
|
||||
reme home_code_diff
|
||||
reme home_commit_candidate message='Improve evidence routing'
|
||||
reme home_evaluate debug=true
|
||||
reme home_evaluate
|
||||
```
|
||||
|
||||
Branch names must be single path-safe names without slashes. Creation, switching
|
||||
and merging require a clean checkout, including untracked files and pending
|
||||
merges. `init` must continue to point to the preparation's initial commit; commits
|
||||
and merges into it are rejected. Detached candidates, symlinks, submodules and
|
||||
changes outside `candidate_paths` are rejected by the managed mutation tools.
|
||||
Merges require an explicit later commit. No remote operations, branch deletion,
|
||||
amend, reset or rebase are exposed. Empty ordinary commits fail through Git.
|
||||
These guarantees apply to cooperating Home Jobs; do not grant an optimizer
|
||||
unrestricted shell writes or direct access to `.git` if it must obey this boundary.
|
||||
Cancellation waits for already-started Git work to finish before returning.
|
||||
|
||||
Code-navigation Jobs resolve `revision` (default `HEAD`) to a full SHA once per
|
||||
call, read raw committed objects and never execute candidate code. Dirty working
|
||||
files and Git archive attributes do not affect these reads. Pin the same SHA
|
||||
across a diagnostic session through ReMe's hidden injection:
|
||||
|
||||
```yaml
|
||||
job_tools: [home_list_code_tree, home_read_code_file, home_search_code]
|
||||
injected_job_kwargs:
|
||||
revision: "<full-commit-sha>"
|
||||
```
|
||||
|
||||
The wrapper hides injected fields from the model and rejects attempts to supply
|
||||
them. For comparisons, configure two agent invocations or two named Job aliases
|
||||
with separately pinned revisions. `HomeCodeStep` takes a fixed `operation` in
|
||||
Step configuration; repository location and operation are never model-selected.
|
||||
The Python entry point is `await home_code.run(operation, **arguments)`.
|
||||
|
||||
Reads are limited to 300 lines and 100,000 content characters, with explicit
|
||||
truncation flags. Search accepts Python multiline regex, merges adjacent match
|
||||
contexts, and displays at most five lines per result block and 100,000 content
|
||||
characters. Files above 5 MB and searches above 100 MB are rejected; regex
|
||||
matching runs in a separate process with a ten-second timeout. Use narrower
|
||||
paths or line ranges to continue inspecting. These tool-operation timeouts are
|
||||
independent of the bounded agent's advisory time budget.
|
||||
|
||||
Evaluation also checks the standalone repository identity and protected baseline
|
||||
before taking its existing clean-HEAD snapshot. Formal evaluation deduplication,
|
||||
automatic ancestor/donor selection and optimization scheduling remain separate
|
||||
policies; these Jobs do not enable an optimizer or change evaluation selection.
|
||||
|
||||
## Evaluation sampling and history
|
||||
|
||||
`home_evaluate` is the single evaluation Job. Set `sampling=true` for adaptive
|
||||
case/query selection, or leave it false for a full baseline or an explicit
|
||||
host-provided `case_ids`/`query_ids` selection. `sampling=true` conflicts with
|
||||
explicit selections in scored mode. `debug=true` ignores the sampling flag,
|
||||
uses debug cases and never reads, writes or registers evaluation history.
|
||||
The former `home_validate` Job is removed; use `home_evaluate sampling=true`.
|
||||
|
||||
Selection and optimization-history recording are independent. `record_history`
|
||||
defaults to `true` for compatibility: a first full optimization baseline contributes
|
||||
observations even with `sampling=false`. Set `record_history=false` to score code
|
||||
without feeding optimization history. This still saves the manifest, scores and
|
||||
result artifacts, but never registers a history entry, increments either counter
|
||||
or updates sampling observations. Rebuilding history preserves that exclusion.
|
||||
Debug always skips history regardless of either flag.
|
||||
|
||||
Evaluation parameters have distinct names and lifetimes:
|
||||
|
||||
| Parameter | Where it belongs | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `sampling` | Job input, default `false` | Enable adaptive selection for this invocation. |
|
||||
| `record_history` | Job input, default `true` | Record started rounds and successful results for optimization. |
|
||||
| `sampling_policy` | Component defaults or host Job input | Case/query limits and weighting settings. |
|
||||
|
||||
Home registers these Component defaults automatically; override the policy in
|
||||
application configuration when needed:
|
||||
|
||||
```yaml
|
||||
components:
|
||||
home.evaluation:
|
||||
default:
|
||||
backend: home_evaluation
|
||||
sampling_policy:
|
||||
max_cases: 5
|
||||
max_queries: null
|
||||
temperature: 1.0
|
||||
ewma_alpha: 0.3
|
||||
uniform_exploration: 0.2
|
||||
```
|
||||
|
||||
The Component policy is immutable. Each invocation validates its own policy,
|
||||
merging a partial `sampling_policy` over Component defaults and persisting the
|
||||
effective values with that run. Explicit null `max_queries` removes the query
|
||||
budget. The previous Component configuration key `sampling` is now
|
||||
`sampling_policy`; passing a policy object as the Job's boolean `sampling` fails.
|
||||
Normal ReMe Job defaults also work: set `sampling: true` and optionally
|
||||
`sampling_policy` directly under `jobs.home_evaluate`. Call-time values take
|
||||
precedence over same-named Job defaults; a call-time policy object replaces the
|
||||
Job-default policy object, then merges with Component defaults.
|
||||
|
||||
With the configured service running, use a second terminal:
|
||||
|
||||
```bash
|
||||
reme home_evaluate sampling=false # Full baseline.
|
||||
reme home_evaluate sampling=false record_history=false # Standalone full evaluation.
|
||||
reme home_evaluate sampling=true # Adaptive formal evaluation.
|
||||
reme home_evaluate debug=true sampling=true # Debug only; no history feedback.
|
||||
```
|
||||
|
||||
Host Python callers can override the policy for one invocation without changing
|
||||
the next invocation's defaults:
|
||||
|
||||
```python
|
||||
result = await app.run_job(
|
||||
"home_evaluate",
|
||||
sampling=True,
|
||||
sampling_policy={"max_cases": 3, "max_queries": 12},
|
||||
)
|
||||
|
||||
# Score specified prepared cases without affecting the optimizer's history.
|
||||
standalone = await app.run_job(
|
||||
"home_evaluate",
|
||||
sampling=False,
|
||||
record_history=False,
|
||||
case_ids=["case-a", "case-b"], # Omit to run the full prepared dataset.
|
||||
)
|
||||
```
|
||||
|
||||
`sampling_policy` and explicit case/query IDs stay out of the default agent tool
|
||||
schema. An optimizer that must obey a fixed budget should receive hidden,
|
||||
enforced values through the normal wrapper injection:
|
||||
|
||||
```yaml
|
||||
job_tools: [home_evaluate]
|
||||
injected_job_kwargs:
|
||||
sampling: true
|
||||
record_history: true
|
||||
sampling_policy:
|
||||
max_cases: 3
|
||||
max_queries: 12
|
||||
```
|
||||
|
||||
The wrapper hides injected parameters and rejects model attempts to provide
|
||||
those keys. Use ordinary call arguments when the host should decide the mode
|
||||
per invocation; use injection when that decision must remain fixed for an agent.
|
||||
|
||||
Case/query selection retains dev_meta b53048e4's EWMA loss, staleness, observation
|
||||
uncertainty, un-rechecked error priority, normalized adaptive/uniform mixture,
|
||||
weighted drawing without replacement and fair-share query budget allocation.
|
||||
Policy defaults are unchanged. Positive integer case/query limits, positive
|
||||
finite temperature, alpha in `(0, 1]` and uniform exploration in `[0, 1]` are
|
||||
validated. Seeds, draw weights, eligibility and exact selections are saved.
|
||||
An explicit construction donor uses the existing dev_meta eligibility rules and
|
||||
checksum checks. A later full rebuild considers the full training set again.
|
||||
|
||||
### Successful evaluation history
|
||||
|
||||
`reme_home.evaluation.history.HomeEvaluationHistory` is the dedicated
|
||||
`home.evaluation_history` Component, automatically bound by `HomeEvaluation`.
|
||||
It owns successful-result registration, lookup, deduplication, cache rebuilding
|
||||
and the observations used for the next draw. Steps remain invocation adapters;
|
||||
they do not own cross-evaluation state.
|
||||
|
||||
A successful history entry requires `record_history=true` and a started, completed, scored training-set
|
||||
run with a full summary and no infrastructure errors, following the existing
|
||||
`Response.success` contract. Low or zero quality scores remain valid observations.
|
||||
Classified candidate failures retain the existing response semantics; only their
|
||||
actual numeric scores train EWMA. A run with any infrastructure error is entirely
|
||||
excluded, including scores from its healthy cases. Debug, cancelled, fail-fast
|
||||
and pre-start failures never enter successful history. A full summary is the durable
|
||||
completion boundary: cancelling the caller during subsequent history bookkeeping
|
||||
does not relabel that already-completed evaluation. Failed registration discards
|
||||
only derived caches so they can rebuild from the final source run status.
|
||||
|
||||
```python
|
||||
history = app.context.components["home.evaluation_history"]["default"]
|
||||
results = history.list_results() # Typed EvaluationRecord objects, chronological.
|
||||
result = history.get_result(results[-1].run_key)
|
||||
history.rebuild() # Rebuild derived indexes from saved artifacts.
|
||||
```
|
||||
|
||||
The Component owns the workspace lock for these synchronous filesystem methods;
|
||||
async host code can run them in a worker thread. Its compact catalog lives at
|
||||
`home/evaluations/history.json`. Each record points to its run via `run_key` and
|
||||
contains commit/branch identity, selected IDs, completion time and summary metrics;
|
||||
questions, answers and memory files stay in their original result directory.
|
||||
|
||||
The separate startup ledger advances only after the first successfully initialized
|
||||
sandbox, including when a later worker or execution fails. Extra/replacement
|
||||
workers do not count twice. `home/sampling/state.json` distinguishes started
|
||||
`round` from `successful_round`. Only successful history advances the probability
|
||||
clock or changes EWMA; failed runs cannot age cases or alter the next draw's
|
||||
weights. Started plans live under
|
||||
`home/sampling/plans/<branch>/<commit-prefix>/<run-id>.json`. Their `round` is the
|
||||
startup ledger round; `selection_round` records the successful-history snapshot
|
||||
used for an adaptive draw.
|
||||
|
||||
For `record_history=false`, full or explicit evaluation does not access the history
|
||||
manager. Adaptive selection can still read existing observations, while leaving
|
||||
them unchanged; its draw is saved locally as `manifest.selection_plan`, without
|
||||
a startup-ledger round or shared sampling plan. `manifest.options.record_history`
|
||||
persists the requested setting, and the response's `record_history` indicates
|
||||
whether the run participates in history (always false for debug). Legacy manifests
|
||||
without the setting retain the previous history behavior.
|
||||
|
||||
Formal runs remain serialized by `home/metadata/validation.lock`; debug bypasses
|
||||
that lock. Missing caches reconstruct from started plans and successful scored
|
||||
artifacts. Older sampling caches are rebuilt under the success-only rule.
|
||||
`history.rebuild()` rewrites only the catalog and sampling cache. Preserve source
|
||||
manifests, plans and result artifacts when rebuilding; candidate code, datasets
|
||||
and memory files are never modified.
|
||||
|
||||
## Evaluation result inspection
|
||||
|
||||
Two read-only Jobs navigate saved evaluations independently of successful
|
||||
optimization history. They include debug, failed, cancelled, running and standalone
|
||||
(`record_history=false`) runs. Reads never register results, update sampling,
|
||||
rebuild summaries or write artifacts.
|
||||
|
||||
`home_list_validation_results` accepts optional `branch_name` and `commit_sha`
|
||||
filters plus `offset=0`, `limit=20` (1–100). Without filters it lists all runs,
|
||||
newest first, with a stable path tie-breaker. It returns `row_count`, `rows` and
|
||||
`next_offset`, compact identity/options, saved summary counts and scores, failure
|
||||
previews and artifact paths. A malformed run is reported as a `read_error` row
|
||||
without stopping the unfiltered catalog.
|
||||
|
||||
`commit_sha` accepts a full SHA or hexadecimal prefix. A prefix must match exactly
|
||||
one of **all commits reachable from the selected local branch**, including commits
|
||||
that have never been evaluated. Omitting `branch_name` with a commit filter uses
|
||||
the current candidate branch. Missing or ambiguous matches fail explicitly; the
|
||||
response includes the resolved branch and full SHA. A branch-only filter can also
|
||||
browse historical runs from a deleted branch.
|
||||
|
||||
`home_inspect_validation` requires `validation_path` (absolute or relative to
|
||||
`home/`) and accepts optional `case_id`, `query_id`, `offset=0` and `limit=20`:
|
||||
|
||||
| Selection | Returned rows |
|
||||
| --- | --- |
|
||||
| Run only | Case summaries, missing scores first, then low scores. |
|
||||
| Run and case | Query summaries in manifest order, plus independently paged failed construction Jobs. |
|
||||
| Run, case and query | One query's question, answer, rubric, score and judge/error previews. |
|
||||
|
||||
`query_id` requires `case_id`; both must be selected by the run manifest. Cases or
|
||||
queries without published evidence are explicitly marked `missing`, with null
|
||||
scores. The tool uses a saved full summary, or an existing partial summary when
|
||||
the full summary is absent. Missing summaries are reported without reconstruction.
|
||||
Errors are previewed to 600 characters, and selected query fields to 2,000 each;
|
||||
truncated fields carry a marker. `rows_source`, JSON `pointer` values, and log and
|
||||
memory paths identify the original evidence. Failed construction Jobs have their
|
||||
own `row_count`, `rows`, `next_offset` under `construction.failed_jobs`; their
|
||||
pagination is independent of the query rows, using the supplied offset/limit.
|
||||
|
||||
With the configured service running, use a second terminal:
|
||||
|
||||
```bash
|
||||
reme home_list_validation_results branch_name=candidate commit_sha=abc123 limit=10
|
||||
reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1
|
||||
reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1 case_id=case-a query_id=query-a
|
||||
```
|
||||
|
||||
Host callers use the same Jobs through `app.run_job()`. For a case-scoped agent,
|
||||
inject `validation_path` and `case_id` with `injected_job_kwargs`; the wrapper hides
|
||||
and enforces these values. Both Jobs reject path escapes and symlinks, and fail
|
||||
closed for restricted `_allowed_paths` contexts, like the other Home Jobs.
|
||||
|
||||
### Memory checkpoint inspection
|
||||
|
||||
Two read-only Jobs investigate a case's exported memory Git repository. Locate
|
||||
an evaluation and case with `home_list_validation_results` and
|
||||
`home_inspect_validation` first. Neither Job changes artifacts, Git refs, the
|
||||
index, evaluation history or sampling state.
|
||||
|
||||
- `home_inspect_memory_history(validation_path, case_id, path=null, offset=0,
|
||||
limit=20)` returns `status`, `workspace_exists`, `has_git_history`, `head`,
|
||||
`repository`, and newest-first checkpoint `rows` with `row_count` and
|
||||
`next_offset`. Limits are 1–100. Each row includes the full commit SHA, parents,
|
||||
timestamp, subject (up to 2,000 characters) and a `session_id` when the subject
|
||||
follows `session: <id>`. `path` follows a literal workspace-relative file or
|
||||
directory. Missing exports, absent Git history and empty repositories have
|
||||
distinct statuses; corrupt repositories fail explicitly.
|
||||
- `home_compare_memory(validation_path, case_id, base_ref=null,
|
||||
target_ref="HEAD", target_validation_path=null, path=null)` compares two memory
|
||||
commits. Within one evaluation, `base_ref` is required. For another evaluation,
|
||||
supply `target_validation_path`; both sides then default to `HEAD`. The case
|
||||
must be selected in both manifests. Returns resolved commits, candidate commit
|
||||
identities, manifest fingerprints, added/removed/modified paths, content-tree
|
||||
hashes, a unified `diff`, and `truncated`.
|
||||
|
||||
References accept HEAD-relative expressions and hexadecimal SHA prefixes unique
|
||||
among commits reachable from that memory repository's HEAD. Resolved expressions
|
||||
must also be HEAD-reachable. This is independent of candidate-code branch history.
|
||||
Comparisons read committed regular files from Git objects, ignoring dirty or
|
||||
untracked checkout contents. File mode changes alone are not content differences.
|
||||
Binary changes are identified without printing their bytes. Each snapshot is
|
||||
limited to 2,000 files, 20 MB total and 5 MB per file; diff output is limited to
|
||||
100,000 characters. Narrow `path` if a limit is reached. Tree hashes describe the
|
||||
selected path scope. Linked Git storage, alternates, shallow/partial histories and
|
||||
symbolic links in Git metadata are rejected.
|
||||
|
||||
After starting the service with the Home configuration, call from another terminal:
|
||||
|
||||
```bash
|
||||
reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1
|
||||
reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 case_id=case1 base_ref=HEAD~1
|
||||
reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 target_validation_path=evaluations/candidate/1234567/1 case_id=case1
|
||||
```
|
||||
|
||||
`validation_path` is absolute or relative to `home/`; for example the relative
|
||||
form is `evaluations/candidate/abcdef0/1`. For agent tools, host `injected_job_kwargs` can pin
|
||||
`validation_path` and `case_id`; the wrapper hides injected fields and rejects
|
||||
model overrides. Like the other Home inspection Jobs, these Jobs reject
|
||||
`_allowed_paths` restrictions rather than bypassing them. Query-to-session evidence
|
||||
tracing and the case study agent are separate functionality.
|
||||
|
|
|
|||
6
plugins/home/src/reme_home/agent/__init__.py
Normal file
6
plugins/home/src/reme_home/agent/__init__.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
"""Agent foundations for Home's investigation and optimization workflows."""
|
||||
|
||||
from .bounded import BoundedReActAgent, RunTimeBudgetTimer
|
||||
from .wrapper import BoundedAgentWrapper
|
||||
|
||||
__all__ = ["BoundedAgentWrapper", "BoundedReActAgent", "RunTimeBudgetTimer"]
|
||||
308
plugins/home/src/reme_home/agent/bounded.py
Normal file
308
plugins/home/src/reme_home/agent/bounded.py
Normal file
|
|
@ -0,0 +1,308 @@
|
|||
"""AgentScope agent that turns an exhausted ReAct budget into a final answer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, AsyncGenerator, Callable
|
||||
|
||||
from agentscope.agent import Agent
|
||||
from agentscope.message import HintBlock, Msg, TextBlock, UserMsg
|
||||
from agentscope.model import ChatResponse, FinishedReason
|
||||
from agentscope.tool import ToolChoice
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_REMINDER_RATIOS = (0.50, 0.75, 0.90, 0.95)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TimeBudgetReminder:
|
||||
"""One reminder plus whether it should survive the current reasoning call."""
|
||||
|
||||
content: str
|
||||
persistent: bool
|
||||
|
||||
|
||||
class RunTimeBudgetTimer:
|
||||
"""Track one agent run and emit advisory reminders without stopping it."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
time_budget_minutes: float,
|
||||
*,
|
||||
agent_label: str,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
if (
|
||||
not isinstance(time_budget_minutes, (int, float))
|
||||
or isinstance(time_budget_minutes, bool)
|
||||
or not math.isfinite(time_budget_minutes)
|
||||
or time_budget_minutes <= 0
|
||||
):
|
||||
raise ValueError("time_budget_minutes must be a positive number")
|
||||
if not isinstance(agent_label, str) or not agent_label.strip():
|
||||
raise ValueError("agent_label must be a non-empty string")
|
||||
self.time_budget_minutes = float(time_budget_minutes)
|
||||
self.agent_label = agent_label.strip()
|
||||
self._clock = clock
|
||||
self._started_at: float | None = None
|
||||
self._last_reminded_ratio = 0.0
|
||||
self._next_reminder_is_persistent = False
|
||||
self.context_compression_count = 0
|
||||
|
||||
@property
|
||||
def active(self) -> bool:
|
||||
"""Return whether a run is currently being timed."""
|
||||
|
||||
return self._started_at is not None
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start a fresh run timer."""
|
||||
|
||||
self._started_at = self._clock()
|
||||
self._last_reminded_ratio = 0.0
|
||||
self._next_reminder_is_persistent = False
|
||||
self.context_compression_count = 0
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop timing the current run."""
|
||||
|
||||
self._started_at = None
|
||||
|
||||
def record_context_compression(self) -> None:
|
||||
"""Record one completed context compression in the current run."""
|
||||
|
||||
self.context_compression_count += 1
|
||||
|
||||
def next_reminder(self) -> TimeBudgetReminder | None:
|
||||
"""Return the latest newly reached soft-budget reminder."""
|
||||
|
||||
if self._started_at is None:
|
||||
return None
|
||||
elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0)
|
||||
elapsed_ratio = elapsed_minutes / self.time_budget_minutes
|
||||
reached_ratio: float | None = None
|
||||
if elapsed_ratio >= 1.0:
|
||||
reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10
|
||||
else:
|
||||
for threshold in (0.90, 0.75, 0.60):
|
||||
if elapsed_ratio >= threshold:
|
||||
reached_ratio = threshold
|
||||
break
|
||||
if reached_ratio is None or reached_ratio <= self._last_reminded_ratio:
|
||||
return None
|
||||
self._last_reminded_ratio = reached_ratio
|
||||
persistent = self._next_reminder_is_persistent
|
||||
self._next_reminder_is_persistent = not persistent
|
||||
elapsed = _format_minutes(elapsed_minutes)
|
||||
percentage = round(reached_ratio * 100)
|
||||
return TimeBudgetReminder(
|
||||
content=(
|
||||
f"<system-reminder>The {self.agent_label} run has used {elapsed} ({percentage}% of its "
|
||||
f"{_format_minutes(self.time_budget_minutes)} soft time budget). "
|
||||
f"The run has completed {self.context_compression_count} context compressions. "
|
||||
"This reminder does not terminate the run. Prioritize the highest-value remaining investigation and "
|
||||
"produce the required final report as soon as the evidence is sufficient.</system-reminder>"
|
||||
),
|
||||
persistent=persistent,
|
||||
)
|
||||
|
||||
|
||||
def _format_minutes(minutes: float) -> str:
|
||||
"""Format a timer duration compactly."""
|
||||
|
||||
if minutes < 1:
|
||||
return f"{max(0, round(minutes * 60))} seconds"
|
||||
rounded = round(minutes, 1)
|
||||
value = int(rounded) if rounded.is_integer() else rounded
|
||||
unit = "minute" if value == 1 else "minutes"
|
||||
return f"{value} {unit}"
|
||||
|
||||
|
||||
def _remove_hint_by_identity(messages: list[Msg], target: HintBlock) -> None:
|
||||
"""Remove only the temporary hint, retaining the reply message and its usage."""
|
||||
for message in messages:
|
||||
for index, block in enumerate(message.content):
|
||||
if block is target:
|
||||
del message.content[index]
|
||||
return
|
||||
|
||||
|
||||
class BoundedReActAgent(Agent):
|
||||
"""Add advisory budgets and safe finalization to AgentScope's bounded loop.
|
||||
|
||||
AgentScope owns iteration accounting, tool execution, events and cancellation.
|
||||
Its one final reasoning call after ``max_iters`` receives no tools. Native
|
||||
structured-output requests retain AgentScope's schema and grace-iteration rules.
|
||||
Instances, like AgentScope agents, must not serve concurrent replies.
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, run_time_budget_timer: RunTimeBudgetTimer | None = None, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.run_time_budget_timer = run_time_budget_timer
|
||||
self._sent_iteration_reminders: set[int] = set()
|
||||
self._original_task_prompt = ""
|
||||
|
||||
async def _reply_impl(self, inputs: Any = None, **kwargs: Any) -> AsyncGenerator[Any, None]:
|
||||
"""Own the advisory timer for this invocation, including error cleanup."""
|
||||
self._sent_iteration_reminders.clear()
|
||||
if isinstance(inputs, Msg):
|
||||
self._original_task_prompt = inputs.get_text_content() or ""
|
||||
elif isinstance(inputs, list):
|
||||
self._original_task_prompt = "\n\n".join(message.get_text_content() or "" for message in inputs)
|
||||
elif inputs is None:
|
||||
# ReMe observes inputs before reply(); retain that task for finalization.
|
||||
self._original_task_prompt = next(
|
||||
(
|
||||
message.get_text_content() or ""
|
||||
for message in reversed(self.state.context)
|
||||
if message.role == "user"
|
||||
),
|
||||
"",
|
||||
)
|
||||
timer = self.run_time_budget_timer
|
||||
if timer is not None:
|
||||
timer.start()
|
||||
try:
|
||||
async for item in super()._reply_impl(inputs=inputs, **kwargs):
|
||||
yield item
|
||||
finally:
|
||||
if timer is not None:
|
||||
timer.stop()
|
||||
|
||||
async def compress_context(self, *args: Any, **kwargs: Any) -> None:
|
||||
"""Count successful compressions; allow final reports after compression failure."""
|
||||
summary_before, context_before = self.state.summary, self.state.context
|
||||
try:
|
||||
await super().compress_context(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
if not self._is_finalizing():
|
||||
raise
|
||||
logger.warning(
|
||||
"Compression before the final report failed for Home agent %s; attempting finalization: %s",
|
||||
self.name,
|
||||
type(exc).__name__,
|
||||
)
|
||||
return
|
||||
if self.run_time_budget_timer is not None and (
|
||||
self.state.summary is not summary_before or self.state.context is not context_before
|
||||
):
|
||||
self.run_time_budget_timer.record_context_compression()
|
||||
|
||||
async def _reasoning(self, *args: Any, **kwargs: Any) -> AsyncGenerator[Any, None]:
|
||||
"""Deliver each iteration threshold once and alternate time reminders."""
|
||||
timer = self.run_time_budget_timer
|
||||
reminder = timer.next_reminder() if timer is not None else None
|
||||
reminder_hint = None
|
||||
if reminder is not None:
|
||||
reminder_hint = HintBlock(source="home_run_time_guard", hint=reminder.content)
|
||||
self.state.append_context(self.name, [reminder_hint])
|
||||
completed = self.state.cur_iter
|
||||
thresholds = {math.ceil(self.react_config.max_iters * ratio) for ratio in _REMINDER_RATIOS}
|
||||
reached = thresholds - self._sent_iteration_reminders
|
||||
reached = {threshold for threshold in reached if threshold <= completed}
|
||||
if reached:
|
||||
remaining = max(0, self.react_config.max_iters - completed)
|
||||
self.state.append_context(
|
||||
self.name,
|
||||
[
|
||||
HintBlock(
|
||||
source="home_iteration_guard",
|
||||
hint=(
|
||||
f"<system-reminder>You have completed {completed} reasoning-acting iterations; "
|
||||
f"{remaining} iteration(s) remain.</system-reminder>"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
self._sent_iteration_reminders.update(reached)
|
||||
try:
|
||||
async for item in super()._reasoning(*args, **kwargs):
|
||||
yield item
|
||||
finally:
|
||||
if reminder is not None and not reminder.persistent and reminder_hint is not None:
|
||||
_remove_hint_by_identity(self.state.context, reminder_hint)
|
||||
|
||||
def _is_finalizing(self) -> bool:
|
||||
"""Only the iteration cap triggers finalization; elapsed time is advisory."""
|
||||
return self.state.cur_iter >= self.react_config.max_iters and self.state.reply_context.structured_schema is None
|
||||
|
||||
async def _call_model(self, messages: list[Msg], tools: list[dict], tool_choice: ToolChoice | None = None):
|
||||
"""Use the native finalization slot; never start a second synthesis call."""
|
||||
if not self._is_finalizing():
|
||||
return await super()._call_model(messages=messages, tools=tools, tool_choice=tool_choice)
|
||||
|
||||
last_text = next(
|
||||
(
|
||||
message.get_text_content()
|
||||
for message in reversed(self.state.context)
|
||||
if message.role == "assistant" and message.name == self.name and message.get_text_content()
|
||||
),
|
||||
"(no previous displayable model response)",
|
||||
)
|
||||
messages = list(messages)
|
||||
messages.append(
|
||||
UserMsg(
|
||||
name="home_iteration_guard",
|
||||
content=(
|
||||
"The iteration budget is exhausted. Return the best final report from collected evidence. "
|
||||
"Do not call tools or propose more investigation. Distinguish findings from uncertainty.\n"
|
||||
f"Original task: {self._original_task_prompt or '(see the conversation above)'}"
|
||||
),
|
||||
),
|
||||
)
|
||||
response_usage = None
|
||||
try:
|
||||
limit = int(self.model.context_size * self.context_config.trigger_ratio)
|
||||
while await self.model.count_tokens(messages=messages, tools=[]) >= limit:
|
||||
if len(messages) <= 2:
|
||||
raise ValueError("Final instructions exceed the model context budget")
|
||||
del messages[1]
|
||||
response = await super()._call_model(messages=messages, tools=[], tool_choice=ToolChoice(mode="none"))
|
||||
response = await self._consume_final_response(response)
|
||||
response_usage = response.usage
|
||||
if response.finished_reason == FinishedReason.INTERRUPTED:
|
||||
return response
|
||||
content = [block for block in response.content if isinstance(block, TextBlock) and block.text.strip()]
|
||||
if not content:
|
||||
raise ValueError("Final model response contains no displayable answer")
|
||||
# Ignore unsolicited tool calls, even when accompanied by valid text.
|
||||
response.content = content
|
||||
return response
|
||||
except Exception as exc:
|
||||
logger.exception("Final report failed for Home agent %s", self.name)
|
||||
return ChatResponse(
|
||||
content=[
|
||||
TextBlock(
|
||||
text=(
|
||||
f"Reached the maximum of {self.react_config.max_iters} reasoning-acting iterations. "
|
||||
f"Final report generation failed ({type(exc).__name__}); "
|
||||
f"the following is incomplete prior model output:\n\n{last_text}"
|
||||
),
|
||||
),
|
||||
],
|
||||
is_last=True,
|
||||
usage=response_usage,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _consume_final_response(response: ChatResponse | AsyncGenerator[ChatResponse, None]) -> ChatResponse:
|
||||
"""Require a completed response for streaming and non-streaming models."""
|
||||
if isinstance(response, ChatResponse):
|
||||
return response
|
||||
if not inspect.isasyncgen(response):
|
||||
raise TypeError(f"Unexpected final response: {type(response)}")
|
||||
completed = None
|
||||
try:
|
||||
async for chunk in response:
|
||||
if chunk.is_last:
|
||||
completed = chunk
|
||||
finally:
|
||||
await response.aclose()
|
||||
if completed is None:
|
||||
raise ValueError("Final response stream ended without a completed response")
|
||||
return completed
|
||||
21
plugins/home/src/reme_home/agent/wrapper.py
Normal file
21
plugins/home/src/reme_home/agent/wrapper.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""ReMe agent wrapper with Home's bounded investigation runtime."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from reme.components.agent_wrapper import AsAgentWrapper
|
||||
|
||||
from .bounded import BoundedReActAgent, RunTimeBudgetTimer
|
||||
|
||||
|
||||
class BoundedAgentWrapper(AsAgentWrapper):
|
||||
"""Reuse ReMe Job tools, injected arguments, permissions and persisted sessions.
|
||||
|
||||
``react_config.max_iters`` configures the iteration cap. Optional
|
||||
``time_budget_minutes`` adds advisory reminders; each built SDK agent owns
|
||||
its own timer so requests never share budget state.
|
||||
"""
|
||||
|
||||
def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> BoundedReActAgent:
|
||||
budget = options.get("time_budget_minutes")
|
||||
timer = None if budget is None else RunTimeBudgetTimer(budget, agent_label=self.name)
|
||||
return BoundedReActAgent(run_time_budget_timer=timer, **kwargs)
|
||||
1
plugins/home/src/reme_home/code/__init__.py
Normal file
1
plugins/home/src/reme_home/code/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
"""Home candidate version management and immutable code inspection."""
|
||||
72
plugins/home/src/reme_home/code/component.py
Normal file
72
plugins/home/src/reme_home/code/component.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
"""Application-owned Git work serialized with Home preparation and snapshots."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from reme.components.base_component import BaseComponent
|
||||
|
||||
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
|
||||
from . import repository, snapshot
|
||||
from .models import ARGUMENTS
|
||||
|
||||
OPERATIONS = {
|
||||
"status": repository.status,
|
||||
"list_branches": lambda repo, _record: repository.branches(repo),
|
||||
"create_branch": repository.create_branch,
|
||||
"switch_branch": repository.switch_branch,
|
||||
"merge_branch": repository.merge_branch,
|
||||
"commit": repository.commit,
|
||||
"diff": repository.diff,
|
||||
"list_code_tree": snapshot.list_code_tree,
|
||||
"read_code_file": snapshot.read_code_file,
|
||||
"search_code": snapshot.search_code,
|
||||
}
|
||||
|
||||
|
||||
class HomeCode(BaseComponent):
|
||||
"""Manage only the prepared Home candidate; no remotes or history rewrites."""
|
||||
|
||||
component_type = "home.code"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.workspace = self.bind("default", HomeWorkspace, optional=False)
|
||||
self._pending: set[asyncio.Task] = set()
|
||||
|
||||
async def _close(self):
|
||||
if self._pending:
|
||||
await asyncio.gather(*self._pending, return_exceptions=True)
|
||||
|
||||
async def run(self, operation: str, **kwargs) -> dict:
|
||||
"""Validate before dispatch, and finish owned Git work before propagating cancellation."""
|
||||
if not self.is_started:
|
||||
raise RuntimeError("Start the Home application before managing candidate code")
|
||||
if operation not in OPERATIONS:
|
||||
raise ValueError(f"Unknown code operation: {operation}")
|
||||
arguments = ARGUMENTS[operation].model_validate(kwargs).model_dump()
|
||||
task = asyncio.create_task(asyncio.to_thread(self._run, operation, arguments))
|
||||
self._pending.add(task)
|
||||
cancelled = False
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
result = await asyncio.shield(task)
|
||||
if cancelled:
|
||||
raise asyncio.CancelledError
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
if task.done():
|
||||
raise
|
||||
cancelled = True
|
||||
except Exception:
|
||||
if cancelled:
|
||||
raise asyncio.CancelledError from None
|
||||
raise
|
||||
finally:
|
||||
self._pending.discard(task)
|
||||
|
||||
def _run(self, operation: str, arguments: dict) -> dict:
|
||||
"""Hold the same cross-process lock used when evaluations snapshot HEAD."""
|
||||
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
|
||||
with _initialization_lock(self.workspace.state_path / "workspace.lock"):
|
||||
repo, record = repository.validate_repository(self.workspace.home_path)
|
||||
return OPERATIONS[operation](repo, record, **arguments)
|
||||
68
plugins/home/src/reme_home/code/models.py
Normal file
68
plugins/home/src/reme_home/code/models.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
"""Strict input contracts for candidate version and code-navigation Jobs."""
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from ..preparation.spec import SpecModel
|
||||
|
||||
|
||||
class NoArguments(SpecModel):
|
||||
"""An operation with no public inputs."""
|
||||
|
||||
|
||||
class BranchArguments(SpecModel):
|
||||
"""One existing or new local candidate branch."""
|
||||
|
||||
branch_name: str = Field(min_length=1, max_length=100)
|
||||
|
||||
|
||||
class CommitArguments(SpecModel):
|
||||
"""Commit the complete candidate change with an explicit message."""
|
||||
|
||||
message: str = Field(min_length=1, max_length=10_000, pattern=r"\S")
|
||||
|
||||
|
||||
class DiffArguments(SpecModel):
|
||||
"""Omit revisions for tracked working changes against HEAD."""
|
||||
|
||||
base_revision: str | None = None
|
||||
revision: str | None = None
|
||||
|
||||
|
||||
class TreeArguments(SpecModel):
|
||||
"""Resolve HEAD by default; hosts can pin revision with injected_job_kwargs."""
|
||||
|
||||
revision: str = Field(default="HEAD", min_length=1)
|
||||
path: str | None = None
|
||||
max_results: int = Field(default=500, ge=1, le=2000)
|
||||
|
||||
|
||||
class ReadArguments(SpecModel):
|
||||
"""A bounded line range from a committed regular text file."""
|
||||
|
||||
path: str = Field(min_length=1)
|
||||
revision: str = Field(default="HEAD", min_length=1)
|
||||
start_line: int = Field(default=1, ge=1)
|
||||
end_line: int | None = Field(default=None, ge=1)
|
||||
|
||||
|
||||
class SearchArguments(SpecModel):
|
||||
"""Python regex search with multiline matching and bounded result blocks."""
|
||||
|
||||
pattern: str = Field(min_length=1, max_length=500)
|
||||
revision: str = Field(default="HEAD", min_length=1)
|
||||
path: str | None = None
|
||||
max_results: int = Field(default=30, ge=1, le=30)
|
||||
|
||||
|
||||
ARGUMENTS = {
|
||||
"status": NoArguments,
|
||||
"list_branches": NoArguments,
|
||||
"create_branch": BranchArguments,
|
||||
"switch_branch": BranchArguments,
|
||||
"merge_branch": BranchArguments,
|
||||
"commit": CommitArguments,
|
||||
"diff": DiffArguments,
|
||||
"list_code_tree": TreeArguments,
|
||||
"read_code_file": ReadArguments,
|
||||
"search_code": SearchArguments,
|
||||
}
|
||||
84
plugins/home/src/reme_home/code/regex_worker.py
Normal file
84
plugins/home/src/reme_home/code/regex_worker.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""Stdlib-only, subprocess-isolated Python regex matching for committed text."""
|
||||
|
||||
from bisect import bisect_right
|
||||
from itertools import accumulate
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def search(files: dict[str, str], pattern: str, max_results: int) -> dict:
|
||||
"""Merge adjacent contexts, retaining at most five lines per displayed block."""
|
||||
compiled = re.compile(pattern, re.MULTILINE)
|
||||
blocks = []
|
||||
occurrences = 0
|
||||
for path, text in files.items():
|
||||
lines = text.splitlines()
|
||||
starts = [0, *accumulate(map(len, text.splitlines(keepends=True)))]
|
||||
for match in compiled.finditer(text):
|
||||
if match.start() == match.end():
|
||||
raise ValueError("pattern must consume at least one character")
|
||||
occurrences += 1
|
||||
if occurrences > 100_000:
|
||||
raise ValueError("Too many regex matches; narrow pattern or path")
|
||||
first = bisect_right(starts, match.start())
|
||||
last = bisect_right(starts, match.end() - 1)
|
||||
start, end = max(1, first - 1), min(len(lines), last + 1)
|
||||
if (
|
||||
blocks
|
||||
and blocks[-1]["path"] == path
|
||||
and start <= blocks[-1]["end"] + 1
|
||||
and (max(end, blocks[-1]["end"]) - blocks[-1]["start"] < 5)
|
||||
):
|
||||
block = blocks[-1]
|
||||
block["end"] = max(end, block["end"])
|
||||
block["ranges"].append((first, last))
|
||||
else:
|
||||
blocks.append({"path": path, "start": start, "end": end, "ranges": [(first, last)]})
|
||||
matches = []
|
||||
characters = 0
|
||||
for block in blocks[:max_results]:
|
||||
lines = files[block["path"]].splitlines()
|
||||
displayed_end = min(block["end"], block["start"] + 4)
|
||||
content = "\n".join(
|
||||
f"{'>' if any(first <= index <= last for first, last in block['ranges']) else ' '} "
|
||||
f"{index} | {lines[index - 1]}"
|
||||
for index in range(block["start"], displayed_end + 1)
|
||||
)
|
||||
remaining = 100_000 - characters
|
||||
if remaining <= 0:
|
||||
break
|
||||
content = content[:remaining]
|
||||
characters += len(content)
|
||||
matches.append(
|
||||
{
|
||||
"path": block["path"],
|
||||
"start_line": block["start"],
|
||||
"end_line": block["end"],
|
||||
"displayed_end_line": displayed_end,
|
||||
"occurrence_count": len(block["ranges"]),
|
||||
"content": content,
|
||||
"content_truncated": displayed_end < block["end"] or characters >= 100_000,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"occurrence_count": occurrences,
|
||||
"merged_match_count": len(blocks),
|
||||
"returned_match_count": len(matches),
|
||||
"matches": matches,
|
||||
"truncated": len(blocks) > len(matches) or any(match["content_truncated"] for match in matches),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Emit structured errors without candidate text or traceback noise."""
|
||||
arguments = json.load(sys.stdin)
|
||||
try:
|
||||
result = search(**arguments)
|
||||
except (ValueError, re.error) as exc:
|
||||
result = {"error": str(exc)}
|
||||
json.dump(result, sys.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
213
plugins/home/src/reme_home/code/repository.py
Normal file
213
plugins/home/src/reme_home/code/repository.py
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
"""Local candidate Git operations shared by Home Jobs and evaluation snapshots."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from ..evaluation.artifacts import contained, read_json
|
||||
from ..preparation.code import git
|
||||
from ..preparation.spec import relative_path
|
||||
|
||||
INITIAL_BRANCH = "init"
|
||||
MAX_OUTPUT = 100_000
|
||||
|
||||
|
||||
def resolve_revision(repository: Path, revision: str) -> str:
|
||||
"""Resolve a revision once, without accepting command-line options."""
|
||||
if not isinstance(revision, str) or not revision.strip() or revision.startswith("-"):
|
||||
raise ValueError("revision must be a nonempty Git revision without a leading '-'")
|
||||
return git(repository, "rev-parse", "--verify", "--end-of-options", revision + "^{commit}")
|
||||
|
||||
|
||||
def validate_repository(home: Path, record: dict | None = None) -> tuple[Path, dict]:
|
||||
"""Require the prepared standalone repository and its unchanged baseline ref."""
|
||||
record = record if record is not None else read_json(contained(home, "metadata/preparation.json"))
|
||||
repository = contained(home, "code/repo", directory=True)
|
||||
metadata = contained(repository, ".git", directory=True)
|
||||
if not metadata.is_dir() or git(repository, "rev-parse", "--show-toplevel") != str(repository):
|
||||
raise ValueError("Candidate must be a standalone Git repository")
|
||||
if resolve_revision(repository, "refs/heads/init") != record["code"]["initial_commit"]:
|
||||
raise ValueError("The protected init branch has changed since preparation")
|
||||
git(repository, "merge-base", "--is-ancestor", record["code"]["initial_commit"], "HEAD")
|
||||
return repository, record
|
||||
|
||||
|
||||
def current_branch(repository: Path) -> str:
|
||||
"""Reject detached HEAD so every candidate commit remains branch-reachable."""
|
||||
branch = git(repository, "branch", "--show-current")
|
||||
validate_branch(repository, branch)
|
||||
return branch
|
||||
|
||||
|
||||
def validate_branch(repository: Path, branch: str) -> str:
|
||||
"""Branch names also serve as single evaluation-directory names."""
|
||||
if not branch or branch.startswith("-") or any(char in branch for char in "/\\\0\n\r"):
|
||||
raise ValueError("branch_name must be one path-safe local branch name")
|
||||
if git(repository, "check-ref-format", "--branch", branch) != branch:
|
||||
raise ValueError("Use a literal local branch name")
|
||||
return branch
|
||||
|
||||
|
||||
def require_clean(repository: Path) -> None:
|
||||
"""Do not carry uncommitted or untracked files across branch operations."""
|
||||
if git(repository, "status", "--porcelain=v2", "--untracked-files=all", "--ignore-submodules=none"):
|
||||
raise ValueError("Candidate repository must be clean before changing branches")
|
||||
if (repository / ".git/MERGE_HEAD").exists():
|
||||
raise ValueError("Finish the pending merge first")
|
||||
|
||||
|
||||
def branches(repository: Path) -> dict:
|
||||
"""Describe local refs and the current immutable commit."""
|
||||
local = sorted(git(repository, "for-each-ref", "--format=%(refname:short)", "refs/heads").splitlines())
|
||||
candidates = [branch for branch in local if branch != INITIAL_BRANCH]
|
||||
return {
|
||||
"current_branch": current_branch(repository),
|
||||
"commit_sha": resolve_revision(repository, "HEAD"),
|
||||
"local_branches": local,
|
||||
"candidate_branches": candidates,
|
||||
"candidate_branch_count": len(candidates),
|
||||
}
|
||||
|
||||
|
||||
def status(repository: Path, _record: dict) -> dict:
|
||||
"""Report exact paths, including names containing whitespace."""
|
||||
result = branches(repository)
|
||||
output = git(repository, "status", "--porcelain=v1", "-z", "--untracked-files=all", strip=False)
|
||||
entries = iter(output.split("\0") if output else [])
|
||||
changes = []
|
||||
for entry in entries:
|
||||
if not entry:
|
||||
continue
|
||||
change = {"status": entry[:2], "path": entry[3:]}
|
||||
if "R" in entry[:2] or "C" in entry[:2]:
|
||||
change["original_path"] = next(entries)
|
||||
changes.append(change)
|
||||
result.update(changes=changes, clean=not changes, merge_pending=(repository / ".git/MERGE_HEAD").exists())
|
||||
return result
|
||||
|
||||
|
||||
def create_branch(repository: Path, record: dict, *, branch_name: str) -> dict:
|
||||
"""Fork from the current candidate or baseline and switch to the new branch."""
|
||||
validate_branch(repository, branch_name)
|
||||
if branch_name in branches(repository)["local_branches"]:
|
||||
raise ValueError("Local branch already exists")
|
||||
require_clean(repository)
|
||||
validate_tree(repository, record, "HEAD")
|
||||
git(repository, "switch", "--no-track", "--create", branch_name)
|
||||
return {"status": "created", **branches(repository)}
|
||||
|
||||
|
||||
def switch_branch(repository: Path, record: dict, *, branch_name: str) -> dict:
|
||||
"""Switch only to an existing local ref, including the baseline."""
|
||||
validate_branch(repository, branch_name)
|
||||
if branch_name not in branches(repository)["local_branches"]:
|
||||
raise ValueError("Unknown local branch")
|
||||
require_clean(repository)
|
||||
validate_tree(repository, record, resolve_revision(repository, "refs/heads/" + branch_name))
|
||||
git(repository, "switch", "--no-guess", branch_name)
|
||||
return {"status": "switched", **branches(repository)}
|
||||
|
||||
|
||||
def validate_tree(repository: Path, record: dict, revision: str) -> None:
|
||||
"""Only declared editable roots may differ from the prepared baseline."""
|
||||
roots = [Path(relative_path(path)) for path in record["code"]["candidate_paths"]]
|
||||
changed = git(
|
||||
repository,
|
||||
"diff",
|
||||
"--no-ext-diff",
|
||||
"--no-textconv",
|
||||
"--name-only",
|
||||
"--no-renames",
|
||||
"-z",
|
||||
record["code"]["initial_commit"],
|
||||
revision,
|
||||
"--",
|
||||
strip=False,
|
||||
)
|
||||
for name in filter(None, changed.split("\0")):
|
||||
path = Path(relative_path(name))
|
||||
if ".git" in path.parts or not any(path.is_relative_to(root) for root in roots):
|
||||
raise ValueError(f"Change outside candidate_paths: {name}")
|
||||
for entry in filter(None, git(repository, "ls-tree", "-r", "-z", revision, strip=False).split("\0")):
|
||||
header, name = entry.split("\t", 1)
|
||||
mode, kind, _oid = header.split()
|
||||
if mode not in {"100644", "100755"} or kind != "blob" or ".git" in Path(relative_path(name)).parts:
|
||||
raise ValueError(f"Candidate must contain regular files only: {name}")
|
||||
|
||||
|
||||
def merge_branch(repository: Path, record: dict, *, branch_name: str) -> dict:
|
||||
"""Stage compatible candidates; automatically abort conflicts."""
|
||||
validate_branch(repository, branch_name)
|
||||
state = branches(repository)
|
||||
if state["current_branch"] == INITIAL_BRANCH:
|
||||
raise ValueError("Cannot merge into init")
|
||||
if branch_name in (INITIAL_BRANCH, state["current_branch"]):
|
||||
raise ValueError("Select another candidate branch")
|
||||
if branch_name not in state["candidate_branches"]:
|
||||
raise ValueError("Unknown candidate branch")
|
||||
require_clean(repository)
|
||||
validate_tree(repository, record, "HEAD")
|
||||
validate_tree(repository, record, resolve_revision(repository, "refs/heads/" + branch_name))
|
||||
try:
|
||||
git(
|
||||
repository,
|
||||
"-c",
|
||||
"user.name=Home",
|
||||
"-c",
|
||||
"user.email=home@localhost",
|
||||
"merge",
|
||||
"--no-commit",
|
||||
"--no-ff",
|
||||
branch_name,
|
||||
)
|
||||
except Exception:
|
||||
if (repository / ".git/MERGE_HEAD").exists():
|
||||
git(repository, "merge", "--abort")
|
||||
raise
|
||||
merge_status = "merged_not_committed" if (repository / ".git/MERGE_HEAD").exists() else "already_up_to_date"
|
||||
return {"status": merge_status, "source_branch": branch_name, **branches(repository)}
|
||||
|
||||
|
||||
def commit(repository: Path, record: dict, *, message: str) -> dict:
|
||||
"""Commit all candidate changes without rewriting existing history."""
|
||||
if current_branch(repository) == INITIAL_BRANCH:
|
||||
raise ValueError("Create a candidate branch before committing; init is protected")
|
||||
# Validate existing staged changes before staging anything else, then the complete index.
|
||||
validate_tree(repository, record, git(repository, "write-tree"))
|
||||
changes = status(repository, record)["changes"]
|
||||
roots = [Path(value) for value in record["code"]["candidate_paths"]]
|
||||
for change in changes:
|
||||
for name in (change["path"], change.get("original_path", change["path"])):
|
||||
path = Path(relative_path(name))
|
||||
if ".git" in path.parts or not any(path.is_relative_to(root) for root in roots):
|
||||
raise ValueError(f"Change outside candidate_paths: {name}")
|
||||
target = repository / path
|
||||
if target.is_symlink():
|
||||
raise ValueError(f"Candidate must contain regular files only: {name}")
|
||||
git(repository, "add", "--all", "--", ".")
|
||||
validate_tree(repository, record, git(repository, "write-tree"))
|
||||
git(repository, "-c", "user.name=Home", "-c", "user.email=home@localhost", "commit", "--message", message)
|
||||
return {"status": "committed", **branches(repository)}
|
||||
|
||||
|
||||
def diff(repository: Path, _record: dict, *, base_revision: str | None = None, revision: str | None = None) -> dict:
|
||||
"""Compare commits or all tracked changes against HEAD; never run external diff helpers."""
|
||||
if revision is not None and base_revision is None:
|
||||
raise ValueError("revision requires base_revision")
|
||||
base = resolve_revision(repository, base_revision or "HEAD")
|
||||
target = resolve_revision(repository, revision) if revision is not None else None
|
||||
output = git(
|
||||
repository,
|
||||
"diff",
|
||||
"--no-ext-diff",
|
||||
"--no-textconv",
|
||||
"--no-renames",
|
||||
"--ignore-submodules=all",
|
||||
base,
|
||||
*([target] if target else []),
|
||||
"--",
|
||||
)
|
||||
return {
|
||||
"base_revision": base,
|
||||
"revision": target,
|
||||
"content": output[:MAX_OUTPUT],
|
||||
"truncated": len(output) > MAX_OUTPUT,
|
||||
}
|
||||
143
plugins/home/src/reme_home/code/snapshot.py
Normal file
143
plugins/home/src/reme_home/code/snapshot.py
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
"""Bounded navigation of immutable Git objects, independent of the checkout."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from ..preparation.code import git, git_bytes
|
||||
from ..preparation.spec import relative_path
|
||||
from .repository import MAX_OUTPUT, resolve_revision
|
||||
|
||||
MAX_FILE_BYTES = 5_000_000
|
||||
MAX_SEARCH_BYTES = 100_000_000
|
||||
MAX_READ_LINES = 300
|
||||
|
||||
|
||||
def code_path(path: str) -> str:
|
||||
"""Accept literal repository-relative paths, never Git pathspec expressions."""
|
||||
value = relative_path(path)
|
||||
if ".git" in Path(value).parts or "\0" in value:
|
||||
raise ValueError("Invalid code path")
|
||||
return value
|
||||
|
||||
|
||||
def tree(repository: Path, revision: str, path: str | None = None) -> list[tuple[str, str]]:
|
||||
"""Read regular-file object IDs with NUL-delimited names and no archive filters."""
|
||||
# ls-tree does not support pathspec magic. Filter exact names/prefixes ourselves.
|
||||
entries = git(repository, "ls-tree", "-r", "-z", "--full-tree", revision, strip=False)
|
||||
prefix = code_path(path) if path is not None else None
|
||||
result = []
|
||||
for entry in filter(None, entries.split("\0")):
|
||||
header, name = entry.split("\t", 1)
|
||||
if prefix is not None and name != prefix and not name.startswith(prefix + "/"):
|
||||
continue
|
||||
mode, kind, oid = header.split()
|
||||
code_path(name)
|
||||
if mode not in {"100644", "100755"} or kind != "blob":
|
||||
raise ValueError(f"Code snapshot must contain regular files only: {name}")
|
||||
result.append((name, oid))
|
||||
return result
|
||||
|
||||
|
||||
def blob(repository: Path, oid: str) -> bytes:
|
||||
"""Check blob size before loading its bytes; never follow tracked symlinks."""
|
||||
if int(git(repository, "cat-file", "-s", oid)) > MAX_FILE_BYTES:
|
||||
raise ValueError("Code file exceeds 5 MB; narrow the path")
|
||||
return git_bytes(repository, "cat-file", "blob", oid)
|
||||
|
||||
|
||||
def list_code_tree(
|
||||
repository: Path,
|
||||
_record: dict,
|
||||
*,
|
||||
revision: str = "HEAD",
|
||||
path: str | None = None,
|
||||
max_results: int = 500,
|
||||
) -> dict:
|
||||
"""List tracked paths at a resolved commit."""
|
||||
revision = resolve_revision(repository, revision)
|
||||
entries = tree(repository, revision, path)
|
||||
return {
|
||||
"revision": revision,
|
||||
"path": path,
|
||||
"paths": [name for name, _ in entries[:max_results]],
|
||||
"truncated": len(entries) > max_results,
|
||||
}
|
||||
|
||||
|
||||
def read_code_file(
|
||||
repository: Path,
|
||||
_record: dict,
|
||||
*,
|
||||
path: str,
|
||||
revision: str = "HEAD",
|
||||
start_line: int = 1,
|
||||
end_line: int | None = None,
|
||||
) -> dict:
|
||||
"""Return at most 300 lines and 100,000 characters from one committed text file."""
|
||||
path = code_path(path)
|
||||
revision = resolve_revision(repository, revision)
|
||||
entries = dict(tree(repository, revision, path))
|
||||
if path not in entries:
|
||||
raise ValueError(f"No tracked regular file at {path}")
|
||||
data = blob(repository, entries[path])
|
||||
if b"\0" in data:
|
||||
raise ValueError("Code file is binary")
|
||||
lines = data.decode("utf-8", errors="replace").splitlines()
|
||||
requested_end = end_line if end_line is not None else start_line + MAX_READ_LINES - 1
|
||||
if requested_end < start_line:
|
||||
raise ValueError("end_line must be greater than or equal to start_line")
|
||||
displayed_end = min(requested_end, start_line + MAX_READ_LINES - 1, len(lines))
|
||||
content = "\n".join(lines[start_line - 1 : displayed_end])
|
||||
if content:
|
||||
content += "\n"
|
||||
truncated = len(content) > MAX_OUTPUT or (bool(content) and (start_line > 1 or displayed_end < len(lines)))
|
||||
content = content[:MAX_OUTPUT]
|
||||
if truncated:
|
||||
content += "\n<output truncated; request another line range to continue>"
|
||||
return {
|
||||
"revision": revision,
|
||||
"path": path,
|
||||
"start_line": start_line,
|
||||
"end_line": displayed_end,
|
||||
"total_lines": len(lines),
|
||||
"truncated": truncated,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
|
||||
def search_code(
|
||||
repository: Path,
|
||||
_record: dict,
|
||||
*,
|
||||
pattern: str,
|
||||
revision: str = "HEAD",
|
||||
path: str | None = None,
|
||||
max_results: int = 30,
|
||||
) -> dict:
|
||||
"""Search with Python multiline regex; isolate regex execution under a hard subprocess timeout."""
|
||||
revision = resolve_revision(repository, revision)
|
||||
files = {}
|
||||
total = 0
|
||||
for name, oid in tree(repository, revision, path):
|
||||
size = int(git(repository, "cat-file", "-s", oid))
|
||||
total += size
|
||||
if total > MAX_SEARCH_BYTES:
|
||||
raise ValueError("Code search exceeds 100 MB; narrow path")
|
||||
data = blob(repository, oid)
|
||||
if b"\0" not in data:
|
||||
files[name] = data.decode("utf-8", errors="replace")
|
||||
# Arbitrary Python regex can backtrack indefinitely. Run only the text matcher
|
||||
# in a short-lived stdlib-only process; it never imports or executes candidate code.
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-I", str(Path(__file__).with_name("regex_worker.py"))],
|
||||
input=json.dumps({"files": files, "pattern": pattern, "max_results": max_results}).encode(),
|
||||
capture_output=True,
|
||||
check=True,
|
||||
timeout=10,
|
||||
)
|
||||
payload = json.loads(result.stdout)
|
||||
if "error" in payload:
|
||||
raise ValueError(payload["error"])
|
||||
return {"revision": revision, "pattern": pattern, "path": path, **payload}
|
||||
22
plugins/home/src/reme_home/code/step.py
Normal file
22
plugins/home/src/reme_home/code/step.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
"""Thin Job adapters; operations are fixed by Step configuration, never model input."""
|
||||
|
||||
from reme.steps.base_step import BaseStep, Ref
|
||||
|
||||
from .component import HomeCode
|
||||
from .models import ARGUMENTS
|
||||
|
||||
|
||||
class HomeCodeStep(BaseStep):
|
||||
"""Expose one configured candidate operation using the normal ReMe Job lifecycle."""
|
||||
|
||||
code: HomeCode = Ref(HomeCode, "home.code")
|
||||
|
||||
async def execute(self):
|
||||
"""Leave repository location and operation selection under host control."""
|
||||
if self.context.get("_allowed_paths") is not None:
|
||||
raise ValueError("Home code Jobs require unrestricted access to their managed workspace")
|
||||
operation = self.kwargs["operation"]
|
||||
fields = ARGUMENTS[operation].model_fields
|
||||
arguments = {name: self.context.get(name) for name in fields if name in self.context}
|
||||
self.context.response.answer = await self.code.run(operation, **arguments)
|
||||
return self.context.response
|
||||
|
|
@ -4,6 +4,7 @@ import asyncio
|
|||
import os
|
||||
import shutil
|
||||
import traceback
|
||||
from contextlib import nullcontext
|
||||
|
||||
from deploy.benchmark_docker import BenchmarkDockerPool, CasePlan, DockerReMeSandboxFactory, SourceCandidate
|
||||
|
||||
|
|
@ -22,10 +23,12 @@ from .artifacts import (
|
|||
answer_exception,
|
||||
failure_kind,
|
||||
)
|
||||
from .inputs import EvaluationSpec, load_inputs, reuse_inputs, snapshot_code
|
||||
from .inputs import EvaluationSpec, load_inputs, reuse_inputs, snapshot_code, replay_sampling_eligibility
|
||||
from .compaction import compact_construction, compact_query
|
||||
from ..judge import HomeJudge
|
||||
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
|
||||
from ..sampling import SamplingPolicy, create_sampling_plan
|
||||
from .history import HomeEvaluationHistory
|
||||
|
||||
|
||||
async def blocking(function, *args, **kwargs):
|
||||
|
|
@ -43,11 +46,15 @@ class HomeEvaluation(BaseComponent):
|
|||
|
||||
component_type = "home.evaluation"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
def __init__(self, sampling_policy: dict | None = None, **kwargs):
|
||||
if "sampling" in kwargs:
|
||||
raise ValueError("Use sampling_policy for Component defaults; sampling is a Job boolean")
|
||||
super().__init__(**kwargs)
|
||||
self.workspace = self.bind("default", HomeWorkspace, optional=False)
|
||||
self.judge = self.bind("default", HomeJudge, optional=False)
|
||||
self.history = self.bind("default", HomeEvaluationHistory, optional=False)
|
||||
self._pending: set[asyncio.Task] = set()
|
||||
self.sampling_policy = SamplingPolicy.model_validate(sampling_policy if sampling_policy is not None else {})
|
||||
|
||||
async def _close(self):
|
||||
if self._pending:
|
||||
|
|
@ -58,7 +65,10 @@ class HomeEvaluation(BaseComponent):
|
|||
if not self.is_started:
|
||||
raise RuntimeError("Start the Home application before evaluation")
|
||||
spec = EvaluationSpec.model_validate(kwargs)
|
||||
task = asyncio.create_task(self._evaluate(spec))
|
||||
override = spec.sampling_policy.model_dump(exclude_unset=True) if spec.sampling_policy is not None else {}
|
||||
policy = SamplingPolicy.model_validate({**self.sampling_policy.model_dump(), **override})
|
||||
spec = spec.model_copy(update={"sampling_policy": policy})
|
||||
task = asyncio.create_task(self._serialized_evaluate(spec))
|
||||
self._pending.add(task)
|
||||
try:
|
||||
return await asyncio.shield(task)
|
||||
|
|
@ -69,11 +79,40 @@ class HomeEvaluation(BaseComponent):
|
|||
finally:
|
||||
self._pending.discard(task)
|
||||
|
||||
async def _serialized_evaluate(self, spec):
|
||||
"""Match dev_meta's serialized formal validations; debug never owns sampling state."""
|
||||
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
|
||||
lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock")
|
||||
with lock:
|
||||
return await self._evaluate(spec)
|
||||
|
||||
def _prepare(self, spec, environment):
|
||||
home = self.workspace.home_path
|
||||
sampling_state = self.history.sampling_state() if spec.sampling and not spec.debug else None
|
||||
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
|
||||
with _initialization_lock(self.workspace.state_path / "workspace.lock"):
|
||||
record, dataset, cases = load_inputs(home, spec)
|
||||
plan = None
|
||||
if spec.sampling and not spec.debug:
|
||||
eligibility = replay_sampling_eligibility(home, spec, cases)
|
||||
plan = create_sampling_plan(
|
||||
cases,
|
||||
sampling_state,
|
||||
spec.sampling_policy,
|
||||
eligible_query_ids=eligibility,
|
||||
)
|
||||
selected = {case.case_id: case for case in cases}
|
||||
cases = []
|
||||
for key in plan["case_ids"]:
|
||||
case = selected[key]
|
||||
queries = {query.query_id: query for query in case.queries}
|
||||
cases.append(
|
||||
case.model_copy(
|
||||
update={
|
||||
"queries": [queries[query_id] for query_id in plan["query_ids"][key]],
|
||||
},
|
||||
),
|
||||
)
|
||||
judge_identity = self.judge.identity(dataset.dataset)
|
||||
snapshot, commit, branch = snapshot_code(home, record)
|
||||
fingerprints = {
|
||||
|
|
@ -102,7 +141,22 @@ class HomeEvaluation(BaseComponent):
|
|||
if not isinstance(rubric, list) or not rubric:
|
||||
raise ValueError(f"BEAM query {query.query_id} has no scoring rubric")
|
||||
evaluation_query(dataset.dataset, query)
|
||||
return record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs
|
||||
return record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs, plan
|
||||
|
||||
def _sampling_started(self, run, manifest, plan, policy=None, *, record_history=True):
|
||||
"""The factory invokes this only after a real sandbox successfully initializes."""
|
||||
if not record_history:
|
||||
manifest["sandbox_started_at"] = utc_now().isoformat()
|
||||
if plan is not None:
|
||||
# Keep draw evidence local without claiming a started-history round.
|
||||
manifest["selection_plan"] = plan
|
||||
write_json(run / "manifest.json", manifest)
|
||||
return
|
||||
self.history.record_started(run, policy if policy is not None else self.sampling_policy, plan)
|
||||
manifest.update(read_json(run / "manifest.json"))
|
||||
|
||||
def _finish_sampling(self, run):
|
||||
self.history.record(run)
|
||||
|
||||
def _allocate(self, branch, commit, manifest):
|
||||
"""Allocate ascending IDs under a collision-safe commit prefix."""
|
||||
|
|
@ -130,7 +184,7 @@ class HomeEvaluation(BaseComponent):
|
|||
async def _evaluate(self, spec):
|
||||
environment = {**os.environ, **self.app_context.app_config.environment}
|
||||
prepared = await blocking(self._prepare, spec, environment)
|
||||
record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs = prepared
|
||||
record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs, sampling_plan = prepared
|
||||
judge = None if spec.debug else self.judge
|
||||
run = None
|
||||
try:
|
||||
|
|
@ -199,6 +253,17 @@ class HomeEvaluation(BaseComponent):
|
|||
plugins=tuple(record["code"]["plugins"]),
|
||||
command_timeout=spec.timeout_seconds,
|
||||
env=environment,
|
||||
on_sandbox_started=(
|
||||
None
|
||||
if spec.debug
|
||||
else lambda: self._sampling_started(
|
||||
run,
|
||||
manifest,
|
||||
sampling_plan,
|
||||
spec.sampling_policy,
|
||||
record_history=spec.record_history,
|
||||
)
|
||||
),
|
||||
)
|
||||
semaphore = asyncio.Semaphore(spec.concurrency)
|
||||
by_id = {case.case_id: case for case in cases}
|
||||
|
|
@ -242,20 +307,34 @@ class HomeEvaluation(BaseComponent):
|
|||
previous.update(cleanup_error=result.error, failure_kind="infra_error")
|
||||
write_json(failure, previous)
|
||||
summary = summarize(run, cases, scoring_required=not spec.debug)
|
||||
manifest.update(status="completed", finished_at=utc_now().isoformat())
|
||||
manifest.update(
|
||||
status="completed",
|
||||
success=summary["case_status_counts"]["infra_error"] == 0,
|
||||
finished_at=utc_now().isoformat(),
|
||||
)
|
||||
write_json(run / "manifest.json", manifest)
|
||||
write_json(run / "summary.json", summary, compact_leaf_containers=True)
|
||||
if not spec.debug and spec.record_history:
|
||||
await blocking(self._finish_sampling, run)
|
||||
return {
|
||||
"result_dir": str(run),
|
||||
"commit_sha": commit,
|
||||
"execution_mode": manifest["execution_mode"],
|
||||
"record_history": spec.record_history and not spec.debug,
|
||||
"status": "completed",
|
||||
**{key: value for key, value in summary.items() if key != "cases"},
|
||||
}
|
||||
except BaseException as error:
|
||||
# A full summary is the durable completion boundary. Cancellation
|
||||
# during drained history bookkeeping cancels the caller, not the
|
||||
# already completed evaluation or its successful history entry.
|
||||
registration_failed = run is not None and manifest.get("status") == "completed"
|
||||
if isinstance(error, asyncio.CancelledError) and registration_failed:
|
||||
raise
|
||||
if run is not None:
|
||||
manifest.update(
|
||||
status="cancelled" if isinstance(error, asyncio.CancelledError) else "failed",
|
||||
success=False,
|
||||
finished_at=utc_now().isoformat(),
|
||||
)
|
||||
write_json(run / "manifest.json", manifest)
|
||||
|
|
@ -274,6 +353,11 @@ class HomeEvaluation(BaseComponent):
|
|||
scoring_required=not spec.debug,
|
||||
)
|
||||
write_json(run / "partial_summary.json", partial, compact_leaf_containers=True)
|
||||
if registration_failed and not spec.debug and spec.record_history:
|
||||
try:
|
||||
await blocking(self.history.invalidate)
|
||||
except Exception as cache_error:
|
||||
error.add_note(f"History cache invalidation failed: {cache_error}")
|
||||
if isinstance(error, asyncio.CancelledError) or run is None:
|
||||
raise
|
||||
raise RuntimeError(f"Evaluation failed; artifacts: {run}: {error}") from error
|
||||
|
|
|
|||
345
plugins/home/src/reme_home/evaluation/history.py
Normal file
345
plugins/home/src/reme_home/evaluation/history.py
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
"""Successful evaluation history and rebuildable sampling evidence owned by Home."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Mapping, Literal
|
||||
import math
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from reme.components.base_component import BaseComponent
|
||||
|
||||
from ..preparation.data.models import utc_now
|
||||
from ..preparation.spec import SpecModel
|
||||
from ..sampling import SamplingPolicy, _empty_record, _observe, _query_key
|
||||
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
|
||||
from .artifacts import contained, read_json, write_json
|
||||
|
||||
|
||||
class EvaluationRecord(SpecModel):
|
||||
"""A compact catalog entry backed by an immutable successful result directory."""
|
||||
|
||||
run_key: str
|
||||
branch_name: str
|
||||
commit_sha: str
|
||||
validation_id: str
|
||||
round: int
|
||||
case_ids: list[str]
|
||||
query_ids: dict[str, list[str]]
|
||||
execution_mode: str
|
||||
finished_at: str
|
||||
case_count: int
|
||||
candidate_failure_count: int
|
||||
scored_query_count: int
|
||||
mean_query_score: float | None
|
||||
|
||||
|
||||
class HistoryIndex(SpecModel):
|
||||
"""A disposable index; manifests, summaries and plans remain authoritative."""
|
||||
|
||||
schema_version: Literal[1] = 1
|
||||
results: list[EvaluationRecord] = Field(default_factory=list)
|
||||
|
||||
|
||||
def successful_result(home: Path, run: Path) -> tuple[dict, dict] | None:
|
||||
"""Match the non-debug Response.success contract, including valid low scores."""
|
||||
manifest = read_json(contained(home, run / "manifest.json"))
|
||||
if manifest.get("options", {}).get("record_history") is False:
|
||||
return None
|
||||
if (
|
||||
manifest.get("mode") != "scored"
|
||||
or manifest.get("dataset_kind") != "training-set"
|
||||
or manifest.get("status") != "completed"
|
||||
or manifest.get("success") is False
|
||||
or not manifest.get("sandbox_started_at")
|
||||
):
|
||||
return None
|
||||
path = contained(home, run / "summary.json")
|
||||
if not path.is_file():
|
||||
return None
|
||||
summary = read_json(path)
|
||||
if summary.get("case_status_counts", {}).get("infra_error") != 0 or not summary.get("case_count"):
|
||||
return None
|
||||
return manifest, summary
|
||||
|
||||
|
||||
def result_record(home: Path, run: Path) -> EvaluationRecord | None:
|
||||
"""Exclude failed, cancelled, debug and incomplete directories from the catalog."""
|
||||
result = successful_result(home, run)
|
||||
if result is None:
|
||||
return None
|
||||
manifest, summary = result
|
||||
return EvaluationRecord(
|
||||
run_key=run.relative_to(home / "evaluations").as_posix(),
|
||||
branch_name=manifest["branch_name"],
|
||||
commit_sha=manifest["commit_sha"],
|
||||
validation_id=manifest["validation_id"],
|
||||
round=manifest["sampling"]["round"],
|
||||
case_ids=manifest["case_ids"],
|
||||
query_ids=manifest["query_ids"],
|
||||
execution_mode=manifest["execution_mode"],
|
||||
finished_at=manifest["finished_at"],
|
||||
case_count=summary["case_count"],
|
||||
candidate_failure_count=summary["case_status_counts"]["candidate_failure"],
|
||||
scored_query_count=summary["scored_query_count"],
|
||||
mean_query_score=summary["mean_query_score"],
|
||||
)
|
||||
|
||||
|
||||
def read_state(home: Path) -> dict:
|
||||
"""Read a derived cache; started plans and scored artifacts retain its source evidence."""
|
||||
path = contained(home, "sampling/state.json")
|
||||
if not path.exists():
|
||||
return rebuild_sampling_state(home, persist=False)
|
||||
payload = read_json(path)
|
||||
if not isinstance(payload, dict) or payload.get("schema_version") not in {1, 2}:
|
||||
raise ValueError("Unsupported sampling state schema")
|
||||
if not isinstance(payload.get("round"), int) or isinstance(payload["round"], bool) or payload["round"] < 0:
|
||||
raise ValueError("Invalid sampling round")
|
||||
for name in ("cases", "queries", "started_validations"):
|
||||
if not isinstance(payload.get(name, {}), dict):
|
||||
raise ValueError(f"Invalid sampling {name}")
|
||||
payload.setdefault(name, {})
|
||||
if not isinstance(payload.get("processed_validations"), list):
|
||||
raise ValueError("Invalid processed validations")
|
||||
if payload["schema_version"] == 1 or "successful_round" not in payload:
|
||||
return rebuild_sampling_state(home, persist=False)
|
||||
if (
|
||||
not isinstance(payload["successful_round"], int)
|
||||
or isinstance(payload["successful_round"], bool)
|
||||
or (payload["successful_round"] < 0)
|
||||
):
|
||||
raise ValueError("Invalid successful-history round")
|
||||
return payload
|
||||
|
||||
|
||||
def start_validation(home: Path, run: Path, policy: SamplingPolicy, plan: dict | None = None) -> dict:
|
||||
"""Record once after the first successfully initialized sandbox, never before."""
|
||||
manifest = read_json(contained(home, run / "manifest.json"))
|
||||
if (
|
||||
manifest.get("mode") == "debug"
|
||||
or manifest.get("dataset_kind") != "training-set"
|
||||
or manifest.get("options", {}).get("record_history") is False
|
||||
):
|
||||
return {}
|
||||
key = run.relative_to(home / "evaluations").as_posix()
|
||||
state = read_state(home)
|
||||
if key in state["started_validations"]:
|
||||
return read_json(contained(home, manifest["sampling"]["plan_path"]))
|
||||
plan = (
|
||||
dict(plan)
|
||||
if plan is not None
|
||||
else {
|
||||
"schema_version": 1,
|
||||
"mode": "explicit",
|
||||
"case_ids": manifest["case_ids"],
|
||||
"query_ids": manifest["query_ids"],
|
||||
"policy": policy.model_dump(),
|
||||
}
|
||||
)
|
||||
# Preserve the draw's state snapshot separately from the committed start round.
|
||||
if "round" in plan:
|
||||
plan["selection_round"] = plan["round"]
|
||||
plan.update(
|
||||
round=state["round"] + 1,
|
||||
validation_id=manifest["validation_id"],
|
||||
validation_key=key,
|
||||
sandbox_started_at=utc_now().isoformat(),
|
||||
)
|
||||
relative = f"sampling/plans/{key}.json"
|
||||
write_json(contained(home, relative), plan)
|
||||
manifest["sampling"] = {"round": plan["round"], "mode": plan["mode"], "plan_path": relative}
|
||||
manifest["sandbox_started_at"] = plan["sandbox_started_at"]
|
||||
write_json(run / "manifest.json", manifest)
|
||||
state["round"] = plan["round"]
|
||||
state["started_validations"][key] = plan["round"]
|
||||
write_json(contained(home, "sampling/state.json"), state)
|
||||
return plan
|
||||
|
||||
|
||||
def update_sampling_state(home: Path, run: Path, *, _state: dict | None = None) -> None:
|
||||
"""Update observations only from successful completed formal evaluations."""
|
||||
result = successful_result(home, run)
|
||||
if result is None:
|
||||
return
|
||||
manifest, summary = result
|
||||
plan = read_json(contained(home, manifest["sampling"]["plan_path"]))
|
||||
policy = SamplingPolicy.model_validate(plan["policy"])
|
||||
state = read_state(home) if _state is None else _state
|
||||
key = plan["validation_key"]
|
||||
if key in state["processed_validations"]:
|
||||
return
|
||||
round_id = state["successful_round"] + 1
|
||||
cases, queries = state["cases"], state["queries"]
|
||||
summary_cases = summary.get("cases", {})
|
||||
items = (
|
||||
summary_cases.items()
|
||||
if isinstance(summary_cases, Mapping)
|
||||
else ((case.get("case_id"), case) for case in summary_cases)
|
||||
)
|
||||
for case_id, case_summary in items:
|
||||
if not isinstance(case_id, str) or not isinstance(case_summary, Mapping):
|
||||
continue
|
||||
folder = contained(run, f"cases/{case_id}", directory=True)
|
||||
query_path = contained(folder, "queries/result.json")
|
||||
query_rows = read_json(query_path).get("queries", []) if query_path.is_file() else []
|
||||
failure_path = contained(folder, "failure.json")
|
||||
failure = read_json(failure_path) if failure_path.is_file() else {}
|
||||
# Home summaries omit the redundant case status; reconstruct it using
|
||||
# the same authoritative failure/query evidence as summarize().
|
||||
status = case_summary.get("status") or (
|
||||
"infra_error"
|
||||
if failure.get("failure_kind") == "infra_error"
|
||||
or any(query.get("infrastructure_error") for query in query_rows)
|
||||
else "candidate_failure" if failure or any(query.get("error") for query in query_rows) else "completed"
|
||||
)
|
||||
case_record = cases.setdefault(case_id, _empty_record())
|
||||
case_record["last_selected_round"] = max(case_record.get("last_selected_round") or 0, round_id)
|
||||
if status == "infra_error":
|
||||
case_record["infra_error_count"] += 1
|
||||
continue
|
||||
if status == "candidate_failure":
|
||||
case_record["candidate_failure_count"] += 1
|
||||
scores = []
|
||||
for query in query_rows:
|
||||
query_id = query.get("query_id")
|
||||
if not isinstance(query_id, str):
|
||||
continue
|
||||
query_record = queries.setdefault(_query_key(case_id, query_id), _empty_record())
|
||||
query_record["last_selected_round"] = max(query_record.get("last_selected_round") or 0, round_id)
|
||||
if query.get("infrastructure_error"):
|
||||
query_record["infra_error_count"] += 1
|
||||
continue
|
||||
if query.get("failure_kind") == "candidate_failure":
|
||||
query_record["candidate_failure_count"] += 1
|
||||
score = query.get("score")
|
||||
if isinstance(score, (int, float)) and not isinstance(score, bool) and math.isfinite(score):
|
||||
scores.append(float(score))
|
||||
_observe(query_record, 1.0 - score, round_id, policy.ewma_alpha)
|
||||
if scores:
|
||||
_observe(case_record, 1.0 - sum(scores) / len(scores), round_id, policy.ewma_alpha)
|
||||
state["successful_round"] = round_id
|
||||
state["processed_validations"].append(key)
|
||||
if _state is None:
|
||||
write_json(contained(home, "sampling/state.json"), state)
|
||||
|
||||
|
||||
def rebuild_sampling_state(home: Path, *, persist: bool = True) -> dict:
|
||||
"""Rebuild the disposable cache from immutable started plans and scored artifacts."""
|
||||
state = {
|
||||
"schema_version": 2,
|
||||
"round": 0,
|
||||
"successful_round": 0,
|
||||
"cases": {},
|
||||
"queries": {},
|
||||
"started_validations": {},
|
||||
"processed_validations": [],
|
||||
}
|
||||
runs = []
|
||||
for path in contained(home, "evaluations", directory=True).glob("*/*/*/manifest.json"):
|
||||
manifest = read_json(contained(home, path))
|
||||
if (
|
||||
manifest.get("mode") == "debug"
|
||||
or manifest.get("dataset_kind") != "training-set"
|
||||
or manifest.get("options", {}).get("record_history") is False
|
||||
or not (manifest.get("sandbox_started_at"))
|
||||
):
|
||||
continue
|
||||
plan = read_json(contained(home, manifest["sampling"]["plan_path"]))
|
||||
runs.append((plan["round"], path.parent, plan["validation_key"]))
|
||||
for round_id, run, key in sorted(runs):
|
||||
state["round"] = max(state["round"], round_id)
|
||||
state["started_validations"][key] = round_id
|
||||
update_sampling_state(home, run, _state=state)
|
||||
if persist:
|
||||
write_json(contained(home, "sampling/state.json"), state)
|
||||
return state
|
||||
|
||||
|
||||
class HomeEvaluationHistory(BaseComponent):
|
||||
"""Own successful-result lookup, registration and sampling caches for one workspace."""
|
||||
|
||||
component_type = "home.evaluation_history"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.workspace = self.bind("default", HomeWorkspace, optional=False)
|
||||
|
||||
def _history_lock(self):
|
||||
if not self.is_started:
|
||||
raise RuntimeError("Start the Home application before reading evaluation history")
|
||||
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
|
||||
return _initialization_lock(self.workspace.state_path / "workspace.lock")
|
||||
|
||||
def _run_path(self, run: Path) -> Path:
|
||||
home = self.workspace.home_path
|
||||
run = contained(home, run, directory=True)
|
||||
relative = run.relative_to(home / "evaluations")
|
||||
if len(relative.parts) != 3:
|
||||
raise ValueError("Expected one evaluation result directory")
|
||||
return run
|
||||
|
||||
def _index(self, *, rebuild: bool = False) -> HistoryIndex:
|
||||
home = self.workspace.home_path
|
||||
path = contained(home, "evaluations/history.json")
|
||||
if path.exists() and not rebuild:
|
||||
return HistoryIndex.model_validate(read_json(path))
|
||||
records = []
|
||||
for manifest in contained(home, "evaluations", directory=True).glob("*/*/*/manifest.json"):
|
||||
record = result_record(home, contained(home, manifest.parent, directory=True))
|
||||
if record is not None:
|
||||
records.append(record)
|
||||
return HistoryIndex(results=sorted(records, key=lambda record: (record.round, record.run_key)))
|
||||
|
||||
def sampling_state(self) -> dict:
|
||||
"""Return owned, reconstructed-on-demand observations for the next draw."""
|
||||
with self._history_lock():
|
||||
state = read_state(self.workspace.home_path)
|
||||
return {"round": state["successful_round"], "cases": state["cases"], "queries": state["queries"]}
|
||||
|
||||
def record_started(self, run: Path, policy: SamplingPolicy, plan: dict | None = None) -> dict:
|
||||
"""Record a started round, independently of whether it later enters successful history."""
|
||||
with self._history_lock():
|
||||
return start_validation(self.workspace.home_path, self._run_path(run), policy, plan)
|
||||
|
||||
def record(self, run: Path) -> EvaluationRecord | None:
|
||||
"""Register a successful result once; unsuccessful runs never train the sampler."""
|
||||
with self._history_lock():
|
||||
home = self.workspace.home_path
|
||||
run = self._run_path(run)
|
||||
record = result_record(home, run)
|
||||
if record is None:
|
||||
return None
|
||||
index = self._index()
|
||||
by_key = {item.run_key: item for item in index.results}
|
||||
by_key[record.run_key] = record
|
||||
update_sampling_state(home, run)
|
||||
index = HistoryIndex(results=sorted(by_key.values(), key=lambda item: (item.round, item.run_key)))
|
||||
write_json(contained(home, "evaluations/history.json"), index.model_dump(mode="json"))
|
||||
return record
|
||||
|
||||
def list_results(self) -> list[EvaluationRecord]:
|
||||
"""Return successful results in chronological order, including low-scoring candidates."""
|
||||
with self._history_lock():
|
||||
return self._index().results
|
||||
|
||||
def get_result(self, run_key: str) -> EvaluationRecord:
|
||||
"""Look up a successful result by its stable workspace-relative identity."""
|
||||
for result in self.list_results():
|
||||
if result.run_key == run_key:
|
||||
return result
|
||||
raise KeyError(f"No successful evaluation: {run_key}")
|
||||
|
||||
def rebuild(self) -> list[EvaluationRecord]:
|
||||
"""Rebuild only derived indexes and sampling observations; never rewrite source evidence."""
|
||||
with self._history_lock():
|
||||
home = self.workspace.home_path
|
||||
index = self._index(rebuild=True)
|
||||
rebuild_sampling_state(home)
|
||||
write_json(contained(home, "evaluations/history.json"), index.model_dump(mode="json"))
|
||||
return index.results
|
||||
|
||||
def invalidate(self) -> None:
|
||||
"""Discard only derived caches after an unsuccessful registration; retain all source evidence."""
|
||||
with self._history_lock():
|
||||
for name in ("evaluations/history.json", "sampling/state.json"):
|
||||
contained(self.workspace.home_path, name).unlink(missing_ok=True)
|
||||
|
|
@ -16,6 +16,8 @@ from ..preparation.code import git, tree_fingerprint
|
|||
from ..preparation.data.models import CaseSpec, DatasetManifest, fingerprint
|
||||
from .artifacts import contained, file_sha256, inspect_archive, read_json
|
||||
from ..preparation.spec import SpecModel
|
||||
from ..code.repository import validate_repository
|
||||
from ..sampling import SamplingPolicy
|
||||
|
||||
|
||||
class EvaluationSpec(SpecModel):
|
||||
|
|
@ -25,6 +27,9 @@ class EvaluationSpec(SpecModel):
|
|||
query_ids: dict[str, list[str]] | None = None
|
||||
reuse_construction_from: str | None = Field(default=None, min_length=1)
|
||||
debug: bool = False
|
||||
sampling: bool = False
|
||||
record_history: bool = True
|
||||
sampling_policy: SamplingPolicy | None = None
|
||||
concurrency: int = Field(default=4, ge=1)
|
||||
timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False)
|
||||
base_image: str = Field(default="reme-benchmark-base:agentscope-2.0.10", min_length=1)
|
||||
|
|
@ -64,6 +69,8 @@ class EvaluationSpec(SpecModel):
|
|||
"""Debug stops on failure by default and never reuses scored evidence."""
|
||||
if self.debug and self.reuse_construction_from:
|
||||
raise ValueError("Construction reuse requires training-set")
|
||||
if self.sampling and not self.debug and (self.case_ids is not None or self.query_ids is not None):
|
||||
raise ValueError("sampling=true selects case/query IDs automatically; omit explicit selections")
|
||||
if "fail_fast" not in self.model_fields_set:
|
||||
self.fail_fast = self.debug
|
||||
return self
|
||||
|
|
@ -119,7 +126,7 @@ def load_inputs(home: Path, spec: EvaluationSpec):
|
|||
|
||||
def snapshot_code(home: Path, record: dict):
|
||||
"""Combine fixed runtime with clean current HEAD; reject all uncommitted changes."""
|
||||
repository = contained(home, "code/repo", directory=True)
|
||||
repository, _ = validate_repository(home, record)
|
||||
runtime = contained(home, "code/runtime", directory=True)
|
||||
if tree_fingerprint(runtime) != record["code"]["runtime_fingerprint"]:
|
||||
raise ValueError("Fixed runtime has changed since preparation")
|
||||
|
|
@ -235,3 +242,49 @@ def reuse_inputs(home: Path, spec: EvaluationSpec, cases: list, fingerprints: di
|
|||
if errors:
|
||||
raise ValueError("Invalid construction snapshots: " + "; ".join(errors))
|
||||
return {"result_dir": str(source), "commit_sha": manifest["commit_sha"]}, archives
|
||||
|
||||
|
||||
def replay_sampling_eligibility(home: Path, spec: EvaluationSpec, cases: list[CaseSpec]) -> dict | None:
|
||||
"""Port dev_meta's replay eligibility without restricting future full rebuilds."""
|
||||
if spec.reuse_construction_from is None:
|
||||
return None
|
||||
source = contained(home, spec.reuse_construction_from, directory=True)
|
||||
evaluations = contained(home, "evaluations", directory=True)
|
||||
if not source.is_relative_to(evaluations) or source == evaluations:
|
||||
raise ValueError("reuse_construction_from must be inside home/evaluations")
|
||||
manifest = read_json(contained(source, "manifest.json"))
|
||||
summary = read_json(contained(source, "summary.json"))
|
||||
if (
|
||||
manifest.get("dataset_kind") != "training-set"
|
||||
or manifest.get("mode") == "debug"
|
||||
or (manifest.get("status", summary.get("status")) != "completed")
|
||||
):
|
||||
raise ValueError("Replay sampling requires a completed scored training-set run")
|
||||
case_ids = manifest.get("case_ids")
|
||||
if not isinstance(case_ids, list) or not case_ids or len(set(case_ids)) != len(case_ids):
|
||||
raise ValueError("Replay source must select unique nonempty cases")
|
||||
reusable = set()
|
||||
for case_id in case_ids:
|
||||
entity_id(case_id)
|
||||
folder = contained(source, f"cases/{case_id}", directory=True)
|
||||
result_path = contained(folder, "memory_construction/result.json")
|
||||
archive = contained(folder, "memory_construction/reme_workspace.tar.gz")
|
||||
if not result_path.is_file() or not archive.is_file() or read_json(result_path).get("success") is not True:
|
||||
continue
|
||||
reusable.add(case_id)
|
||||
if (len(case_ids) - len(reusable)) / len(case_ids) > 0.05:
|
||||
raise ValueError("Replay source construction failure ratio exceeds 5%")
|
||||
for case_id in reusable:
|
||||
payload = read_json(contained(source, f"cases/{case_id}/queries/result.json"))
|
||||
queries = payload.get("queries")
|
||||
if not queries or not all(
|
||||
isinstance(query, dict)
|
||||
and query.get("error") is None
|
||||
and isinstance(query.get("answer"), str)
|
||||
and query["answer"].strip()
|
||||
for query in queries
|
||||
):
|
||||
raise ValueError(f"Replay source has no completed successful answers for {case_id}")
|
||||
# A construction snapshot can answer new query subsets from the same case;
|
||||
# dev_meta intentionally considers every installed query of a reusable case.
|
||||
return {case.case_id: [query.query_id for query in case.queries] for case in cases if case.case_id in reusable}
|
||||
|
|
|
|||
358
plugins/home/src/reme_home/evaluation/inspection_step.py
Normal file
358
plugins/home/src/reme_home/evaluation/inspection_step.py
Normal file
|
|
@ -0,0 +1,358 @@
|
|||
"""Read-only evaluation navigation Jobs, independent of optimization history."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
|
||||
from reme.steps.base_step import BaseStep, Ref
|
||||
|
||||
from ..code.repository import current_branch, resolve_revision, validate_branch, validate_repository
|
||||
from ..preparation.code import git
|
||||
from ..preparation.spec import SpecModel
|
||||
from ..workspace import HomeWorkspace, _initialization_lock
|
||||
from .artifacts import contained, read_json
|
||||
from .inputs import entity_id
|
||||
|
||||
TOTAL_FIELDS = (
|
||||
"schema_version",
|
||||
"scoring_policy",
|
||||
"case_count",
|
||||
"case_status_counts",
|
||||
"construction_status_counts",
|
||||
"failure_stage_case_counts",
|
||||
"expected_query_count",
|
||||
"scored_query_count",
|
||||
"query_status_counts",
|
||||
"mean_query_score",
|
||||
"observed_mean_query_score",
|
||||
)
|
||||
|
||||
|
||||
class ListValidationResultsInput(SpecModel):
|
||||
"""Filters affect catalog navigation, never which evaluations enter history."""
|
||||
|
||||
branch_name: str | None = None
|
||||
commit_sha: str | None = Field(default=None, pattern=r"^[0-9a-fA-F]+$")
|
||||
offset: int = Field(default=0, ge=0)
|
||||
limit: int = Field(default=20, ge=1, le=100)
|
||||
|
||||
@field_validator("branch_name")
|
||||
@classmethod
|
||||
def validate_branch_name(cls, value):
|
||||
"""Keep historical branch filters within one directory segment."""
|
||||
return entity_id(value) if value is not None else None
|
||||
|
||||
|
||||
class InspectValidationInput(SpecModel):
|
||||
"""A query belongs to a case and every inspection belongs to one run."""
|
||||
|
||||
validation_path: str = Field(min_length=1)
|
||||
case_id: str | None = None
|
||||
query_id: str | None = None
|
||||
offset: int = Field(default=0, ge=0)
|
||||
limit: int = Field(default=20, ge=1, le=100)
|
||||
|
||||
@field_validator("case_id", "query_id")
|
||||
@classmethod
|
||||
def validate_entity(cls, value):
|
||||
"""Reject path traversal before resolving any artifact."""
|
||||
return entity_id(value) if value is not None else None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_selection(self):
|
||||
"""Require a case when selecting one query."""
|
||||
if self.query_id is not None and self.case_id is None:
|
||||
raise ValueError("query_id requires case_id")
|
||||
return self
|
||||
|
||||
|
||||
def _preview(value, limit=600):
|
||||
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
|
||||
return value if len(text) <= limit else text[:limit] + "… [truncated; read source field]"
|
||||
|
||||
|
||||
def _pointer(value):
|
||||
return str(value).replace("~", "~0").replace("/", "~1")
|
||||
|
||||
|
||||
def _page(rows, spec):
|
||||
return {
|
||||
"row_count": len(rows),
|
||||
"rows": rows[spec.offset : spec.offset + spec.limit],
|
||||
"next_offset": spec.offset + spec.limit if spec.offset + spec.limit < len(rows) else None,
|
||||
}
|
||||
|
||||
|
||||
def _object(root, relative, *, required=False):
|
||||
path = contained(root, relative)
|
||||
if not path.exists() and not required:
|
||||
return None
|
||||
payload = read_json(path)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"Expected JSON object: {path}")
|
||||
return payload
|
||||
|
||||
|
||||
def _summary(run):
|
||||
for name in ("summary.json", "partial_summary.json"):
|
||||
value = _object(run, name)
|
||||
if value is not None:
|
||||
return value, str(run / name)
|
||||
return None, None
|
||||
|
||||
|
||||
def _run_path(home, value):
|
||||
run = contained(home, value, directory=True)
|
||||
relative = run.relative_to(home / "evaluations")
|
||||
if len(relative.parts) != 3 or not run.is_dir():
|
||||
raise ValueError("validation_path must identify one directory inside home/evaluations")
|
||||
return run
|
||||
|
||||
|
||||
def _commit_filter(home, spec):
|
||||
if spec.commit_sha is None:
|
||||
return spec.branch_name, None
|
||||
# Resolve a local branch first, then match ALL of its reachable commits.
|
||||
# Git's global abbreviation rules could otherwise reject a branch-unique prefix.
|
||||
with _initialization_lock(contained(home, "metadata/workspace.lock")):
|
||||
repo, _ = validate_repository(home)
|
||||
branch = spec.branch_name if spec.branch_name is not None else current_branch(repo)
|
||||
validate_branch(repo, branch)
|
||||
tip = resolve_revision(repo, "refs/heads/" + branch)
|
||||
prefix = spec.commit_sha.lower()
|
||||
matches = [sha for sha in git(repo, "rev-list", tip, "--").splitlines() if sha.startswith(prefix)]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"commit_sha prefix {prefix!r} has {len(matches)} matches on branch {branch!r}; require one")
|
||||
return branch, matches[0]
|
||||
|
||||
|
||||
def _catalog_row(run, manifest):
|
||||
summary, summary_path = _summary(run)
|
||||
options = manifest.get("options", {})
|
||||
row = {
|
||||
"validation_path": str(run),
|
||||
"manifest_path": str(run / "manifest.json"),
|
||||
**{
|
||||
key: manifest.get(key)
|
||||
for key in (
|
||||
"branch_name",
|
||||
"commit_sha",
|
||||
"validation_id",
|
||||
"status",
|
||||
"success",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
"dataset",
|
||||
"dataset_kind",
|
||||
"mode",
|
||||
"execution_mode",
|
||||
"construction_source",
|
||||
"fingerprints",
|
||||
)
|
||||
},
|
||||
"sampling": options.get("sampling", False),
|
||||
"record_history": options.get("record_history", True) and manifest.get("mode") != "debug",
|
||||
"selected_case_count": len(manifest.get("case_ids", [])),
|
||||
"selected_query_count": sum(len(ids) for ids in manifest.get("query_ids", {}).values()),
|
||||
"summary_path": summary_path,
|
||||
"summary_missing": summary is None,
|
||||
"totals": {key: summary[key] for key in TOTAL_FIELDS if key in summary} if summary else {},
|
||||
}
|
||||
failure = _object(run, "failure.json")
|
||||
row["failure"] = _preview(failure)
|
||||
row["failure_path"] = str(run / "failure.json") if failure is not None else None
|
||||
return row
|
||||
|
||||
|
||||
def _list_results(home, spec):
|
||||
branch, commit = _commit_filter(home, spec)
|
||||
evaluations = contained(home, "evaluations", directory=True)
|
||||
rows = []
|
||||
for path in evaluations.glob("*/*/*/manifest.json"):
|
||||
if branch is not None and path.parent.parent.parent.name != branch:
|
||||
continue
|
||||
row = {"validation_path": str(path.parent), "manifest_path": str(path)}
|
||||
try:
|
||||
run = _run_path(home, path.parent)
|
||||
manifest = _object(run, "manifest.json", required=True)
|
||||
row.update({key: manifest.get(key) for key in ("branch_name", "commit_sha", "started_at")})
|
||||
if commit is not None and row["commit_sha"] != commit:
|
||||
continue
|
||||
row = _catalog_row(run, manifest)
|
||||
except (ValueError, OSError, TypeError, KeyError, AttributeError) as error:
|
||||
row["read_error"] = _preview(str(error))
|
||||
if commit is None or row.get("commit_sha") == commit:
|
||||
rows.append(row)
|
||||
rows.sort(key=lambda row: (str(row.get("started_at") or ""), row["validation_path"]), reverse=True)
|
||||
return {"branch_name": branch, "commit_sha": commit, **_page(rows, spec)}
|
||||
|
||||
|
||||
def _case_rows(manifest, summary):
|
||||
cases = (summary or {}).get("cases", {})
|
||||
indexed = (
|
||||
{key: (value, f"/cases/{_pointer(key)}") for key, value in cases.items()}
|
||||
if isinstance(cases, dict)
|
||||
else {value["case_id"]: (value, f"/cases/{index}") for index, value in enumerate(cases)}
|
||||
)
|
||||
rows = []
|
||||
for case_id in manifest.get("case_ids", []):
|
||||
case, pointer = indexed.get(case_id, ({}, None))
|
||||
rows.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"mean_score": case.get("mean_score"),
|
||||
"construction": case.get("construction"),
|
||||
"construction_error_count": case.get("construction_error_count", 0),
|
||||
"query_error_count": case.get("query_error_count", 0),
|
||||
"query_tokens": case.get("average_query_total_token_usage"),
|
||||
"missing": pointer is None,
|
||||
"pointer": pointer,
|
||||
},
|
||||
)
|
||||
rows.sort(
|
||||
key=lambda row: (
|
||||
row["mean_score"] is not None,
|
||||
row["mean_score"] if row["mean_score"] is not None else -1,
|
||||
-row["construction_error_count"],
|
||||
row["case_id"],
|
||||
),
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _case_inspection(run, manifest, spec):
|
||||
folder = contained(run, f"cases/{spec.case_id}", directory=True)
|
||||
build_path, query_path = "memory_construction/result.json", "queries/result.json"
|
||||
build, payload = _object(folder, build_path), _object(folder, query_path)
|
||||
failures = []
|
||||
jobs = (build or {}).get("jobs", [])
|
||||
for index, job in enumerate(jobs):
|
||||
outcome = job.get("result") or {}
|
||||
if outcome.get("success") is False:
|
||||
failures.append(
|
||||
{
|
||||
"job": job.get("job"),
|
||||
"error": _preview(outcome.get("error")),
|
||||
"pointer": f"/jobs/{index}/result",
|
||||
},
|
||||
)
|
||||
queries = (payload or {}).get("queries", [])
|
||||
by_id = {query["query_id"]: (query, index) for index, query in enumerate(queries)}
|
||||
selected = manifest["query_ids"][spec.case_id]
|
||||
if spec.query_id is not None and spec.query_id not in selected:
|
||||
raise ValueError("query_id is not selected by this evaluation manifest")
|
||||
rows = []
|
||||
for query_id in ([spec.query_id] if spec.query_id is not None else selected):
|
||||
query, index = by_id.get(query_id, ({}, None))
|
||||
row = {
|
||||
"query_id": query_id,
|
||||
"score": query.get("score"),
|
||||
"question_type": query.get("question_type"),
|
||||
"failure_kind": query.get("failure_kind"),
|
||||
"error": _preview(query.get("error")),
|
||||
"answer_success": (query.get("answer_result") or {}).get("success"),
|
||||
"judge_success": (query.get("judge_result") or {}).get("success"),
|
||||
"missing": index is None,
|
||||
"pointer": f"/queries/{index}" if index is not None else None,
|
||||
}
|
||||
if spec.query_id is not None:
|
||||
row["fields"] = {
|
||||
key: _preview(query[key], 2000)
|
||||
for key in (
|
||||
"question",
|
||||
"golden_answer",
|
||||
"answer",
|
||||
"score",
|
||||
"question_type",
|
||||
"answer_result",
|
||||
"judge_result",
|
||||
"error",
|
||||
"exception",
|
||||
"failure_kind",
|
||||
"infrastructure_error",
|
||||
)
|
||||
if key in query
|
||||
}
|
||||
rows.append(row)
|
||||
result = {
|
||||
"case_id": spec.case_id,
|
||||
"missing_files": [name for name, value in ((build_path, build), (query_path, payload)) if value is None],
|
||||
"case_failure": _preview(_object(folder, "failure.json")),
|
||||
"construction": {
|
||||
"source": str(folder / build_path),
|
||||
"success": (build or {}).get("success"),
|
||||
"status": (build or {}).get("status"),
|
||||
"job_count": len(jobs),
|
||||
"failed_jobs": _page(failures, spec),
|
||||
},
|
||||
"build_log": str(contained(folder, "memory_construction/build.log")),
|
||||
"memory_workspace": str(contained(folder, "memory_construction/reme_workspace", directory=True)),
|
||||
"rows_source": str(folder / query_path),
|
||||
**_page(rows, spec),
|
||||
}
|
||||
if spec.query_id is not None:
|
||||
query_folder = contained(folder, f"queries/{spec.query_id}", directory=True)
|
||||
result.update(
|
||||
query_missing=rows[0]["missing"],
|
||||
answer_log=str(contained(query_folder, "answer.log")),
|
||||
query_failure=_preview(_object(query_folder, "failure.json")),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _inspect_result(home, spec):
|
||||
run = _run_path(home, spec.validation_path)
|
||||
manifest = _object(run, "manifest.json", required=True)
|
||||
if spec.case_id is not None and spec.case_id not in manifest.get("case_ids", []):
|
||||
raise ValueError("case_id is not selected by this evaluation manifest")
|
||||
result = {"validation_path": str(run), "manifest_path": str(run / "manifest.json")}
|
||||
if spec.case_id is not None:
|
||||
return {**result, **_case_inspection(run, manifest, spec)}
|
||||
summary, summary_path = _summary(run)
|
||||
return {
|
||||
**_catalog_row(run, manifest),
|
||||
"rows_source": summary_path,
|
||||
**_page(_case_rows(manifest, summary), spec),
|
||||
}
|
||||
|
||||
|
||||
async def _execute(step, model, function):
|
||||
if step.context.get("_allowed_paths") is not None:
|
||||
raise ValueError("Home inspection Jobs require unrestricted access to their managed evaluation artifacts")
|
||||
spec = model.model_validate({name: step.context.get(name) for name in model.model_fields if name in step.context})
|
||||
if not step.workspace.is_started:
|
||||
raise RuntimeError("Start the Home workspace before inspecting evaluations")
|
||||
task = asyncio.create_task(asyncio.to_thread(function, step.workspace.home_path, spec))
|
||||
cancelled = False
|
||||
while True:
|
||||
try:
|
||||
result = await asyncio.shield(task)
|
||||
if cancelled:
|
||||
raise asyncio.CancelledError
|
||||
step.context.response.answer = result
|
||||
return step.context.response
|
||||
except asyncio.CancelledError:
|
||||
if task.done():
|
||||
raise
|
||||
cancelled = True
|
||||
|
||||
|
||||
class HomeListValidationResultsStep(BaseStep):
|
||||
"""List all evaluation artifacts, including failures, debug and standalone runs."""
|
||||
|
||||
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
|
||||
|
||||
async def execute(self):
|
||||
"""Navigate runs without reading or rebuilding optimization history."""
|
||||
return await _execute(self, ListValidationResultsInput, _list_results)
|
||||
|
||||
|
||||
class HomeInspectValidationStep(BaseStep):
|
||||
"""Inspect a run, a case or one query with bounded result previews."""
|
||||
|
||||
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
|
||||
|
||||
async def execute(self):
|
||||
"""Read only the requested result scope and return artifact coordinates."""
|
||||
return await _execute(self, InspectValidationInput, _inspect_result)
|
||||
284
plugins/home/src/reme_home/evaluation/memory_inspection_step.py
Normal file
284
plugins/home/src/reme_home/evaluation/memory_inspection_step.py
Normal file
|
|
@ -0,0 +1,284 @@
|
|||
"""Read-only navigation and comparison of exported memory Git checkpoints."""
|
||||
|
||||
import difflib
|
||||
import hashlib
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
|
||||
from reme.steps.base_step import BaseStep, Ref
|
||||
|
||||
from ..code.repository import MAX_OUTPUT, resolve_revision
|
||||
from ..code.snapshot import blob, code_path, tree
|
||||
from ..preparation.code import git as safe_git
|
||||
from ..preparation.spec import SpecModel
|
||||
from ..workspace import HomeWorkspace
|
||||
from .inputs import entity_id
|
||||
from .artifacts import contained
|
||||
from .inspection_step import _execute, _object, _page, _run_path
|
||||
|
||||
MAX_TREE_FILES = 2000
|
||||
MAX_TREE_BYTES = 20_000_000
|
||||
|
||||
|
||||
def git(repository, *arguments):
|
||||
"""Ignore replace refs while reusing the shared noninteractive Git executor."""
|
||||
return safe_git(repository, "--no-replace-objects", *arguments)
|
||||
|
||||
|
||||
class MemoryScopeInput(SpecModel):
|
||||
"""An existing evaluation and one manifest-selected case."""
|
||||
|
||||
validation_path: str = Field(min_length=1)
|
||||
case_id: str = Field(min_length=1)
|
||||
path: str | None = None
|
||||
|
||||
@field_validator("case_id")
|
||||
@classmethod
|
||||
def validate_case(cls, value):
|
||||
"""Accept a single case directory segment."""
|
||||
return entity_id(value)
|
||||
|
||||
@field_validator("path")
|
||||
@classmethod
|
||||
def validate_path(cls, value):
|
||||
"""Accept literal repository-relative paths only."""
|
||||
return code_path(value) if value is not None else None
|
||||
|
||||
|
||||
class MemoryHistoryInput(MemoryScopeInput):
|
||||
"""Page through checkpoints, newest first."""
|
||||
|
||||
offset: int = Field(default=0, ge=0)
|
||||
limit: int = Field(default=20, ge=1, le=100)
|
||||
|
||||
|
||||
class CompareMemoryInput(MemoryScopeInput):
|
||||
"""Compare two commits in one or two exported repositories."""
|
||||
|
||||
base_ref: str | None = Field(default=None, min_length=1)
|
||||
target_ref: str = Field(default="HEAD", min_length=1)
|
||||
target_validation_path: str | None = Field(default=None, min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_base(self):
|
||||
"""Require an explicit earlier checkpoint for a same-repository comparison."""
|
||||
if self.target_validation_path is None and self.base_ref is None:
|
||||
raise ValueError("base_ref is required when target_validation_path is omitted")
|
||||
return self
|
||||
|
||||
|
||||
def _repository(home, validation_path, case_id):
|
||||
run = _run_path(home, validation_path)
|
||||
manifest = _object(run, "manifest.json", required=True)
|
||||
if case_id not in manifest.get("case_ids", []):
|
||||
raise ValueError("case_id is not selected by this evaluation manifest")
|
||||
repository = contained(run, f"cases/{case_id}/memory_construction/reme_workspace", directory=True)
|
||||
return run, manifest, repository
|
||||
|
||||
|
||||
def _head(repository):
|
||||
"""Reject indirect Git storage and distinguish absent/empty/corrupt repositories."""
|
||||
if not repository.exists():
|
||||
return "workspace_missing", None
|
||||
metadata = contained(repository, ".git")
|
||||
if not metadata.exists():
|
||||
return "history_missing", None
|
||||
if not metadata.is_dir():
|
||||
raise ValueError("Memory history requires a standalone .git directory")
|
||||
# Exported artifacts must not redirect reads to another repository or host path.
|
||||
if any(path.is_symlink() for path in metadata.rglob("*")):
|
||||
raise ValueError("Memory Git metadata must not contain symbolic links")
|
||||
for name in ("commondir", "objects/info/alternates", "objects/info/http-alternates", "shallow"):
|
||||
if (metadata / name).exists():
|
||||
raise ValueError(f"Unsupported indirect or incomplete memory history: {name}")
|
||||
if any(metadata.glob("objects/pack/*.promisor")):
|
||||
raise ValueError("Partial memory repositories are unsupported")
|
||||
if git(repository, "for-each-ref", "--format=%(refname)", "refs/replace"):
|
||||
raise ValueError("Memory replace refs are unsupported")
|
||||
configuration = git(repository, "config", "--local", "--list")
|
||||
if any(line.lower().startswith(("include.", "includeif.")) for line in configuration.splitlines()):
|
||||
raise ValueError("Memory Git configuration must not include external files")
|
||||
if any(
|
||||
"promisor=" in line.lower() or line.lower().startswith("extensions.partialclone=")
|
||||
for line in configuration.splitlines()
|
||||
):
|
||||
raise ValueError("Partial memory repositories are unsupported")
|
||||
if git(repository, "rev-parse", "--show-toplevel") != str(repository):
|
||||
raise ValueError("Memory history must be a standalone repository")
|
||||
if git(repository, "rev-parse", "--is-shallow-repository") != "false":
|
||||
raise ValueError("Memory history must not be shallow")
|
||||
try:
|
||||
return "available", resolve_revision(repository, "HEAD")
|
||||
except subprocess.CalledProcessError:
|
||||
# Only an unborn symbolic HEAD is an empty repository, not a broken ref.
|
||||
reference = git(repository, "symbolic-ref", "-q", "HEAD")
|
||||
if git(repository, "for-each-ref", "--format=%(refname)", reference):
|
||||
raise ValueError("Memory HEAD cannot be resolved") from None
|
||||
if not (metadata / "objects").is_dir():
|
||||
raise ValueError("Memory object database is missing") from None
|
||||
return "history_empty", None
|
||||
|
||||
|
||||
def _resolve(repository, reference, head):
|
||||
if re.fullmatch(r"[0-9a-fA-F]+", reference or ""):
|
||||
matches = [
|
||||
sha for sha in git(repository, "rev-list", head, "--").splitlines() if sha.startswith(reference.lower())
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"Memory commit prefix has {len(matches)} matches reachable from HEAD; require one")
|
||||
return matches[0]
|
||||
revision = resolve_revision(repository, reference)
|
||||
git(repository, "merge-base", "--is-ancestor", revision, head)
|
||||
return revision
|
||||
|
||||
|
||||
def inspect_history(home, spec):
|
||||
"""Return saved Git history without refreshing the worktree or index."""
|
||||
run, _manifest, repository = _repository(home, spec.validation_path, spec.case_id)
|
||||
status, head = _head(repository)
|
||||
result = {
|
||||
"validation_path": str(run),
|
||||
"case_id": spec.case_id,
|
||||
"repository": str(repository),
|
||||
"workspace_exists": repository.exists(),
|
||||
"has_git_history": head is not None,
|
||||
"status": status,
|
||||
"head": head,
|
||||
"path": spec.path,
|
||||
}
|
||||
if head is None:
|
||||
return {**result, **_page([], spec)}
|
||||
# Literal pathspecs prevent user-controlled wildcard/pathspec expansion.
|
||||
path_args = ["--", ":(literal)" + spec.path] if spec.path is not None else ["--"]
|
||||
count = int(git(repository, "rev-list", "--count", head, *path_args))
|
||||
raw = git(
|
||||
repository,
|
||||
"log",
|
||||
"--no-show-signature",
|
||||
f"--skip={spec.offset}",
|
||||
f"--max-count={spec.limit}",
|
||||
"--format=%H%x00%P%x00%aI%x00%s",
|
||||
head,
|
||||
*path_args,
|
||||
)
|
||||
rows = []
|
||||
for line in raw.splitlines():
|
||||
sha, parents, timestamp, subject = line.split("\0", 3)
|
||||
rows.append(
|
||||
{
|
||||
"commit_sha": sha,
|
||||
"parents": parents.split(),
|
||||
"timestamp": timestamp,
|
||||
"subject": subject[:2000],
|
||||
"subject_truncated": len(subject) > 2000,
|
||||
"session_id": subject[9:2009] if subject.startswith("session: ") else None,
|
||||
},
|
||||
)
|
||||
return {
|
||||
**result,
|
||||
"row_count": count,
|
||||
"rows": rows,
|
||||
"next_offset": spec.offset + len(rows) if spec.offset + len(rows) < count else None,
|
||||
}
|
||||
|
||||
|
||||
def _files(repository, revision, path):
|
||||
entries = tree(repository, revision, path)
|
||||
if len(entries) > MAX_TREE_FILES:
|
||||
raise ValueError("Memory comparison exceeds 2000 files; narrow path")
|
||||
result = {}
|
||||
total = 0
|
||||
for name, oid in entries:
|
||||
total += int(git(repository, "cat-file", "-s", oid))
|
||||
if total > MAX_TREE_BYTES:
|
||||
raise ValueError("Memory comparison exceeds 20 MB per snapshot; narrow path")
|
||||
result[name] = blob(repository, oid)
|
||||
return result
|
||||
|
||||
|
||||
def _hash(files):
|
||||
digest = hashlib.sha256()
|
||||
for name, data in sorted(files.items()):
|
||||
digest.update(name.encode() + b"\0" + hashlib.sha256(data).digest())
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def compare_memory(home, spec):
|
||||
"""Compare committed regular-file contents, independent of dirty checkout files."""
|
||||
base_run, base_manifest, base_repo = _repository(home, spec.validation_path, spec.case_id)
|
||||
target_run, target_manifest, target_repo = _repository(
|
||||
home,
|
||||
spec.target_validation_path or spec.validation_path,
|
||||
spec.case_id,
|
||||
)
|
||||
base_status, base_head = _head(base_repo)
|
||||
target_status, target_head = _head(target_repo)
|
||||
if base_head is None or target_head is None:
|
||||
raise ValueError(f"Comparison requires memory commits on both sides: {base_status}, {target_status}")
|
||||
base_sha = _resolve(base_repo, spec.base_ref or "HEAD", base_head)
|
||||
target_sha = _resolve(target_repo, spec.target_ref, target_head)
|
||||
before = _files(base_repo, base_sha, spec.path)
|
||||
after = _files(target_repo, target_sha, spec.path)
|
||||
added = sorted(after.keys() - before.keys())
|
||||
removed = sorted(before.keys() - after.keys())
|
||||
modified = sorted(name for name in before.keys() & after.keys() if before[name] != after[name])
|
||||
parts = []
|
||||
length = 0
|
||||
truncated = False
|
||||
for name in sorted(added + removed + modified):
|
||||
left, right = before.get(name, b""), after.get(name, b"")
|
||||
try:
|
||||
if b"\0" in left or b"\0" in right:
|
||||
raise UnicodeDecodeError("utf-8", b"\0", 0, 1, "binary")
|
||||
lines = difflib.unified_diff(
|
||||
left.decode("utf-8").splitlines(keepends=True),
|
||||
right.decode("utf-8").splitlines(keepends=True),
|
||||
fromfile="base/" + name,
|
||||
tofile="target/" + name,
|
||||
)
|
||||
except UnicodeDecodeError:
|
||||
lines = iter([f"Binary files differ: {name}\n"])
|
||||
for line in lines:
|
||||
if not line.endswith("\n"):
|
||||
line += "\n\\ No newline at end of file\n"
|
||||
available = MAX_OUTPUT - length
|
||||
parts.append(line[:available])
|
||||
length += min(len(line), available)
|
||||
if len(line) > available:
|
||||
truncated = True
|
||||
break
|
||||
if truncated:
|
||||
break
|
||||
return {
|
||||
"base_validation_path": str(base_run),
|
||||
"target_validation_path": str(target_run),
|
||||
"case_id": spec.case_id,
|
||||
"path": spec.path,
|
||||
"base_commit": base_sha,
|
||||
"target_commit": target_sha,
|
||||
"base_candidate_commit": base_manifest.get("commit_sha"),
|
||||
"target_candidate_commit": target_manifest.get("commit_sha"),
|
||||
"base_fingerprints": base_manifest.get("fingerprints"),
|
||||
"target_fingerprints": target_manifest.get("fingerprints"),
|
||||
"added": added,
|
||||
"removed": removed,
|
||||
"modified": modified,
|
||||
"base_tree_sha256": _hash(before),
|
||||
"target_tree_sha256": _hash(after),
|
||||
"diff": "".join(parts),
|
||||
"truncated": truncated,
|
||||
}
|
||||
|
||||
|
||||
class HomeMemoryInspectionStep(BaseStep):
|
||||
"""Two read-only Jobs sharing one invocation-scoped Step implementation."""
|
||||
|
||||
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
|
||||
|
||||
async def execute(self):
|
||||
"""Select the operation from host configuration, never model input."""
|
||||
operations = {"history": (MemoryHistoryInput, inspect_history), "compare": (CompareMemoryInput, compare_memory)}
|
||||
model, function = operations[self.kwargs["operation"]]
|
||||
return await _execute(self, model, function)
|
||||
|
|
@ -7,14 +7,14 @@ from .inputs import EvaluationSpec
|
|||
|
||||
|
||||
class HomeEvaluateStep(BaseStep):
|
||||
"""Case IDs come from runtime inputs; no selection means all prepared cases."""
|
||||
"""One evaluation Job with optional host-configured adaptive sampling."""
|
||||
|
||||
evaluation: HomeEvaluation = Ref(HomeEvaluation, "home.evaluation")
|
||||
|
||||
async def execute(self):
|
||||
"""Evaluate via the Docker pool and persist host-side scores and evidence."""
|
||||
if self.context.get("_allowed_paths") is not None:
|
||||
raise ValueError("home_evaluate requires unrestricted access to its workspace output directories")
|
||||
raise ValueError("Home evaluation Jobs require unrestricted access to their workspace output directories")
|
||||
arguments = {name: self.context.get(name) for name in EvaluationSpec.model_fields if name in self.context}
|
||||
# Do not silently accept a removed toggle that conflicts with reuse paths.
|
||||
if "eval_only" in self.context:
|
||||
|
|
|
|||
|
|
@ -1,20 +1,39 @@
|
|||
backends:
|
||||
home_memory_inspection_step: reme_home.evaluation.memory_inspection_step:HomeMemoryInspectionStep
|
||||
home_code: reme_home.code.component:HomeCode
|
||||
home_code_step: reme_home.code.step:HomeCodeStep
|
||||
home_bounded: reme_home.agent:BoundedAgentWrapper
|
||||
home_preparation: reme_home.preparation.component:HomePreparation
|
||||
home_prepare_step: reme_home.preparation.component:HomePrepareStep
|
||||
home_workspace: reme_home.workspace:HomeWorkspace
|
||||
home_judge: reme_home.judge:HomeJudge
|
||||
home_evaluation: reme_home.evaluation.component:HomeEvaluation
|
||||
home_evaluation_history: reme_home.evaluation.history:HomeEvaluationHistory
|
||||
home_evaluate_step: reme_home.evaluation.step:HomeEvaluateStep
|
||||
home_list_validation_results_step: reme_home.evaluation.inspection_step:HomeListValidationResultsStep
|
||||
home_inspect_validation_step: reme_home.evaluation.inspection_step:HomeInspectValidationStep
|
||||
auto_bash_step: reme_home.auto_bash:AutoBashStep
|
||||
|
||||
application_defaults:
|
||||
components:
|
||||
home.code:
|
||||
default:
|
||||
backend: home_code
|
||||
home.judge:
|
||||
default:
|
||||
backend: home_judge
|
||||
home.evaluation_history:
|
||||
default:
|
||||
backend: home_evaluation_history
|
||||
home.evaluation:
|
||||
default:
|
||||
backend: home_evaluation
|
||||
sampling_policy:
|
||||
max_cases: 5
|
||||
max_queries: null
|
||||
temperature: 1.0
|
||||
ewma_alpha: 0.3
|
||||
uniform_exploration: 0.2
|
||||
home.preparation:
|
||||
default:
|
||||
backend: home_preparation
|
||||
|
|
@ -22,12 +41,105 @@ application_defaults:
|
|||
default:
|
||||
backend: home_workspace
|
||||
jobs:
|
||||
home_evaluate:
|
||||
home_inspect_memory_history:
|
||||
backend: base
|
||||
description: "Evaluate committed candidate code on prepared cases; optionally restore prior construction tar files"
|
||||
description: "List a case's exported memory Git checkpoints, newest first; use home_inspect_validation to locate the case"
|
||||
parameters:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [validation_path, case_id]
|
||||
properties:
|
||||
validation_path: {type: string, minLength: 1}
|
||||
case_id: {type: string, minLength: 1}
|
||||
path:
|
||||
type: [string, "null"]
|
||||
description: "Literal memory-workspace-relative file or directory to follow"
|
||||
offset: {type: integer, minimum: 0, default: 0}
|
||||
limit: {type: integer, minimum: 1, maximum: 100, default: 20}
|
||||
steps:
|
||||
- backend: home_memory_inspection_step
|
||||
operation: history
|
||||
home_compare_memory:
|
||||
backend: base
|
||||
description: "Compare a case's memory checkpoints or committed final memory across evaluations; narrow path when output is truncated"
|
||||
parameters:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [validation_path, case_id]
|
||||
properties:
|
||||
validation_path: {type: string, minLength: 1}
|
||||
case_id: {type: string, minLength: 1}
|
||||
base_ref:
|
||||
type: [string, "null"]
|
||||
description: "Required within one evaluation; defaults to HEAD for cross-evaluation comparisons"
|
||||
target_ref:
|
||||
type: string
|
||||
default: HEAD
|
||||
description: "Target memory commit or revision; hexadecimal prefixes must be unique among HEAD-reachable commits"
|
||||
target_validation_path:
|
||||
type: [string, "null"]
|
||||
description: "Optional other evaluation containing the same case"
|
||||
path:
|
||||
type: [string, "null"]
|
||||
description: "Literal memory-workspace-relative file or directory"
|
||||
steps:
|
||||
- backend: home_memory_inspection_step
|
||||
operation: compare
|
||||
home_list_validation_results:
|
||||
backend: base
|
||||
description: "List evaluation runs including debug, failed and standalone runs; inspect a returned validation_path with home_inspect_validation"
|
||||
parameters:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
properties:
|
||||
branch_name:
|
||||
type: [string, "null"]
|
||||
description: "Filter by branch; with commit_sha defaults to the current candidate branch"
|
||||
commit_sha:
|
||||
type: [string, "null"]
|
||||
pattern: "^[0-9a-fA-F]+$"
|
||||
description: "Full commit SHA or hexadecimal prefix matching exactly one reachable commit on the selected branch"
|
||||
offset: {type: integer, minimum: 0, default: 0}
|
||||
limit: {type: integer, minimum: 1, maximum: 100, default: 20}
|
||||
steps:
|
||||
- backend: home_list_validation_results_step
|
||||
home_inspect_validation:
|
||||
backend: base
|
||||
description: "Read an evaluation's case summaries, a case's query summaries, or one query's answer/judge previews; follow next_offset and source paths for details"
|
||||
parameters:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [validation_path]
|
||||
properties:
|
||||
validation_path:
|
||||
type: string
|
||||
minLength: 1
|
||||
description: "Result directory returned by home_evaluate or home_list_validation_results; absolute or relative to home/"
|
||||
case_id:
|
||||
type: [string, "null"]
|
||||
description: "Selected case to inspect; omit to list cases"
|
||||
query_id:
|
||||
type: [string, "null"]
|
||||
description: "Selected query to inspect in detail; requires case_id"
|
||||
offset: {type: integer, minimum: 0, default: 0}
|
||||
limit: {type: integer, minimum: 1, maximum: 100, default: 20}
|
||||
steps:
|
||||
- backend: home_inspect_validation_step
|
||||
home_evaluate:
|
||||
backend: base
|
||||
description: "Evaluate committed code; optionally sample cases/queries or restore prior construction tar files"
|
||||
parameters:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
properties:
|
||||
sampling:
|
||||
type: boolean
|
||||
default: false
|
||||
description: "Adaptively select case/query IDs using the host sampling_policy; debug ignores this flag"
|
||||
record_history:
|
||||
type: boolean
|
||||
default: true
|
||||
description: "Register successful scored results and started rounds for optimization; false saves results without history or sampling updates; debug always skips history"
|
||||
# case_ids/query_ids are internal runtime inputs, supplied by the host
|
||||
# through injected_job_kwargs or run_job(), never model-selected tools.
|
||||
reuse_construction_from:
|
||||
|
|
@ -101,3 +213,191 @@ application_defaults:
|
|||
- command
|
||||
steps:
|
||||
- backend: auto_bash_step
|
||||
home_code_status:
|
||||
backend: base
|
||||
description: Inspect candidate branch, commit and uncommitted files
|
||||
parameters:
|
||||
additionalProperties: false
|
||||
properties: {}
|
||||
type: object
|
||||
steps:
|
||||
- backend: home_code_step
|
||||
operation: status
|
||||
home_list_candidate_branches:
|
||||
backend: base
|
||||
description: List the protected init branch and local candidate branches
|
||||
parameters:
|
||||
additionalProperties: false
|
||||
properties: {}
|
||||
type: object
|
||||
steps:
|
||||
- backend: home_code_step
|
||||
operation: list_branches
|
||||
home_create_candidate_branch:
|
||||
backend: base
|
||||
description: Create and switch to a new candidate branch; requires clean code
|
||||
parameters:
|
||||
additionalProperties: false
|
||||
properties:
|
||||
branch_name:
|
||||
maxLength: 100
|
||||
minLength: 1
|
||||
type: string
|
||||
required:
|
||||
- branch_name
|
||||
type: object
|
||||
steps:
|
||||
- backend: home_code_step
|
||||
operation: create_branch
|
||||
home_switch_candidate_branch:
|
||||
backend: base
|
||||
description: Switch to an existing local branch; requires clean code
|
||||
parameters:
|
||||
additionalProperties: false
|
||||
properties:
|
||||
branch_name:
|
||||
maxLength: 100
|
||||
minLength: 1
|
||||
type: string
|
||||
required:
|
||||
- branch_name
|
||||
type: object
|
||||
steps:
|
||||
- backend: home_code_step
|
||||
operation: switch_branch
|
||||
home_merge_candidate_branch:
|
||||
backend: base
|
||||
description: Stage another candidate branch without committing; conflicts are aborted
|
||||
parameters:
|
||||
additionalProperties: false
|
||||
properties:
|
||||
branch_name:
|
||||
maxLength: 100
|
||||
minLength: 1
|
||||
type: string
|
||||
required:
|
||||
- branch_name
|
||||
type: object
|
||||
steps:
|
||||
- backend: home_code_step
|
||||
operation: merge_branch
|
||||
home_commit_candidate:
|
||||
backend: base
|
||||
description: Commit all changes within candidate_paths; init and existing history are protected
|
||||
parameters:
|
||||
additionalProperties: false
|
||||
properties:
|
||||
message:
|
||||
maxLength: 10000
|
||||
minLength: 1
|
||||
pattern: \S
|
||||
type: string
|
||||
required:
|
||||
- message
|
||||
type: object
|
||||
steps:
|
||||
- backend: home_code_step
|
||||
operation: commit
|
||||
home_code_diff:
|
||||
backend: base
|
||||
description: Inspect tracked working changes against HEAD or compare immutable commits
|
||||
parameters:
|
||||
additionalProperties: false
|
||||
properties:
|
||||
base_revision:
|
||||
anyOf:
|
||||
- type: string
|
||||
- type: 'null'
|
||||
default: null
|
||||
revision:
|
||||
anyOf:
|
||||
- type: string
|
||||
- type: 'null'
|
||||
default: null
|
||||
type: object
|
||||
steps:
|
||||
- backend: home_code_step
|
||||
operation: diff
|
||||
home_list_code_tree:
|
||||
backend: base
|
||||
description: List tracked regular files at a committed revision
|
||||
parameters:
|
||||
additionalProperties: false
|
||||
properties:
|
||||
revision:
|
||||
default: HEAD
|
||||
minLength: 1
|
||||
type: string
|
||||
path:
|
||||
anyOf:
|
||||
- type: string
|
||||
- type: 'null'
|
||||
default: null
|
||||
max_results:
|
||||
default: 500
|
||||
maximum: 2000
|
||||
minimum: 1
|
||||
type: integer
|
||||
type: object
|
||||
steps:
|
||||
- backend: home_code_step
|
||||
operation: list_code_tree
|
||||
home_read_code_file:
|
||||
backend: base
|
||||
description: Read up to 300 lines from a committed code file
|
||||
parameters:
|
||||
additionalProperties: false
|
||||
properties:
|
||||
path:
|
||||
minLength: 1
|
||||
type: string
|
||||
revision:
|
||||
default: HEAD
|
||||
minLength: 1
|
||||
type: string
|
||||
start_line:
|
||||
default: 1
|
||||
minimum: 1
|
||||
type: integer
|
||||
end_line:
|
||||
anyOf:
|
||||
- minimum: 1
|
||||
type: integer
|
||||
- type: 'null'
|
||||
default: null
|
||||
required:
|
||||
- path
|
||||
type: object
|
||||
steps:
|
||||
- backend: home_code_step
|
||||
operation: read_code_file
|
||||
home_search_code:
|
||||
backend: base
|
||||
description: Search committed code with a bounded Python multiline regular expression
|
||||
parameters:
|
||||
additionalProperties: false
|
||||
properties:
|
||||
pattern:
|
||||
maxLength: 500
|
||||
minLength: 1
|
||||
type: string
|
||||
revision:
|
||||
default: HEAD
|
||||
minLength: 1
|
||||
type: string
|
||||
path:
|
||||
anyOf:
|
||||
- type: string
|
||||
- type: 'null'
|
||||
default: null
|
||||
max_results:
|
||||
default: 30
|
||||
maximum: 30
|
||||
minimum: 1
|
||||
type: integer
|
||||
required:
|
||||
- pattern
|
||||
type: object
|
||||
steps:
|
||||
- backend: home_code_step
|
||||
operation: search_code
|
||||
|
|
|
|||
|
|
@ -10,18 +10,35 @@ import tomllib
|
|||
from .spec import PreparationSpec, relative_path
|
||||
|
||||
|
||||
def git(repository: Path, *arguments: str) -> str:
|
||||
def git_bytes(repository: Path, *arguments: str) -> bytes:
|
||||
"""Run bounded, noninteractive Git commands without a shell or user hooks."""
|
||||
environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
|
||||
environment.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0")
|
||||
result = subprocess.run(
|
||||
["git", "-c", "core.hooksPath=" + os.devnull, "-c", "commit.gpgSign=false", "-C", str(repository), *arguments],
|
||||
[
|
||||
"git",
|
||||
"-c",
|
||||
"core.hooksPath=" + os.devnull,
|
||||
"-c",
|
||||
"commit.gpgSign=false",
|
||||
"-c",
|
||||
"core.fsmonitor=false",
|
||||
"-C",
|
||||
str(repository),
|
||||
*arguments,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
env=environment,
|
||||
timeout=120,
|
||||
)
|
||||
return result.stdout.decode().strip()
|
||||
return result.stdout
|
||||
|
||||
|
||||
def git(repository: Path, *arguments: str, strip: bool = True) -> str:
|
||||
"""Decode Git output, optionally preserving porcelain whitespace and path bytes."""
|
||||
output = git_bytes(repository, *arguments).decode("utf-8")
|
||||
return output.strip() if strip else output
|
||||
|
||||
|
||||
def tree_fingerprint(root: Path) -> str:
|
||||
|
|
|
|||
|
|
@ -76,3 +76,19 @@ flag. Host-installed benchmark judge plugins score answers outside Docker and
|
|||
outside the candidate repository.
|
||||
|
||||
See `validation-artifact-guide.md` for saved evidence and score interpretation.
|
||||
|
||||
## Candidate versions
|
||||
|
||||
Use the Home code Jobs to list, create, switch, merge and commit candidates in
|
||||
`home/code/repo`. `init` is the protected preparation baseline. Branch operations
|
||||
require clean code; merges stay uncommitted for inspection. Commit all intended
|
||||
changes with `home_commit_candidate` before debug or scored evaluation. Mutations
|
||||
are limited to prepared `candidate_paths`; keep repository metadata and runtime
|
||||
outside the optimizer's write permissions.
|
||||
|
||||
`home_list_code_tree`, `home_read_code_file` and `home_search_code` inspect raw
|
||||
committed files. Their `revision` defaults to current HEAD and returns a resolved
|
||||
SHA. For diagnosis or version comparisons, the host should inject a fixed SHA
|
||||
through `injected_job_kwargs` for each agent/Job alias, so subsequent checkout
|
||||
changes cannot affect the evidence. See the Home plugin README for inputs and
|
||||
output limits.
|
||||
|
|
|
|||
|
|
@ -120,3 +120,84 @@ structured exception details are treated as infrastructure errors. Construction
|
|||
failure-rate tolerance applies only to classified candidate errors.
|
||||
Construction reuse answers and judges again with the current candidate; it does
|
||||
not reuse donor answers or scores.
|
||||
|
||||
## Evaluation history and adaptive sampling evidence
|
||||
|
||||
`home_evaluate sampling=true` enables adaptive selection; false preserves full or
|
||||
explicit evaluation. Debug always bypasses history. `sampling_policy` is the
|
||||
host policy input, distinct from the boolean toggle. The effective policy is
|
||||
captured in `manifest.options` and the started plan.
|
||||
|
||||
`record_history=true` (the default) includes a successful full baseline even when
|
||||
`sampling=false`. `record_history=false` runs the same scoring and artifact pipeline
|
||||
without registering results or advancing any optimization/sampling counters. The
|
||||
choice is stored in `manifest.options.record_history` and respected by history
|
||||
registration and rebuilding; a missing legacy setting means true. Debug always
|
||||
skips history. The response reports effective participation as `record_history`.
|
||||
An untracked adaptive run may read existing observations and saves its draw locally
|
||||
in `manifest.selection_plan`, without a shared started plan or ledger round.
|
||||
|
||||
`home.evaluation_history` owns the successful-result catalog at
|
||||
`home/evaluations/history.json`. Entries require completed scored training runs
|
||||
with `success=true` (or equivalent legacy summary counts) and no infrastructure
|
||||
errors. Low scores remain useful. Failed/cancelled/debug/partial runs are excluded;
|
||||
a mixed run with an infrastructure failure contributes no quality observations.
|
||||
Source manifests, summaries and query results remain authoritative.
|
||||
|
||||
`sandbox_started_at` marks the first successfully initialized sandbox. The
|
||||
history-enabled manifest's `sampling.plan_path` points to its policy, seed, weights, selection
|
||||
and started round under `home/sampling/plans/`. Pre-start failures have no marker
|
||||
or plan, and additional workers do not create extra rounds. The derived sampling
|
||||
cache separates started `round` and `successful_round`: only successful history
|
||||
advances staleness/recheck timing and trains EWMA. Adaptive plans preserve their
|
||||
probability clock in `selection_round`. Actual numeric scores alone count as
|
||||
quality observations; missing/unjudged answers never become zero scores.
|
||||
|
||||
Missing caches and `HomeEvaluationHistory.rebuild()` reconstruct the catalog and
|
||||
sampling observations from saved evidence. Rebuilding never modifies datasets,
|
||||
candidate code or memory files. See the Home README for configuration, host
|
||||
per-call policy overrides and enforced agent injection examples.
|
||||
|
||||
## Read-only evaluation inspection
|
||||
|
||||
Use `home_list_validation_results` to find runs and pass a returned
|
||||
`validation_path` to `home_inspect_validation`. The catalog includes failed,
|
||||
debug, running and untracked runs, not only successful optimization history.
|
||||
Both Jobs page with `offset` and `limit` (default 20, maximum 100), returning
|
||||
`row_count`, `rows`, `next_offset`. Catalog filters are `branch_name` and
|
||||
`commit_sha`; a hexadecimal commit prefix must uniquely match among all commits
|
||||
reachable from the specified branch, or the current branch when omitted.
|
||||
|
||||
Inspection without `case_id` returns case summaries; add `case_id` for query
|
||||
summaries and `query_id` for bounded answer/judge fields. Each row's JSON pointer
|
||||
is relative to `rows_source`. Failed construction Jobs have separate pagination
|
||||
and pointers relative to `construction.source`. Query fields are limited to
|
||||
2,000-character previews and errors to 600 characters. Follow returned source
|
||||
paths for complete evidence. Missing files/results remain explicit, with null
|
||||
scores, and partial summaries are used when full summaries are absent. Reads do
|
||||
not rebuild summaries or modify evaluation history, sampling state or artifacts.
|
||||
|
||||
## Memory checkpoint tools
|
||||
|
||||
After locating one case, call `home_inspect_memory_history` with its
|
||||
`validation_path` and `case_id` to page through exported memory checkpoints.
|
||||
Optional `path` follows a literal memory-workspace-relative file or directory.
|
||||
The response distinguishes a missing workspace, absent history and an empty
|
||||
repository; invalid history fails explicitly. Session IDs are extracted only
|
||||
from the established `session: <id>` commit-subject convention.
|
||||
|
||||
Use `home_compare_memory` with `base_ref` and optional `target_ref` (default
|
||||
`HEAD`) to inspect how memory changed within that evaluation. Add
|
||||
`target_validation_path` to compare the same case across evaluations; both
|
||||
references then default to `HEAD`. References must resolve to commits reachable
|
||||
from each memory HEAD, and hexadecimal prefixes must be unique within that
|
||||
reachable history. Compare candidate identities and fingerprints before drawing
|
||||
cross-evaluation conclusions.
|
||||
|
||||
Diffs describe committed tracked regular-file contents rather than dirty checkout
|
||||
files, untracked files or mode-only changes. Binary changes are identified without
|
||||
printing bytes. Narrow `path` when the 2,000-file, 20-MB snapshot, 5-MB file or
|
||||
100,000-character diff limit is reached. Hashes cover the selected path scope.
|
||||
These tools neither reconstruct missing exports nor modify Git/evaluation/history
|
||||
artifacts. Raw logs and source JSON remain available through the existing result
|
||||
inspection paths.
|
||||
|
|
|
|||
169
plugins/home/src/reme_home/sampling.py
Normal file
169
plugins/home/src/reme_home/sampling.py
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
"""Adaptive formal-validation sampling ported from dev_meta b53048e4.
|
||||
|
||||
Keep the source weighting, weighted ordering and fair-share query allocation.
|
||||
EvaluationHistory owns persisted observations and started-validation plans.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
import time
|
||||
from typing import Any, Mapping
|
||||
|
||||
from pydantic import ConfigDict, Field
|
||||
|
||||
from .preparation.data.models import CaseSpec
|
||||
from .preparation.spec import SpecModel
|
||||
|
||||
|
||||
class SamplingPolicy(SpecModel):
|
||||
"""Host-owned limits with the same defaults as dev_meta's validation.sampling."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
max_cases: int = Field(default=5, ge=1)
|
||||
max_queries: int | None = Field(default=None, ge=1)
|
||||
temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
|
||||
ewma_alpha: float = Field(default=0.3, gt=0, le=1, allow_inf_nan=False)
|
||||
uniform_exploration: float = Field(default=0.2, ge=0, le=1, allow_inf_nan=False)
|
||||
|
||||
|
||||
def create_sampling_plan(
|
||||
cases: list[CaseSpec],
|
||||
state: Mapping[str, Any],
|
||||
policy: SamplingPolicy,
|
||||
*,
|
||||
eligible_query_ids: Mapping[str, list[str]] | None = None,
|
||||
) -> dict:
|
||||
"""Draw without replacement, without advancing state or writing an attempted plan."""
|
||||
if eligible_query_ids is not None:
|
||||
cases = [case for case in cases if case.case_id in eligible_query_ids]
|
||||
if not cases:
|
||||
raise ValueError("training-set contains no cases to sample")
|
||||
round_id = int(state.get("round", 0)) + 1
|
||||
seed = time.time_ns()
|
||||
rng = random.Random(seed)
|
||||
case_records = state.get("cases", {})
|
||||
case_weights = _mixed_weights(
|
||||
{case.case_id: case_records.get(case.case_id, {}) for case in cases},
|
||||
round_id,
|
||||
policy,
|
||||
)
|
||||
ordered_cases = _weighted_order(cases, case_weights, rng, lambda case: case.case_id)
|
||||
query_limited = policy.max_queries is not None
|
||||
remaining_queries = policy.max_queries
|
||||
selected_cases: list[str] = []
|
||||
selected_queries: dict[str, list[str]] = {}
|
||||
query_weights: dict[str, dict[str, float]] = {}
|
||||
query_records = state.get("queries", {})
|
||||
for case in ordered_cases:
|
||||
if len(selected_cases) >= policy.max_cases:
|
||||
break
|
||||
if remaining_queries is not None and remaining_queries <= 0:
|
||||
break
|
||||
query_ids = [query.query_id for query in case.queries]
|
||||
if eligible_query_ids is not None:
|
||||
allowed = set(eligible_query_ids.get(case.case_id, []))
|
||||
query_ids = [query_id for query_id in query_ids if query_id in allowed]
|
||||
if not query_ids:
|
||||
continue
|
||||
if query_limited:
|
||||
weights = _mixed_weights(
|
||||
{query_id: query_records.get(_query_key(case.case_id, query_id), {}) for query_id in query_ids},
|
||||
round_id,
|
||||
policy,
|
||||
)
|
||||
query_weights[case.case_id] = weights
|
||||
ordered_queries = _weighted_order(query_ids, weights, rng, lambda value: value)
|
||||
open_case_slots = max(policy.max_cases - len(selected_cases), 1)
|
||||
fair_share = max(math.ceil(int(remaining_queries or 0) / open_case_slots), 1)
|
||||
take = min(len(ordered_queries), fair_share, int(remaining_queries or 0))
|
||||
query_ids = ordered_queries[:take]
|
||||
if not query_ids:
|
||||
continue
|
||||
remaining_queries = int(remaining_queries or 0) - len(query_ids)
|
||||
selected_cases.append(case.case_id)
|
||||
selected_queries[case.case_id] = query_ids
|
||||
if not selected_cases:
|
||||
raise ValueError("sampling budget selected no executable case/query pairs")
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"mode": "sampled",
|
||||
"round": round_id,
|
||||
"seed": seed,
|
||||
"policy": policy.model_dump(),
|
||||
"case_ids": selected_cases,
|
||||
"query_ids": selected_queries,
|
||||
"case_draw_weights": case_weights,
|
||||
"query_draw_weights": query_weights,
|
||||
"eligibility": dict(eligible_query_ids) if eligible_query_ids is not None else None,
|
||||
}
|
||||
|
||||
|
||||
def _observe(record: dict[str, Any], loss: float, round_id: int, alpha: float) -> None:
|
||||
previous = record.get("ewma_loss")
|
||||
record["ewma_loss"] = loss if previous is None else alpha * loss + (1.0 - alpha) * float(previous)
|
||||
record["observed_count"] += 1
|
||||
record["last_valid_score_round"] = round_id
|
||||
if loss > 0:
|
||||
record["last_error_round"] = round_id
|
||||
record["successful_observations_after_last_error"] = 0
|
||||
elif record.get("last_error_round") is not None:
|
||||
record["successful_observations_after_last_error"] += 1
|
||||
|
||||
|
||||
def _adaptive_weight(record: Mapping[str, Any], round_id: int, policy: SamplingPolicy) -> float:
|
||||
observed = int(record.get("observed_count", 0))
|
||||
raw_risk = record.get("ewma_loss")
|
||||
risk = float(raw_risk) if isinstance(raw_risk, (int, float)) and not isinstance(raw_risk, bool) else 0.5
|
||||
last_selected = record.get("last_selected_round")
|
||||
staleness = round_id if last_selected is None else max(round_id - int(last_selected), 0)
|
||||
uncertainty = 1.0 / math.sqrt(observed + 1.0)
|
||||
last_error = record.get("last_error_round")
|
||||
successes = int(record.get("successful_observations_after_last_error", 0))
|
||||
unrechecked_error = 0.0
|
||||
if last_error is not None:
|
||||
unrechecked_error = math.log1p(max(round_id - int(last_error), 0)) / (successes + 1.0)
|
||||
priority = risk + 0.2 * math.log1p(staleness) + 0.2 * uncertainty + 0.1 * unrechecked_error
|
||||
return math.exp(min(priority / policy.temperature, 50.0))
|
||||
|
||||
|
||||
def _mixed_weights(
|
||||
records: Mapping[str, Mapping[str, Any]],
|
||||
round_id: int,
|
||||
policy: SamplingPolicy,
|
||||
) -> dict[str, float]:
|
||||
"""Mix normalized adaptive weights with a true uniform exploration mass."""
|
||||
|
||||
adaptive = {key: _adaptive_weight(record, round_id, policy) for key, record in records.items()}
|
||||
total = sum(adaptive.values())
|
||||
uniform = 1.0 / len(adaptive)
|
||||
return {
|
||||
key: policy.uniform_exploration * uniform + (1.0 - policy.uniform_exploration) * value / total
|
||||
for key, value in adaptive.items()
|
||||
}
|
||||
|
||||
|
||||
def _weighted_order(values: list[Any], weights: Mapping[str, float], rng: random.Random, key: Any) -> list[Any]:
|
||||
return sorted(
|
||||
values,
|
||||
key=lambda value: rng.random() ** (1.0 / max(weights[key(value)], 1e-12)),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
|
||||
def _empty_record() -> dict[str, Any]:
|
||||
return {
|
||||
"observed_count": 0,
|
||||
"ewma_loss": None,
|
||||
"last_selected_round": None,
|
||||
"last_valid_score_round": None,
|
||||
"last_error_round": None,
|
||||
"successful_observations_after_last_error": 0,
|
||||
"candidate_failure_count": 0,
|
||||
"infra_error_count": 0,
|
||||
}
|
||||
|
||||
|
||||
def _query_key(case_id: str, query_id: str) -> str:
|
||||
return json.dumps([case_id, query_id], ensure_ascii=False, separators=(",", ":"))
|
||||
|
|
@ -6,6 +6,7 @@ from pathlib import Path
|
|||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from reme.schema import Response
|
||||
|
||||
|
|
@ -64,6 +65,13 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
|
|||
assert observed["plugins"] == ["home", "beam-judge" if benchmark == "beam" else "lme-judge"]
|
||||
assert observed["service"] == {"backend": "cli"}
|
||||
assert observed["workspace_dir"] == str(tmp_path / "experiment")
|
||||
assert observed["components"]["home.evaluation"]["default"]["sampling_policy"] == {
|
||||
"max_cases": 5,
|
||||
"max_queries": None,
|
||||
"temperature": 1.0,
|
||||
"ewma_alpha": 0.3,
|
||||
"uniform_exploration": 0.2,
|
||||
}
|
||||
application.start.assert_awaited_once()
|
||||
application.close.assert_awaited_once()
|
||||
assert application.run_job.await_count == 1
|
||||
|
|
@ -71,6 +79,36 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
|
|||
assert result["workspace_dir"] == str(tmp_path / "experiment")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch):
|
||||
"""Partial policies merge typed defaults and never leak into preparation inputs."""
|
||||
config = yaml.safe_load((ROOT / "benchmark/beam/config_home.yaml").read_text())
|
||||
config["sampling_policy"] = {"max_cases": 2, "max_queries": 7}
|
||||
path = tmp_path / "home.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
application = AsyncMock()
|
||||
application.run_job.return_value = Response(answer={})
|
||||
observed = {}
|
||||
|
||||
def factory(**kwargs):
|
||||
observed.update(kwargs)
|
||||
return application
|
||||
|
||||
monkeypatch.setattr(home_runner, "Application", factory)
|
||||
await home_runner.prepare(path)
|
||||
policy = observed["components"]["home.evaluation"]["default"]["sampling_policy"]
|
||||
assert policy["max_cases"] == 2 and policy["max_queries"] == 7 and policy["ewma_alpha"] == 0.3
|
||||
assert "sampling_policy" not in application.run_job.call_args.kwargs["spec"]
|
||||
for invalid in ({"max_cases": 0}, {"unknown": 1}, {"max_queries": "7"}):
|
||||
config["sampling_policy"] = invalid
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with pytest.raises(ValueError):
|
||||
await home_runner.prepare(path)
|
||||
assert application.start.await_count == 1
|
||||
assert application.run_job.await_count == 1
|
||||
assert application.run_job.call_args.args == ("home_prepare",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["startup", "preparation"])
|
||||
async def test_runner_propagates_failure_and_closes(monkeypatch, failure):
|
||||
|
|
|
|||
351
plugins/home/tests/test_bounded_agent.py
Normal file
351
plugins/home/tests/test_bounded_agent.py
Normal file
|
|
@ -0,0 +1,351 @@
|
|||
"""Home budgets exercised through AgentScope's real reply and event loop."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access,unused-argument
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from agentscope.agent import Agent, ReActConfig
|
||||
from agentscope.agent._config import ModelConfig
|
||||
from agentscope.event import ModelCallEndEvent, ReplyEndEvent, ReplyEndReason
|
||||
from agentscope.message import AssistantMsg, HintBlock, TextBlock, ThinkingBlock, ToolCallBlock, UserMsg
|
||||
from agentscope.model import ChatResponse, ChatUsage, FinishedReason
|
||||
from agentscope.tool import FunctionTool, Toolkit
|
||||
|
||||
from reme_home.agent import BoundedReActAgent, RunTimeBudgetTimer
|
||||
|
||||
|
||||
class RecordingModel:
|
||||
model = "isolated-test"
|
||||
context_size = 10000
|
||||
formatter = SimpleNamespace(supported_input_media_types=[])
|
||||
|
||||
def __init__(self, *, final=None, stream=False):
|
||||
self.calls = []
|
||||
self.final = final
|
||||
self.stream = stream
|
||||
|
||||
async def count_tokens(self, messages, tools):
|
||||
return 10
|
||||
|
||||
async def __call__(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
if kwargs["tool_choice"] is not None and kwargs["tool_choice"].mode == "none":
|
||||
if isinstance(self.final, BaseException):
|
||||
raise self.final
|
||||
response = self.final or ChatResponse(content=[TextBlock(text="final report")], is_last=True)
|
||||
if self.stream:
|
||||
|
||||
async def chunks():
|
||||
yield ChatResponse(content=[TextBlock(text="final ")], is_last=False)
|
||||
yield response
|
||||
|
||||
return chunks()
|
||||
return response
|
||||
return ChatResponse(
|
||||
content=[ThinkingBlock(thinking="investigating")],
|
||||
is_last=True,
|
||||
usage=ChatUsage(input_tokens=2, output_tokens=3, time=0),
|
||||
)
|
||||
|
||||
|
||||
def hints(agent, source):
|
||||
return [
|
||||
block
|
||||
for message in agent.state.context
|
||||
for block in message.content
|
||||
if isinstance(block, HintBlock) and block.source == source
|
||||
]
|
||||
|
||||
|
||||
def make_agent(model=None, timer=None, max_iters=2):
|
||||
return BoundedReActAgent(
|
||||
name="investigator",
|
||||
system_prompt="Investigate the evidence.",
|
||||
model=model or RecordingModel(),
|
||||
react_config=ReActConfig(max_iters=max_iters, interruption_raise_cancelled_error=True),
|
||||
model_config=ModelConfig(max_retries=0),
|
||||
run_time_budget_timer=timer,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("budget", [True, 0, -1, float("nan"), float("inf"), "10"])
|
||||
def test_timer_rejects_invalid_budget(budget):
|
||||
with pytest.raises(ValueError, match="positive"):
|
||||
RunTimeBudgetTimer(budget, agent_label="diagnostic")
|
||||
|
||||
|
||||
def test_timer_thresholds_restart_and_skipped_thresholds():
|
||||
now = [0.0]
|
||||
timer = RunTimeBudgetTimer(100, agent_label="diagnostic", clock=lambda: now[0])
|
||||
assert timer.next_reminder() is None
|
||||
timer.start()
|
||||
for percentage, persistent in [(59, None), (80, False), (90, True), (100, False), (110, True)]:
|
||||
now[0] = percentage * 60
|
||||
reminder = timer.next_reminder()
|
||||
if persistent is None:
|
||||
assert reminder is None
|
||||
else:
|
||||
assert reminder.persistent is persistent
|
||||
assert "does not terminate" in reminder.content
|
||||
assert timer.next_reminder() is None
|
||||
timer.record_context_compression()
|
||||
timer.stop()
|
||||
assert timer.next_reminder() is None
|
||||
timer.start()
|
||||
assert timer.context_compression_count == 0
|
||||
assert timer.next_reminder() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_native_loop_has_exactly_one_tool_free_finalization(stream):
|
||||
usage = ChatUsage(input_tokens=5, output_tokens=7, time=0, cache_input_tokens=2)
|
||||
model = RecordingModel(
|
||||
final=ChatResponse(content=[TextBlock(text="report")], is_last=True, usage=usage),
|
||||
stream=stream,
|
||||
)
|
||||
agent = make_agent(model)
|
||||
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
|
||||
agent.run_time_budget_timer = timer
|
||||
items = [
|
||||
item
|
||||
async for item in agent.reply_stream(UserMsg(name="caller", content="Find the cause"), yield_final_msg=True)
|
||||
]
|
||||
assert len(model.calls) == 3
|
||||
assert agent.state.cur_iter == 3
|
||||
request = model.calls[-1]
|
||||
assert request["tools"] == []
|
||||
assert request["tool_choice"].mode == "none"
|
||||
assert "Find the cause" in request["messages"][-1].get_text_content()
|
||||
assert items[-1].get_text_content() == "report"
|
||||
# Reminders must preserve cumulative reply usage as well as per-call events.
|
||||
assert items[-1].usage.input_tokens == 9
|
||||
assert items[-1].usage.output_tokens == 13
|
||||
assert items[-1].usage.cache_input_tokens == 2
|
||||
calls = [item for item in items if isinstance(item, ModelCallEndEvent)]
|
||||
assert len(calls) == 3
|
||||
assert sum(item.input_tokens for item in calls) == 9
|
||||
assert sum(item.output_tokens for item in calls) == 13
|
||||
ends = [item for item in items if isinstance(item, ReplyEndEvent)]
|
||||
assert len(ends) == 1 and ends[0].finished_reason == ReplyEndReason.EXCEED_MAX_ITERS
|
||||
assert not timer.active
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalization_discards_unsolicited_tools():
|
||||
executed = []
|
||||
|
||||
def forbidden():
|
||||
executed.append(True)
|
||||
|
||||
model = RecordingModel(
|
||||
final=ChatResponse(
|
||||
content=[TextBlock(text="report"), ToolCallBlock(id="call", name="forbidden", input="{}")],
|
||||
is_last=True,
|
||||
),
|
||||
)
|
||||
agent = make_agent(model)
|
||||
agent.toolkit = Toolkit(tools=[FunctionTool(forbidden)])
|
||||
result = await agent.reply(UserMsg(name="caller", content="Investigate"))
|
||||
assert result.get_text_content() == "report"
|
||||
assert not executed
|
||||
assert not agent.state.get_unfinished_tool_calls(agent.name)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"failure",
|
||||
[
|
||||
RuntimeError("provider unavailable"),
|
||||
ChatResponse(content=[], is_last=True),
|
||||
ChatResponse(content=[], is_last=True, usage=ChatUsage(input_tokens=5, output_tokens=7, time=0)),
|
||||
],
|
||||
)
|
||||
async def test_finalization_failure_returns_explicit_incomplete_output(failure):
|
||||
model = RecordingModel(final=failure)
|
||||
agent = make_agent(model)
|
||||
result = await agent.reply(UserMsg(name="caller", content="Investigate"))
|
||||
assert "incomplete prior model output" in result.get_text_content()
|
||||
assert len(model.calls) == 3
|
||||
final_usage = failure.usage if isinstance(failure, ChatResponse) else None
|
||||
assert result.usage.input_tokens == 4 + (final_usage.input_tokens if final_usage else 0)
|
||||
assert result.usage.output_tokens == 6 + (final_usage.output_tokens if final_usage else 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancellation_stops_timer_without_fallback():
|
||||
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
|
||||
agent = make_agent(RecordingModel(final=asyncio.CancelledError()), timer)
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await agent.reply(UserMsg(name="caller", content="Investigate"))
|
||||
assert not timer.active
|
||||
assert not any("incomplete prior model output" in (msg.get_text_content() or "") for msg in agent.state.context)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interrupted_response_preserves_native_interruption():
|
||||
model = RecordingModel(final=ChatResponse(content=[], is_last=True, finished_reason=FinishedReason.INTERRUPTED))
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await make_agent(model).reply(UserMsg(name="caller", content="Investigate"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_reply_resets_iteration_reminders():
|
||||
model = RecordingModel()
|
||||
agent = make_agent(model)
|
||||
for task in ("first", "second"):
|
||||
result = await agent.reply(UserMsg(name="caller", content=task))
|
||||
assert result.get_text_content() == "final report"
|
||||
assert agent._original_task_prompt == task
|
||||
assert len(model.calls) == 6
|
||||
reminders = hints(agent, "home_iteration_guard")
|
||||
assert len(reminders) == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normal_reply_does_not_force_finalization():
|
||||
class ImmediateModel(RecordingModel):
|
||||
async def __call__(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
return ChatResponse(content=[TextBlock(text="already sufficient")], is_last=True)
|
||||
|
||||
model = ImmediateModel()
|
||||
result = await make_agent(model).reply(UserMsg(name="caller", content="Investigate"))
|
||||
assert result.get_text_content() == "already sufficient"
|
||||
assert len(model.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_context_trims_old_evidence_but_preserves_system_and_task():
|
||||
class SmallContextModel(RecordingModel):
|
||||
async def count_tokens(self, messages, tools):
|
||||
return 10000 if len(messages) > 2 else 10
|
||||
|
||||
model = SmallContextModel()
|
||||
agent = make_agent(model)
|
||||
agent.state.cur_iter = 2
|
||||
agent._original_task_prompt = "original task"
|
||||
messages = [UserMsg(name="system", content="system prompt"), UserMsg(name="caller", content="old evidence")]
|
||||
await agent._call_model(messages, tools=[{"name": "unused"}])
|
||||
assert len(model.calls[-1]["messages"]) == 2
|
||||
assert model.calls[-1]["messages"][0] is messages[0]
|
||||
assert "original task" in model.calls[-1]["messages"][-1].get_text_content()
|
||||
assert len(messages) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_stream_completion_returns_previous_text_without_retry():
|
||||
class IncompleteStreamModel(RecordingModel):
|
||||
async def __call__(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
|
||||
async def chunks():
|
||||
yield ChatResponse(content=[TextBlock(text="partial")], is_last=False)
|
||||
|
||||
return chunks()
|
||||
|
||||
model = IncompleteStreamModel()
|
||||
agent = make_agent(model)
|
||||
agent.state.cur_iter = 2
|
||||
agent.state.context.append(AssistantMsg(name=agent.name, content="previous finding"))
|
||||
response = await agent._call_model([], tools=[])
|
||||
assert "incomplete prior model output" in response.content[0].text
|
||||
assert "previous finding" in response.content[0].text
|
||||
assert len(model.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_time_reminders_alternate_and_do_not_stop_investigation():
|
||||
now = [0.0]
|
||||
timer = RunTimeBudgetTimer(100, agent_label="diagnostic", clock=lambda: now[0])
|
||||
model = RecordingModel()
|
||||
agent = make_agent(model, timer, max_iters=100)
|
||||
timer.start()
|
||||
for percentage in (60, 75, 90, 100, 110):
|
||||
now[0] = percentage * 60
|
||||
_ = [item async for item in agent._reasoning()]
|
||||
persistent = hints(agent, "home_run_time_guard")
|
||||
assert len(persistent) == 2
|
||||
assert "(75%" in persistent[0].hint
|
||||
assert "(100%" in persistent[1].hint
|
||||
assert len(model.calls) == 5
|
||||
assert timer.active
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ephemeral_reminder_removed_when_reasoning_fails(monkeypatch):
|
||||
now = [0.0]
|
||||
timer = RunTimeBudgetTimer(1, agent_label="diagnostic", clock=lambda: now[0])
|
||||
agent = make_agent(timer=timer)
|
||||
timer.start()
|
||||
now[0] = 40
|
||||
|
||||
async def fail(*args, **kwargs):
|
||||
raise RuntimeError("failed")
|
||||
yield # pylint: disable=unreachable
|
||||
|
||||
monkeypatch.setattr(Agent, "_reasoning", fail)
|
||||
with pytest.raises(RuntimeError):
|
||||
_ = [item async for item in agent._reasoning()]
|
||||
assert not hints(agent, "home_run_time_guard")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_only_successful_context_replacement_counts_as_compression(monkeypatch):
|
||||
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
|
||||
agent = make_agent(timer=timer)
|
||||
|
||||
async def no_change(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(Agent, "compress_context", no_change)
|
||||
await agent.compress_context()
|
||||
assert timer.context_compression_count == 0
|
||||
|
||||
async def change(self):
|
||||
self.state.context = []
|
||||
|
||||
monkeypatch.setattr(Agent, "compress_context", change)
|
||||
await agent.compress_context()
|
||||
assert timer.context_compression_count == 1
|
||||
|
||||
|
||||
class CompressionFailureModel(RecordingModel):
|
||||
def __init__(self, failure, fail_after=2):
|
||||
super().__init__()
|
||||
self.failure = failure
|
||||
self.fail_after = fail_after
|
||||
|
||||
async def count_tokens(self, messages, tools):
|
||||
return 9000 if len(self.calls) >= self.fail_after and len(messages) > 2 else 10
|
||||
|
||||
async def generate_structured_output(self, **kwargs):
|
||||
raise self.failure
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_report_survives_real_compression_failure():
|
||||
model = CompressionFailureModel(RuntimeError("compression provider unavailable"))
|
||||
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
|
||||
agent = make_agent(model, timer)
|
||||
agent.context_config.compression_fallback_to_truncation = False
|
||||
result = await agent.reply(UserMsg(name="caller", content="Investigate"))
|
||||
assert result.get_text_content() == "final report"
|
||||
assert len(model.calls) == 3
|
||||
assert model.calls[-1]["tools"] == []
|
||||
assert len(model.calls[-1]["messages"]) == 2
|
||||
assert timer.context_compression_count == 0
|
||||
assert not timer.active
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", [RuntimeError("compression failed"), asyncio.CancelledError()])
|
||||
async def test_nonfinal_compression_errors_and_cancellation_propagate(failure):
|
||||
model = CompressionFailureModel(failure, fail_after=1 if isinstance(failure, RuntimeError) else 2)
|
||||
agent = make_agent(model)
|
||||
agent.context_config.compression_fallback_to_truncation = False
|
||||
with pytest.raises(type(failure)):
|
||||
await agent.reply(UserMsg(name="caller", content="Investigate"))
|
||||
assert len(model.calls) == model.fail_after
|
||||
210
plugins/home/tests/test_bounded_wrapper.py
Normal file
210
plugins/home/tests/test_bounded_wrapper.py
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
"""Bounded ReMe agents use real Job tools, injection and persisted sessions."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access
|
||||
|
||||
import json
|
||||
from importlib.metadata import EntryPoint
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from agentscope.message import HintBlock, TextBlock, ToolCallBlock
|
||||
from agentscope.model import ChatResponse, ChatUsage
|
||||
from agentscope.tool import FunctionTool, Toolkit
|
||||
from reme_home.agent import BoundedAgentWrapper, BoundedReActAgent, RunTimeBudgetTimer
|
||||
from test_bounded_agent import RecordingModel
|
||||
|
||||
from reme.components.agent_wrapper import AsAgentWrapper
|
||||
from reme.components.application_context import ApplicationContext
|
||||
from reme.components.job import BaseJob
|
||||
from reme.enumeration import ComponentEnum
|
||||
from reme.plugin import resolve_plugin_runtime
|
||||
from reme.steps.base_step import BaseStep
|
||||
from reme.utils import global_counter_get
|
||||
|
||||
|
||||
class CaptureStep(BaseStep):
|
||||
async def execute(self):
|
||||
self.app_context.metadata.setdefault("test.calls", []).append(dict(self.context.data))
|
||||
self.context.response.answer = "captured"
|
||||
|
||||
|
||||
class JobModel(RecordingModel):
|
||||
def __init__(self, arguments):
|
||||
super().__init__()
|
||||
self.arguments = arguments
|
||||
|
||||
async def __call__(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
content = (
|
||||
[ToolCallBlock(id="capture-call", name="capture", input=json.dumps(self.arguments))]
|
||||
if len(self.calls) == 1
|
||||
else [TextBlock(text="finished")]
|
||||
)
|
||||
return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0))
|
||||
|
||||
|
||||
@pytest.fixture(name="wrapper")
|
||||
def bounded_wrapper(tmp_path):
|
||||
context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
result = BoundedAgentWrapper(
|
||||
app_context=context,
|
||||
name="investigator",
|
||||
as_llm="",
|
||||
time_budget_minutes=30,
|
||||
react_config={"max_iters": 2},
|
||||
)
|
||||
result.as_llm = SimpleNamespace(model=RecordingModel())
|
||||
return result
|
||||
|
||||
|
||||
def test_plugin_registers_bounded_agent_only_when_enabled(monkeypatch):
|
||||
entry = EntryPoint(name="home", value="reme_home", group="reme.plugins")
|
||||
monkeypatch.setattr("reme.plugin.find_entry_points", lambda group, name: [entry])
|
||||
enabled = resolve_plugin_runtime({"plugins": ["home"]})
|
||||
assert enabled.registry.get(ComponentEnum.AGENT_WRAPPER, "home_bounded") is BoundedAgentWrapper
|
||||
assert resolve_plugin_runtime({}).registry.get(ComponentEnum.AGENT_WRAPPER, "home_bounded") is None
|
||||
assert issubclass(BoundedAgentWrapper, AsAgentWrapper)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_each_build_owns_timer_and_honors_call_overrides(wrapper):
|
||||
first, _ = await wrapper._build_agent("first", **wrapper._merged_kwargs({}))
|
||||
second, _ = await wrapper._build_agent("second", **wrapper._merged_kwargs({"time_budget_minutes": 5}))
|
||||
assert isinstance(first, BoundedReActAgent)
|
||||
assert first.run_time_budget_timer.time_budget_minutes == 30
|
||||
assert second.run_time_budget_timer.time_budget_minutes == 5
|
||||
assert first.run_time_budget_timer is not second.run_time_budget_timer
|
||||
assert first.react_config.max_iters == 2
|
||||
no_timer, _ = await wrapper._build_agent("third", **wrapper._merged_kwargs({"time_budget_minutes": None}))
|
||||
assert no_timer.run_time_budget_timer is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("conflict", [False, True])
|
||||
async def test_real_job_tools_enforce_hidden_injected_arguments(wrapper, conflict):
|
||||
context = wrapper.app_context
|
||||
context.registry.register(CaptureStep, "test_capture")
|
||||
job = BaseJob(
|
||||
name="capture",
|
||||
app_context=context,
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {"visible": {"type": "string"}, "case_ids": {"type": "array"}},
|
||||
"required": ["visible", "case_ids"],
|
||||
},
|
||||
steps=[{"backend": "test_capture"}],
|
||||
)
|
||||
context.jobs["capture"] = job
|
||||
arguments = {"visible": "model value"}
|
||||
if conflict:
|
||||
arguments["case_ids"] = ["model-chosen"]
|
||||
model = JobModel(arguments)
|
||||
wrapper.as_llm.model = model
|
||||
await job.start()
|
||||
await wrapper.start()
|
||||
try:
|
||||
result = await wrapper.reply(
|
||||
"Run the capture job",
|
||||
job_tools=["capture"],
|
||||
injected_job_kwargs={"case_ids": ["host-chosen"]},
|
||||
tool_context_id="investigation-1",
|
||||
)
|
||||
schemas = json.dumps(model.calls[0]["tools"])
|
||||
assert "case_ids" not in schemas
|
||||
assert "tool_context_id" not in schemas
|
||||
if conflict:
|
||||
assert "test.calls" not in context.metadata
|
||||
else:
|
||||
assert context.metadata["test.calls"] == [
|
||||
{
|
||||
"visible": "model value",
|
||||
"case_ids": ["host-chosen"],
|
||||
"tool_context_id": "investigation-1",
|
||||
},
|
||||
]
|
||||
assert result["result"] == "finished"
|
||||
assert (wrapper.session_path / f"{result['session_id']}.jsonl").is_file()
|
||||
resumed = await wrapper.reply("Continue", resume=result["session_id"])
|
||||
assert resumed["session_id"] == result["session_id"]
|
||||
finally:
|
||||
await wrapper.close()
|
||||
await job.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrapper_preserves_observed_task_in_forced_finalization(wrapper):
|
||||
result = await wrapper.reply("Identify the original failure")
|
||||
assert result["result"] == "final report"
|
||||
messages = wrapper.as_llm.model.calls[-1]["messages"]
|
||||
assert "Original task: Identify the original failure" in messages[-1].get_text_content()
|
||||
assert result["session_id"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrapper_streams_reme_chunks_and_persists_session(wrapper):
|
||||
chunks = [chunk async for chunk in wrapper.reply_stream("Investigate")]
|
||||
assert any(chunk.chunk == "final report" for chunk in chunks)
|
||||
session_id = chunks[-1].session_id
|
||||
assert session_id
|
||||
assert (wrapper.session_path / f"{session_id}.jsonl").is_file()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("timed", [False, True])
|
||||
async def test_multiround_usage_and_tools_continue_past_soft_time_limit(wrapper, monkeypatch, timed):
|
||||
now = [0.0]
|
||||
timer = RunTimeBudgetTimer(1, agent_label="investigator", clock=lambda: now[0])
|
||||
monkeypatch.setattr("reme_home.agent.wrapper.RunTimeBudgetTimer", lambda *args, **kwargs: timer)
|
||||
executions = []
|
||||
|
||||
async def inspect_evidence():
|
||||
executions.append(now[0])
|
||||
return "evidence"
|
||||
|
||||
class MultiRoundModel(RecordingModel):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.time_reminders = []
|
||||
|
||||
async def __call__(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
self.time_reminders.extend(
|
||||
block.hint
|
||||
for message in kwargs["messages"]
|
||||
for block in message.content
|
||||
if isinstance(block, HintBlock) and block.source == "home_run_time_guard"
|
||||
)
|
||||
now[0] += 65
|
||||
count = len(self.calls)
|
||||
content = (
|
||||
[ToolCallBlock(id=f"inspect-{count}", name="inspect_evidence", input="{}")]
|
||||
if count < 3
|
||||
else [TextBlock(text="finished")]
|
||||
)
|
||||
return ChatResponse(
|
||||
content=content,
|
||||
is_last=True,
|
||||
usage=ChatUsage(input_tokens=10, output_tokens=5, time=0),
|
||||
)
|
||||
|
||||
model = MultiRoundModel()
|
||||
wrapper.as_llm.model = model
|
||||
options = {
|
||||
"react_config": {"max_iters": 4},
|
||||
"time_budget_minutes": 1 if timed else None,
|
||||
"toolkit": Toolkit(tools=[FunctionTool(inspect_evidence)]),
|
||||
}
|
||||
result = await wrapper.reply("Investigate", **options)
|
||||
assert result["usage"] == {"input_tokens": 30, "output_tokens": 15, "total_tokens": 45}
|
||||
assert result["last_message"]["finished_reason"] == "completed"
|
||||
assert len(executions) == 2
|
||||
assert executions[-1] > 60 # A tool really executes after the soft deadline.
|
||||
assert len(model.calls) == 3
|
||||
assert all(call["tools"] for call in model.calls)
|
||||
assert bool(model.time_reminders) is timed
|
||||
assert not timer.active
|
||||
metadata = wrapper.app_context.metadata
|
||||
assert global_counter_get(metadata, ["__token_counter", wrapper.name, "total_tokens"]) == 45
|
||||
resumed = await wrapper.reply("Continue", resume=result["session_id"], **options)
|
||||
assert resumed["usage"]["total_tokens"] == 15
|
||||
assert global_counter_get(metadata, ["__token_counter", wrapper.name, "total_tokens"]) == 60
|
||||
316
plugins/home/tests/test_code.py
Normal file
316
plugins/home/tests/test_code.py
Normal file
|
|
@ -0,0 +1,316 @@
|
|||
"""Candidate version Jobs exercised against real isolated Git repositories."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name,unused-argument
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from threading import Event
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from reme_home.code import snapshot
|
||||
from reme_home.code.models import ARGUMENTS
|
||||
from reme_home.workspace import _initialization_lock
|
||||
|
||||
from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
|
||||
home_application,
|
||||
preparation_inputs,
|
||||
command,
|
||||
)
|
||||
|
||||
|
||||
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def prepared(app, inputs):
|
||||
spec, _ = inputs
|
||||
response = await app.run_job("home_prepare", spec=spec)
|
||||
assert response.success, response.answer
|
||||
repo = Path(app.config.workspace_dir) / "home/code/repo"
|
||||
return app, repo, spec
|
||||
|
||||
|
||||
async def run(app, job, **kwargs):
|
||||
response = await app.run_job(job, **kwargs)
|
||||
assert response.success, response.answer
|
||||
return response.answer
|
||||
|
||||
|
||||
async def create(app, name="candidate"):
|
||||
return await run(app, "home_create_candidate_branch", branch_name=name)
|
||||
|
||||
|
||||
async def commit(app, repo, text="value = 2\n"):
|
||||
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text(text)
|
||||
return await run(app, "home_commit_candidate", message="Improve candidate")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_branch_commit_diff_and_immutable_read(prepared):
|
||||
app, repo, _ = prepared
|
||||
initial = await run(app, "home_list_candidate_branches")
|
||||
assert initial["current_branch"] == "init"
|
||||
assert initial["candidate_branches"] == []
|
||||
created = await create(app)
|
||||
assert created["candidate_branch_count"] == 1
|
||||
result = await commit(app, repo)
|
||||
assert result["commit_sha"] != initial["commit_sha"]
|
||||
assert (await run(app, "home_code_status"))["clean"]
|
||||
diff = await run(app, "home_code_diff", base_revision="init", revision=result["commit_sha"])
|
||||
assert "+value = 2" in diff["content"]
|
||||
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text("dirty = 3\n")
|
||||
status = await run(app, "home_code_status")
|
||||
assert status["changes"] == [{"status": " M", "path": "plugins/beam/src/reme_beam/__init__.py"}]
|
||||
assert not status["clean"]
|
||||
assert "+dirty = 3" in (await run(app, "home_code_diff"))["content"]
|
||||
read = await run(
|
||||
app,
|
||||
"home_read_code_file",
|
||||
path="plugins/beam/src/reme_beam/__init__.py",
|
||||
revision=initial["commit_sha"],
|
||||
)
|
||||
assert read["content"] == "value = 1\n"
|
||||
with pytest.raises(ValueError, match="clean"):
|
||||
await app.context.components["home.code"]["default"].run("switch_branch", branch_name="init")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_host_pinned_job_tool_and_schema(prepared):
|
||||
app, repo, _ = prepared
|
||||
initial = command(repo, "rev-parse", "HEAD")
|
||||
await create(app)
|
||||
await commit(app, repo)
|
||||
job = app.context.jobs["home_read_code_file"]
|
||||
tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"revision": initial})
|
||||
assert "revision" not in tool.input_schema["properties"]
|
||||
# Test the actual wrapper's merge contract through its executable FunctionTool.
|
||||
response = await tool(path="plugins/beam/src/reme_beam/__init__.py")
|
||||
assert "value = 1" in response.content[0].text
|
||||
assert "value = 2" not in response.content[0].text
|
||||
with pytest.raises(ValueError, match="injected"):
|
||||
await tool(path="plugins/beam/src/reme_beam/__init__.py", revision="HEAD")
|
||||
assert all(model.model_json_schema()["additionalProperties"] is False for model in ARGUMENTS.values())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["init", "a/b", "../escape", "-bad", "a\\b", "@{-1}"])
|
||||
async def test_invalid_candidate_branch(prepared, name):
|
||||
app, _, _ = prepared
|
||||
response = await app.run_job("home_create_candidate_branch", branch_name=name)
|
||||
assert not response.success
|
||||
assert (await run(app, "home_code_status"))["current_branch"] == "init"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_protected_init_and_scope(prepared):
|
||||
app, repo, _ = prepared
|
||||
response = await app.run_job("home_commit_candidate", message="not allowed")
|
||||
assert not response.success
|
||||
await create(app)
|
||||
baseline = command(repo, "rev-parse", "HEAD")
|
||||
readme = repo / "plugins/beam/README.md"
|
||||
readme.write_text("outside declared editable roots")
|
||||
response = await app.run_job("home_commit_candidate", message="outside scope")
|
||||
assert not response.success
|
||||
assert command(repo, "rev-parse", "HEAD") == baseline
|
||||
assert not command(repo, "diff", "--cached", "--name-only")
|
||||
readme.write_text("Plugin installation instructions\n")
|
||||
target = repo / "plugins/beam/src/link"
|
||||
target.symlink_to("/etc/passwd")
|
||||
response = await app.run_job("home_commit_candidate", message="link")
|
||||
assert not response.success
|
||||
target.unlink()
|
||||
result = await commit(app, repo)
|
||||
command(repo, "update-ref", "refs/heads/init", result["commit_sha"])
|
||||
response = await app.run_job("home_code_status")
|
||||
assert not response.success
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_then_explicit_commit(prepared):
|
||||
app, repo, _ = prepared
|
||||
await create(app, "left")
|
||||
left = await commit(app, repo)
|
||||
await run(app, "home_switch_candidate_branch", branch_name="init")
|
||||
await create(app, "right")
|
||||
(repo / "plugins/beam/src/new.py").write_text("another mechanism\n")
|
||||
right = await run(app, "home_commit_candidate", message="right")
|
||||
merged = await run(app, "home_merge_candidate_branch", branch_name="left")
|
||||
assert merged["status"] == "merged_not_committed"
|
||||
assert merged["commit_sha"] == right["commit_sha"]
|
||||
assert (await run(app, "home_code_status"))["merge_pending"]
|
||||
response = await app.run_job("home_switch_candidate_branch", branch_name="init")
|
||||
assert not response.success
|
||||
merged = await run(app, "home_commit_candidate", message="Combine compatible improvements")
|
||||
assert command(repo, "rev-list", "--parents", "-n", "1", "HEAD").split() == [
|
||||
merged["commit_sha"],
|
||||
right["commit_sha"],
|
||||
left["commit_sha"],
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_conflicting_merge_aborts_cleanly(prepared):
|
||||
app, repo, _ = prepared
|
||||
await create(app, "left")
|
||||
await commit(app, repo, "left = 2\n")
|
||||
await run(app, "home_switch_candidate_branch", branch_name="init")
|
||||
await create(app, "right")
|
||||
right = await commit(app, repo, "right = 2\n")
|
||||
response = await app.run_job("home_merge_candidate_branch", branch_name="left")
|
||||
assert not response.success
|
||||
assert not (repo / ".git/MERGE_HEAD").exists()
|
||||
state = await run(app, "home_code_status")
|
||||
assert state["clean"] and state["commit_sha"] == right["commit_sha"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_snapshot_regex_paths_and_truncation(prepared):
|
||||
app, repo, _ = prepared
|
||||
await create(app)
|
||||
folder = repo / "plugins/beam/src"
|
||||
(folder / "space name.py").write_text("prefix\nhello\nworld\nsuffix\n" + "x\n" * 400)
|
||||
(folder / ".gitattributes").write_text('"space name.py" export-ignore\n')
|
||||
(folder / "binary").write_bytes(b"a\0b")
|
||||
(folder / "carriage.py").write_bytes(b"first\rsecond\rthird")
|
||||
(folder / "colon:name.py").write_text("hello")
|
||||
# Non-portable paths fail closed instead of becoming Git pathspecs.
|
||||
rejected = await app.run_job("home_commit_candidate", message="bad path")
|
||||
assert not rejected.success
|
||||
(folder / "colon:name.py").unlink()
|
||||
result = await run(app, "home_commit_candidate", message="Navigation fixtures")
|
||||
tree = await run(app, "home_list_code_tree", path="plugins/beam/src", max_results=1)
|
||||
assert tree["revision"] == result["commit_sha"] and tree["truncated"]
|
||||
found = await run(app, "home_search_code", pattern="hello\nworld", path="plugins/beam/src")
|
||||
assert found["occurrence_count"] == 1
|
||||
assert found["matches"][0]["start_line"] == 1
|
||||
assert "hello" in found["matches"][0]["content"]
|
||||
carriage = await run(app, "home_search_code", pattern="second", path="plugins/beam/src/carriage.py")
|
||||
assert "> 2 | second" in carriage["matches"][0]["content"]
|
||||
read = await run(app, "home_read_code_file", path="plugins/beam/src/space name.py")
|
||||
assert read["end_line"] == 300 and read["truncated"]
|
||||
for kwargs in [
|
||||
{"path": "../escape"},
|
||||
{"path": ".git/HEAD"},
|
||||
{"path": "/etc/passwd"},
|
||||
{"path": "plugins/beam/src/binary"},
|
||||
{"path": "plugins/beam/src", "revision": "--help"},
|
||||
]:
|
||||
assert not (await app.run_job("home_read_code_file", **kwargs)).success
|
||||
for pattern in ("[", "^", ""):
|
||||
assert not (await app.run_job("home_search_code", pattern=pattern)).success
|
||||
assert not (await app.run_job("home_list_code_tree", path=":(glob)**")).success
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lock_restrictions_and_unprepared(app):
|
||||
assert not (await app.run_job("home_code_status")).success
|
||||
assert not (await app.run_job("home_code_status", _allowed_paths=[])).success
|
||||
component = app.context.components["home.code"]["default"]
|
||||
with pytest.raises(ValueError, match="Unknown"):
|
||||
await component.run("reset")
|
||||
with pytest.raises(ValueError):
|
||||
await component.run("create_branch", branch_name=1)
|
||||
with _initialization_lock(component.workspace.state_path / "workspace.lock"):
|
||||
assert not (await app.run_job("home_list_code_tree")).success
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_waits_for_owned_git_thread(prepared, monkeypatch):
|
||||
app, _, _ = prepared
|
||||
component = app.context.components["home.code"]["default"]
|
||||
entered, release, finished = Event(), Event(), Event()
|
||||
original = component._run
|
||||
|
||||
def blocked(*args):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
try:
|
||||
return original(*args)
|
||||
finally:
|
||||
finished.set()
|
||||
|
||||
monkeypatch.setattr(component, "_run", blocked)
|
||||
task = asyncio.create_task(component.run("status"))
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
task.cancel()
|
||||
await asyncio.sleep(0)
|
||||
assert not task.done()
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert finished.is_set() and not component._pending
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_size_and_regex_timeout(prepared, monkeypatch):
|
||||
app, repo, _ = prepared
|
||||
await create(app)
|
||||
await commit(app, repo, 'x = "' + "a" * 10000 + '!"\n')
|
||||
original = snapshot.subprocess.run
|
||||
|
||||
def short_timeout(*args, **kwargs):
|
||||
if args[0][0] == snapshot.sys.executable:
|
||||
kwargs["timeout"] = 0.05
|
||||
return original(*args, **kwargs) # pylint: disable=subprocess-run-check
|
||||
|
||||
monkeypatch.setattr(snapshot.subprocess, "run", short_timeout)
|
||||
with pytest.raises(subprocess.TimeoutExpired):
|
||||
snapshot.search_code(repo, {}, pattern="(a+)+$", path="plugins/beam/src")
|
||||
monkeypatch.setattr(snapshot.subprocess, "run", original)
|
||||
monkeypatch.setattr(snapshot, "MAX_FILE_BYTES", 3)
|
||||
response = await app.run_job("home_read_code_file", path="plugins/beam/src/reme_beam/__init__.py")
|
||||
assert not response.success
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("mutation", ["staged", "untracked"])
|
||||
async def test_branch_operations_refuse_dirty_inputs(prepared, mutation):
|
||||
app, repo, _ = prepared
|
||||
if mutation == "staged":
|
||||
target = repo / "plugins/beam/src/reme_beam/__init__.py"
|
||||
target.write_text("staged = True\n")
|
||||
command(repo, "add", ".")
|
||||
else:
|
||||
target = repo / "plugins/beam/src/untracked file.py"
|
||||
target.write_text("untracked = True\n")
|
||||
before = target.read_bytes()
|
||||
assert not (await app.run_job("home_create_candidate_branch", branch_name="dirty")).success
|
||||
assert target.read_bytes() == before
|
||||
assert command(repo, "branch", "--show-current") == "init"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exact_paths_rename_deletion_and_revision_pinning(prepared):
|
||||
app, repo, _ = prepared
|
||||
await create(app)
|
||||
original = repo / "plugins/beam/src/reme_beam/__init__.py"
|
||||
name = "plugins/beam/src/space name\nand newline.py"
|
||||
target = repo / name
|
||||
original.rename(target)
|
||||
first = await run(app, "home_commit_candidate", message="Rename a file")
|
||||
assert (await run(app, "home_read_code_file", path=name))["content"] == "value = 1\n"
|
||||
target.write_text("value = 9\n")
|
||||
pinned = await run(app, "home_commit_candidate", message="Improve renamed code")
|
||||
target.unlink()
|
||||
await run(app, "home_commit_candidate", message="Remove obsolete file")
|
||||
await run(app, "home_switch_candidate_branch", branch_name="init")
|
||||
read = await run(app, "home_read_code_file", path=name, revision=pinned["commit_sha"])
|
||||
assert read["content"] == "value = 9\n"
|
||||
assert name in (await run(app, "home_list_code_tree", revision=first["commit_sha"]))["paths"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reject_symlink_repository_and_detached_candidate(prepared):
|
||||
app, repo, _ = prepared
|
||||
await create(app)
|
||||
commit_sha = (await commit(app, repo))["commit_sha"]
|
||||
command(repo, "checkout", "--detach", commit_sha)
|
||||
assert not (await app.run_job("home_commit_candidate", message="detached")).success
|
||||
assert (await run(app, "home_read_code_file", path="plugins/beam/src/reme_beam/__init__.py"))["content"]
|
||||
moved = repo.with_name("moved")
|
||||
repo.rename(moved)
|
||||
repo.symlink_to(moved, target_is_directory=True)
|
||||
assert not (await app.run_job("home_code_status")).success
|
||||
|
|
@ -126,6 +126,8 @@ class FakeFactory:
|
|||
|
||||
async def create_cases(self, ids, **kwargs):
|
||||
self.workers = [FakeWorker(self) for _ in ids]
|
||||
if self.workers and self.kwargs.get("on_sandbox_started"):
|
||||
self.kwargs["on_sandbox_started"]()
|
||||
return self.workers
|
||||
|
||||
|
||||
|
|
@ -209,6 +211,7 @@ async def test_reuse_tar_runs_current_candidate_without_building(app, prepared):
|
|||
baseline = await app.run_job("home_evaluate")
|
||||
assert baseline.success, baseline.answer
|
||||
repo = prepared / "code/repo"
|
||||
assert (await app.run_job("home_create_candidate_branch", branch_name="new-answer")).success
|
||||
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text("value = 2\n")
|
||||
command(repo, "add", ".")
|
||||
command(repo, "-c", "user.name=Fixture", "-c", "user.email=test@localhost", "commit", "-m", "new answer")
|
||||
|
|
@ -414,6 +417,7 @@ async def test_debug_empty_answers_fail_without_scoring(app, prepared, monkeypat
|
|||
@pytest.mark.asyncio
|
||||
async def test_current_head_is_immutable_and_does_not_ship_judge(app, prepared):
|
||||
repo = prepared / "code/repo"
|
||||
assert (await app.run_job("home_create_candidate_branch", branch_name="changed")).success
|
||||
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text("value = 3\n", encoding="utf-8")
|
||||
command(repo, "add", ".")
|
||||
command(repo, "-c", "user.name=Fixture", "-c", "user.email=test@localhost", "commit", "-m", "change")
|
||||
|
|
|
|||
360
plugins/home/tests/test_evaluation_history.py
Normal file
360
plugins/home/tests/test_evaluation_history.py
Normal file
|
|
@ -0,0 +1,360 @@
|
|||
"""Successful-history ownership and evaluation parameter precedence through real Jobs."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state
|
||||
from reme_home.sampling import SamplingPolicy, create_sampling_plan
|
||||
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
||||
FakeFactory,
|
||||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
)
|
||||
|
||||
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
|
||||
|
||||
|
||||
def history(app):
|
||||
return app.context.components["home.evaluation_history"]["default"]
|
||||
|
||||
|
||||
def manifest(response):
|
||||
return json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_catalog_keeps_successful_low_scores_and_deduplicates(app, prepared):
|
||||
manager = history(app)
|
||||
assert isinstance(manager, HomeEvaluationHistory) and manager.is_started
|
||||
assert manager.list_results() == []
|
||||
app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": '{"score": 0}'}
|
||||
response = await app.run_job("home_evaluate")
|
||||
assert response.success, response.answer
|
||||
results = manager.list_results()
|
||||
assert len(results) == 1 and isinstance(results[0], EvaluationRecord)
|
||||
assert results[0].mean_query_score == 0 and results[0].scored_query_count == 2
|
||||
assert manager.get_result(results[0].run_key) == results[0]
|
||||
run = Path(response.answer["result_dir"])
|
||||
before = manager.sampling_state()
|
||||
assert manager.record(run) == results[0]
|
||||
assert len(manager.list_results()) == 1 and manager.sampling_state() == before
|
||||
payload = json.loads((prepared / "evaluations/history.json").read_text())
|
||||
assert payload["results"][0]["run_key"] == results[0].run_key
|
||||
assert "golden_answer" not in json.dumps(payload)
|
||||
debug = await app.run_job("home_evaluate", sampling=True, debug=True)
|
||||
assert debug.success and manager.record(Path(debug.answer["result_dir"])) is None
|
||||
assert manager.list_results() == results
|
||||
with pytest.raises(KeyError):
|
||||
manager.get_result("unknown")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_run_does_not_change_next_probabilities_or_successful_history(app, prepared, monkeypatch):
|
||||
baseline = await app.run_job("home_evaluate")
|
||||
assert baseline.success
|
||||
manager = history(app)
|
||||
records, observations = manager.list_results(), manager.sampling_state()
|
||||
policy = SamplingPolicy()
|
||||
from reme_home.preparation.data.models import CaseSpec
|
||||
|
||||
cases = [
|
||||
CaseSpec.model_validate_json(path.read_text())
|
||||
for path in sorted(
|
||||
(prepared / "datasets/training-set/cases").glob("*.json"),
|
||||
)
|
||||
]
|
||||
monkeypatch.setattr("reme_home.sampling.time.time_ns", lambda: 5678)
|
||||
before = create_sampling_plan(cases, observations, policy)
|
||||
original = FakeFactory.create_cases
|
||||
|
||||
async def create(factory, *args, **kwargs):
|
||||
workers = await original(factory, *args, **kwargs)
|
||||
for worker in workers:
|
||||
build = worker.run_build
|
||||
|
||||
async def fail_one(jobs, worker=worker, build=build):
|
||||
if worker.case_id == "2":
|
||||
raise ConnectionError("only one case failed")
|
||||
return await build(jobs)
|
||||
|
||||
worker.run_build = fail_one
|
||||
return workers
|
||||
|
||||
monkeypatch.setattr(FakeFactory, "create_cases", create)
|
||||
failed = await app.run_job("home_evaluate", concurrency=1)
|
||||
assert not failed.success
|
||||
assert failed.answer["scored_query_count"] == 1 # Its healthy case is also excluded from feedback.
|
||||
assert manager.record(Path(failed.answer["result_dir"])) is None
|
||||
assert manager.list_results() == records and manager.sampling_state() == observations
|
||||
assert create_sampling_plan(cases, manager.sampling_state(), policy) == before
|
||||
assert read_state(prepared)["round"] == 2
|
||||
assert read_state(prepared)["successful_round"] == 1
|
||||
assert manager.rebuild() == records
|
||||
assert manager.sampling_state() == observations
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_and_observations_rebuild_without_touching_artifacts(app, prepared):
|
||||
baseline = await app.run_job("home_evaluate")
|
||||
sampled = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_cases": 1})
|
||||
assert baseline.success and sampled.success
|
||||
manager = history(app)
|
||||
expected, observations = manager.list_results(), manager.sampling_state()
|
||||
source = Path(sampled.answer["result_dir"]) / "summary.json"
|
||||
original = source.read_bytes()
|
||||
(prepared / "evaluations/history.json").unlink()
|
||||
(prepared / "sampling/state.json").unlink()
|
||||
assert manager.list_results() == expected
|
||||
assert manager.sampling_state() == observations
|
||||
assert manager.rebuild() == expected
|
||||
assert source.read_bytes() == original
|
||||
assert (prepared / "evaluations/history.json").is_file()
|
||||
assert (prepared / "sampling/state.json").is_file()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_policy_merges_component_defaults_without_mutating_them(app, prepared):
|
||||
evaluator = app.context.components["home.evaluation"]["default"]
|
||||
evaluator.sampling_policy = SamplingPolicy(max_cases=2, max_queries=2, temperature=0.7, ewma_alpha=0.6)
|
||||
response = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_cases": 1})
|
||||
assert response.success, response.answer
|
||||
options = manifest(response)["options"]
|
||||
assert options["sampling"] is True
|
||||
assert options["sampling_policy"] == {
|
||||
"max_cases": 1,
|
||||
"max_queries": 2,
|
||||
"temperature": 0.7,
|
||||
"ewma_alpha": 0.6,
|
||||
"uniform_exploration": 0.2,
|
||||
}
|
||||
assert response.answer["case_count"] == 1
|
||||
following = await app.run_job("home_evaluate", sampling=True)
|
||||
assert following.success and following.answer["case_count"] == 2
|
||||
assert evaluator.sampling_policy.max_cases == 2
|
||||
assert manifest(following)["options"]["sampling_policy"]["max_cases"] == 2
|
||||
for kwargs in (
|
||||
{"sampling": "true"},
|
||||
{"sampling": {"max_cases": 1}},
|
||||
{"record_history": "false"},
|
||||
{"sampling_policy": {"max_cases": 0}},
|
||||
{"sampling_policy": {"unknown": 1}},
|
||||
):
|
||||
assert not (await app.run_job("home_evaluate", **kwargs)).success
|
||||
assert len(history(app).list_results()) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_job_defaults_and_hidden_agent_injection_use_one_job(app, prepared):
|
||||
job = app.context.jobs["home_evaluate"]
|
||||
assert "home_validate" not in app.context.jobs
|
||||
job.kwargs.update(sampling=True, sampling_policy={"max_cases": 1})
|
||||
default = await app.run_job("home_evaluate")
|
||||
assert default.success and default.answer["case_count"] == 1
|
||||
explicit = await app.run_job("home_evaluate", sampling=False)
|
||||
assert explicit.success and explicit.answer["case_count"] == 2
|
||||
tool = AsAgentWrapper._make_tool(
|
||||
job,
|
||||
injected_job_kwargs={
|
||||
"sampling": True,
|
||||
"sampling_policy": {"max_cases": 1, "max_queries": 1},
|
||||
},
|
||||
)
|
||||
assert "sampling" not in tool.input_schema["properties"]
|
||||
await tool.call()
|
||||
records = history(app).list_results()
|
||||
assert records[-1].case_count == records[-1].scored_query_count == 1
|
||||
assert len(records) == 3
|
||||
with pytest.raises(ValueError, match="injected"):
|
||||
await tool.call(sampling=False)
|
||||
with pytest.raises(ValueError, match="injected"):
|
||||
await tool.call(sampling_policy={"max_cases": 100})
|
||||
assert len(history(app).list_results()) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("baseline", [False, True])
|
||||
async def test_standalone_full_evaluation_preserves_history_and_rebuild_excludes_it(app, prepared, baseline):
|
||||
manager = history(app)
|
||||
if baseline:
|
||||
assert (await app.run_job("home_evaluate")).success
|
||||
records, observations = manager.list_results(), manager.sampling_state()
|
||||
cache_paths = [prepared / name for name in ("evaluations/history.json", "sampling/state.json")]
|
||||
caches = [path.read_bytes() if path.exists() else None for path in cache_paths]
|
||||
response = await app.run_job("home_evaluate", sampling=False, record_history=False)
|
||||
assert response.success, response.answer
|
||||
assert response.answer["case_count"] == response.answer["scored_query_count"] == 2
|
||||
assert response.answer["mean_query_score"] == 0.5 and response.answer["record_history"] is False
|
||||
run = Path(response.answer["result_dir"])
|
||||
payload = manifest(response)
|
||||
assert payload["status"] == "completed" and payload["success"] is True
|
||||
assert payload["options"]["record_history"] is False and payload["sandbox_started_at"]
|
||||
assert "sampling" not in payload and "selection_plan" not in payload
|
||||
assert (run / "summary.json").is_file() and (run / "cases/1/queries/result.json").is_file()
|
||||
assert manager.record_started(run, SamplingPolicy()) == {}
|
||||
assert manager.record(run) is None
|
||||
assert [path.read_bytes() if path.exists() else None for path in cache_paths] == caches
|
||||
assert manager.list_results() == records and manager.sampling_state() == observations
|
||||
assert manager.rebuild() == records and manager.sampling_state() == observations
|
||||
following = await app.run_job("home_evaluate")
|
||||
assert following.success and following.answer["record_history"] is True
|
||||
state = read_state(prepared)
|
||||
assert state["round"] == state["successful_round"] == 1 + int(baseline)
|
||||
# Legacy manifests without the new flag keep the previous opt-in behavior.
|
||||
tracked_run = Path(following.answer["result_dir"])
|
||||
tracked_manifest = manifest(following)
|
||||
tracked_manifest["options"].pop("record_history")
|
||||
(tracked_run / "manifest.json").write_text(json.dumps(tracked_manifest))
|
||||
assert len(manager.rebuild()) == 1 + int(baseline)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untracked_full_evaluation_does_not_consult_history(app, prepared, monkeypatch):
|
||||
manager = history(app)
|
||||
|
||||
def fail(*args, **kwargs):
|
||||
raise AssertionError("standalone full evaluation must not access history")
|
||||
|
||||
for name in ("sampling_state", "record_started", "record", "invalidate"):
|
||||
monkeypatch.setattr(manager, name, fail)
|
||||
response = await app.run_job("home_evaluate", record_history=False)
|
||||
assert response.success, response.answer
|
||||
assert response.answer["case_count"] == response.answer["scored_query_count"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sampling_and_history_are_independent_and_selection_evidence_is_local(app, prepared):
|
||||
assert (await app.run_job("home_evaluate")).success
|
||||
manager = history(app)
|
||||
records, observations = manager.list_results(), manager.sampling_state()
|
||||
response = await app.run_job(
|
||||
"home_evaluate",
|
||||
sampling=True,
|
||||
record_history=False,
|
||||
sampling_policy={"max_cases": 1},
|
||||
)
|
||||
assert response.success and response.answer["case_count"] == 1
|
||||
payload = manifest(response)
|
||||
assert "sampling" not in payload
|
||||
assert payload["selection_plan"]["case_ids"] == payload["case_ids"]
|
||||
assert payload["selection_plan"]["seed"] and payload["selection_plan"]["policy"]["max_cases"] == 1
|
||||
assert manager.rebuild() == records and manager.sampling_state() == observations
|
||||
assert read_state(prepared)["round"] == read_state(prepared)["successful_round"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["infra", "exception"])
|
||||
async def test_untracked_failures_preserve_existing_history(app, prepared, monkeypatch, failure):
|
||||
assert (await app.run_job("home_evaluate")).success
|
||||
manager = history(app)
|
||||
records, observations = manager.list_results(), manager.sampling_state()
|
||||
paths = [prepared / name for name in ("evaluations/history.json", "sampling/state.json")]
|
||||
caches = [path.read_bytes() for path in paths]
|
||||
if failure == "infra":
|
||||
monkeypatch.setattr(FakeFactory, "bad_export", True)
|
||||
else:
|
||||
original = FakeFactory.create_cases
|
||||
|
||||
async def fail(factory, *args, **kwargs):
|
||||
await original(factory, *args, **kwargs)
|
||||
raise ConnectionError("pool failed after first initialized sandbox")
|
||||
|
||||
monkeypatch.setattr(FakeFactory, "create_cases", fail)
|
||||
response = await app.run_job("home_evaluate", record_history=False)
|
||||
assert not response.success
|
||||
assert [path.read_bytes() for path in paths] == caches
|
||||
assert manager.rebuild() == records and manager.sampling_state() == observations
|
||||
assert read_state(prepared)["round"] == read_state(prepared)["successful_round"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_job_defaults_and_agent_injection_are_enforced(app, prepared):
|
||||
job = app.context.jobs["home_evaluate"]
|
||||
assert job.parameters["properties"]["record_history"]["default"] is True
|
||||
job.kwargs["record_history"] = False
|
||||
standalone = await app.run_job("home_evaluate")
|
||||
assert standalone.success and not history(app).list_results()
|
||||
baseline = await app.run_job("home_evaluate", sampling=False, record_history=True)
|
||||
assert baseline.success and baseline.answer["case_count"] == 2
|
||||
tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"record_history": True})
|
||||
assert "record_history" not in tool.input_schema["properties"]
|
||||
await tool.call()
|
||||
assert len(history(app).list_results()) == 2
|
||||
with pytest.raises(ValueError, match="injected"):
|
||||
await tool.call(record_history=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_cache_rebuild_and_explicit_null_query_budget(app, prepared):
|
||||
evaluator = app.context.components["home.evaluation"]["default"]
|
||||
evaluator.sampling_policy = SamplingPolicy(max_cases=2, max_queries=1)
|
||||
result = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_queries": None})
|
||||
assert result.success and result.answer["scored_query_count"] == 2
|
||||
assert manifest(result)["options"]["sampling_policy"]["max_queries"] is None
|
||||
assert evaluator.sampling_policy.max_queries == 1
|
||||
manager = history(app)
|
||||
observations = manager.sampling_state()
|
||||
legacy = read_state(prepared)
|
||||
legacy["schema_version"] = 1
|
||||
legacy.pop("successful_round")
|
||||
legacy["cases"]["1"]["ewma_loss"] = 999
|
||||
(prepared / "sampling/state.json").write_text(json.dumps(legacy))
|
||||
assert manager.sampling_state() == observations
|
||||
manager.rebuild()
|
||||
assert read_state(prepared)["schema_version"] == 2
|
||||
assert read_state(prepared)["successful_round"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancellation_after_durable_completion_keeps_history_consistent(app, prepared, monkeypatch):
|
||||
import asyncio
|
||||
from threading import Event
|
||||
|
||||
manager = history(app)
|
||||
original = manager.record
|
||||
entered, release = Event(), Event()
|
||||
|
||||
def blocked(run):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return original(run)
|
||||
|
||||
monkeypatch.setattr(manager, "record", blocked)
|
||||
task = asyncio.create_task(app.run_job("home_evaluate"))
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
task.cancel()
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
records = manager.list_results()
|
||||
assert len(records) == 1
|
||||
run = prepared / "evaluations" / records[0].run_key
|
||||
completed = json.loads((run / "manifest.json").read_text())
|
||||
assert completed["status"] == "completed" and completed["success"] is True
|
||||
assert manager.rebuild() == records
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registration_failure_invalidates_only_caches(app, prepared, monkeypatch):
|
||||
manager = history(app)
|
||||
original = manager._index
|
||||
|
||||
def fail():
|
||||
raise OSError("history index could not be loaded")
|
||||
|
||||
monkeypatch.setattr(manager, "_index", fail)
|
||||
response = await app.run_job("home_evaluate")
|
||||
assert not response.success
|
||||
runs = list((prepared / "evaluations").glob("*/*/*/manifest.json"))
|
||||
assert len(runs) == 1
|
||||
run = runs[0].parent
|
||||
failed = json.loads(runs[0].read_text())
|
||||
assert failed["status"] == "failed" and failed["success"] is False
|
||||
assert (run / "summary.json").is_file() and (run / "cases/1/queries/result.json").is_file()
|
||||
assert not (prepared / "sampling/state.json").exists()
|
||||
assert read_state(prepared)["round"] == 1 and read_state(prepared)["successful_round"] == 0
|
||||
monkeypatch.setattr(manager, "_index", original)
|
||||
assert manager.list_results() == [] and not manager.sampling_state()["cases"]
|
||||
312
plugins/home/tests/test_evaluation_inspection.py
Normal file
312
plugins/home/tests/test_evaluation_inspection.py
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
"""Read-only result Jobs against real Home applications and isolated Git history."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
|
||||
import pytest
|
||||
from reme_home.evaluation import inspection_step as inspection
|
||||
from reme_home.evaluation.artifacts import write_json
|
||||
|
||||
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
||||
FakeFactory,
|
||||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
)
|
||||
from test_preparation import command
|
||||
|
||||
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
|
||||
|
||||
|
||||
async def call(app, name, **kwargs):
|
||||
response = await app.run_job(name, **kwargs)
|
||||
assert response.success, response.answer
|
||||
return response.answer
|
||||
|
||||
|
||||
def snapshot(home):
|
||||
return {path.relative_to(home): path.read_bytes() for path in home.rglob("*") if path.is_file()}
|
||||
|
||||
|
||||
def write_run(home, run_id="1", *, branch="candidate", commit="a" * 40, **updates):
|
||||
run = home / "evaluations" / branch / commit[:7] / run_id
|
||||
manifest = {
|
||||
"branch_name": branch,
|
||||
"commit_sha": commit,
|
||||
"validation_id": run_id,
|
||||
"case_ids": ["case~a", "missing-case"],
|
||||
"query_ids": {"case~a": ["q1", "q2"], "missing-case": ["q3"]},
|
||||
"mode": "scored",
|
||||
"dataset_kind": "training-set",
|
||||
"status": "failed",
|
||||
"started_at": f"2026-01-01T00:00:{int(run_id):02d}+00:00",
|
||||
"options": {"record_history": False},
|
||||
**updates,
|
||||
}
|
||||
write_json(run / "manifest.json", manifest)
|
||||
return run
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_evaluation_catalog_and_case_query_navigation_are_read_only(app, prepared):
|
||||
scored = await call(app, "home_evaluate", record_history=False)
|
||||
debug = await call(app, "home_evaluate", debug=True)
|
||||
before = snapshot(prepared)
|
||||
page = await call(app, "home_list_validation_results", limit=1)
|
||||
assert page["row_count"] == 2 and page["next_offset"] == 1
|
||||
assert page["rows"][0]["mode"] == "debug" and page["rows"][0]["record_history"] is False
|
||||
last = await call(app, "home_list_validation_results", offset=1, limit=1)
|
||||
assert last["next_offset"] is None and last["rows"][0]["validation_path"] == scored["result_dir"]
|
||||
assert last["rows"][0]["totals"]["scored_query_count"] == 2
|
||||
run = Path(scored["result_dir"])
|
||||
result = await call(app, "home_inspect_validation", validation_path=run.relative_to(prepared).as_posix(), limit=1)
|
||||
assert result["row_count"] == 2 and result["next_offset"] == 1
|
||||
case = result["rows"][0]
|
||||
assert case["mean_score"] == 0.5 and case["pointer"].startswith("/cases/")
|
||||
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id=case["case_id"])
|
||||
query = result["rows"][0]
|
||||
assert query["score"] == 0.5 and query["pointer"] == "/queries/0"
|
||||
assert "fields" not in query and result["construction"]["failed_jobs"]["rows"] == []
|
||||
result = await call(
|
||||
app,
|
||||
"home_inspect_validation",
|
||||
validation_path=str(run),
|
||||
case_id=case["case_id"],
|
||||
query_id=query["query_id"],
|
||||
)
|
||||
fields = result["rows"][0]["fields"]
|
||||
assert fields["answer"] == "hello" and fields["score"] == 0.5 and "judge_result" in fields
|
||||
assert result["query_missing"] is False and result["answer_log"].endswith("answer.log")
|
||||
assert snapshot(prepared) == before
|
||||
assert app.context.components["home.evaluation_history"]["default"].list_results() == []
|
||||
assert debug["result_dir"] != scored["result_dir"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_missing_and_corrupt_runs_are_visible_without_rebuilding(app, prepared):
|
||||
run = write_run(prepared)
|
||||
write_json(
|
||||
run / "partial_summary.json",
|
||||
{
|
||||
"case_count": 1,
|
||||
"cases": {
|
||||
"case~a": {
|
||||
"mean_score": 0.0,
|
||||
"construction": {"status": "partial_success"},
|
||||
"construction_error_count": 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
write_json(run / "failure.json", {"error": "failure" * 300})
|
||||
missing = write_run(prepared, "2", mode="debug", status="running")
|
||||
corrupt = write_run(prepared, "3")
|
||||
(corrupt / "manifest.json").write_text("invalid JSON")
|
||||
before = snapshot(prepared)
|
||||
result = await call(app, "home_list_validation_results")
|
||||
assert result["row_count"] == 3
|
||||
assert sum("read_error" in row for row in result["rows"]) == 1
|
||||
row = next(row for row in result["rows"] if row["validation_path"] == str(run))
|
||||
assert row["summary_path"].endswith("partial_summary.json") and "truncated" in row["failure"]
|
||||
result = await call(app, "home_inspect_validation", validation_path=str(run))
|
||||
assert result["rows"][0]["case_id"] == "missing-case" and result["rows"][0]["mean_score"] is None
|
||||
assert result["rows"][1]["pointer"] == "/cases/case~0a"
|
||||
result = await call(app, "home_inspect_validation", validation_path=str(missing), case_id="case~a", query_id="q1")
|
||||
assert result["query_missing"] and result["rows"][0]["score"] is None
|
||||
assert len(result["missing_files"]) == 2
|
||||
result = await call(app, "home_inspect_validation", validation_path=str(missing))
|
||||
assert result["summary_missing"] and result["rows_source"] is None and result["row_count"] == 2
|
||||
assert snapshot(prepared) == before and not (missing / "partial_summary.json").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_and_query_paging_previews_and_source_coordinates(app, prepared):
|
||||
run = write_run(prepared)
|
||||
folder = run / "cases/case~a"
|
||||
write_json(
|
||||
folder / "memory_construction/result.json",
|
||||
{
|
||||
"success": False,
|
||||
"jobs": [{"job": f"job-{index}", "result": {"success": False, "error": "x" * 900}} for index in range(3)],
|
||||
},
|
||||
)
|
||||
write_json(
|
||||
folder / "queries/result.json",
|
||||
{
|
||||
"queries": [
|
||||
{
|
||||
"query_id": "q2",
|
||||
"score": 0,
|
||||
"answer": "a" * 3000,
|
||||
"judge_result": {"success": True},
|
||||
"golden_answer": ["rubric"],
|
||||
"answer_result": None,
|
||||
},
|
||||
{"query_id": "q1", "score": None, "error": "e" * 900},
|
||||
],
|
||||
},
|
||||
)
|
||||
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", limit=1)
|
||||
assert result["rows"][0]["query_id"] == "q1" and result["rows"][0]["pointer"] == "/queries/1"
|
||||
assert result["next_offset"] == 1 and "truncated" in result["rows"][0]["error"]
|
||||
failed_jobs = result["construction"]["failed_jobs"]
|
||||
assert failed_jobs["row_count"] == 3 and failed_jobs["next_offset"] == 1
|
||||
assert failed_jobs["rows"][0]["pointer"] == "/jobs/0/result"
|
||||
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", offset=2, limit=1)
|
||||
assert result["rows"] == [] and result["construction"]["failed_jobs"]["rows"][0]["job"] == "job-2"
|
||||
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", query_id="q2")
|
||||
assert result["rows"][0]["pointer"] == "/queries/0"
|
||||
assert "truncated" in result["rows"][0]["fields"]["answer"]
|
||||
assert result["rows"][0]["fields"]["golden_answer"] == ["rubric"]
|
||||
assert result["rows"][0]["score"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prefix_resolves_on_branch_history_not_only_evaluated_commits(app, prepared):
|
||||
repo = prepared / "code/repo"
|
||||
initial = command(repo, "rev-parse", "HEAD")
|
||||
await call(app, "home_create_candidate_branch", branch_name="candidate")
|
||||
write_run(prepared, commit=initial)
|
||||
write_run(prepared, branch="other", commit=initial)
|
||||
result = await call(app, "home_list_validation_results", commit_sha=initial[:7].upper())
|
||||
assert result["branch_name"] == "candidate" and result["commit_sha"] == initial
|
||||
assert result["row_count"] == 1
|
||||
# An unevaluated reachable commit can make a short prefix ambiguous.
|
||||
tree = command(repo, "rev-parse", "HEAD^{tree}")
|
||||
for index in range(500):
|
||||
sha = command(
|
||||
repo,
|
||||
"-c",
|
||||
"user.name=Home",
|
||||
"-c",
|
||||
"user.email=home@local",
|
||||
"commit-tree",
|
||||
tree,
|
||||
"-p",
|
||||
initial,
|
||||
"-m",
|
||||
f"unevaluated {index}",
|
||||
)
|
||||
if sha[0] == initial[0]:
|
||||
break
|
||||
else:
|
||||
pytest.fail("Could not construct a prefix collision")
|
||||
command(repo, "update-ref", "refs/heads/candidate", sha)
|
||||
response = await app.run_job("home_list_validation_results", commit_sha=initial[0])
|
||||
assert not response.success and "2 matches" in response.answer
|
||||
# The very same prefix remains unique on init, despite the other Git object.
|
||||
result = await call(app, "home_list_validation_results", branch_name="init", commit_sha=initial[0])
|
||||
assert result["commit_sha"] == initial and result["row_count"] == 0
|
||||
result = await call(app, "home_list_validation_results", commit_sha=sha)
|
||||
assert result["commit_sha"] == sha and result["rows"] == []
|
||||
absent = "f" * len(initial)
|
||||
assert not (await app.run_job("home_list_validation_results", commit_sha=absent)).success
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_branch_filters_and_deleted_historical_branch_do_not_require_git(app, prepared):
|
||||
write_run(prepared, branch="removed")
|
||||
write_run(prepared, "2", branch="candidate")
|
||||
result = await call(app, "home_list_validation_results", branch_name="removed")
|
||||
assert result["row_count"] == 1 and result["rows"][0]["branch_name"] == "removed"
|
||||
response = await app.run_job("home_list_validation_results", branch_name="removed", commit_sha="a")
|
||||
assert not response.success
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("corruption", ["summary", "options"])
|
||||
async def test_commit_filtered_catalog_reports_corrupt_run_with_identity(app, prepared, corruption):
|
||||
sha = command(prepared / "code/repo", "rev-parse", "HEAD")
|
||||
run = write_run(prepared, branch="init", commit=sha, options=None if corruption == "options" else {})
|
||||
if corruption == "summary":
|
||||
write_json(run / "summary.json", [])
|
||||
result = await call(app, "home_list_validation_results", commit_sha=sha[:7])
|
||||
assert result["row_count"] == 1
|
||||
assert result["rows"][0]["commit_sha"] == sha and "read_error" in result["rows"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"job,kwargs",
|
||||
[
|
||||
("home_list_validation_results", {"offset": -1}),
|
||||
("home_list_validation_results", {"limit": 0}),
|
||||
("home_list_validation_results", {"limit": 101}),
|
||||
("home_list_validation_results", {"limit": True}),
|
||||
("home_list_validation_results", {"offset": "1"}),
|
||||
("home_list_validation_results", {"commit_sha": "HEAD"}),
|
||||
("home_list_validation_results", {"commit_sha": ""}),
|
||||
("home_list_validation_results", {"branch_name": "../init"}),
|
||||
("home_inspect_validation", {}),
|
||||
("home_inspect_validation", {"validation_path": "evaluations"}),
|
||||
("home_inspect_validation", {"validation_path": "../outside"}),
|
||||
("home_inspect_validation", {"validation_path": "/tmp/outside"}),
|
||||
("home_inspect_validation", {"validation_path": "evaluations/a/b/c", "query_id": "q"}),
|
||||
("home_inspect_validation", {"validation_path": "evaluations/a/b/c", "case_id": "a/b"}),
|
||||
("home_list_validation_results", {"_allowed_paths": []}),
|
||||
],
|
||||
)
|
||||
async def test_invalid_input_fails_closed(app, prepared, job, kwargs):
|
||||
assert not (await app.run_job(job, **kwargs)).success
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_ids_and_symlink_artifacts_are_rejected(app, prepared, tmp_path):
|
||||
run = write_run(prepared)
|
||||
assert not (await app.run_job("home_inspect_validation", validation_path=str(run), case_id="other")).success
|
||||
assert not (
|
||||
await app.run_job(
|
||||
"home_inspect_validation",
|
||||
validation_path=str(run),
|
||||
case_id="case~a",
|
||||
query_id="other",
|
||||
)
|
||||
).success
|
||||
secret = tmp_path / "secret.json"
|
||||
secret.write_text('{"secret": "must not be read"}')
|
||||
(run / "summary.json").symlink_to(secret)
|
||||
response = await app.run_job("home_inspect_validation", validation_path=str(run))
|
||||
assert not response.success and "must not be read" not in response.answer
|
||||
result = await call(app, "home_list_validation_results")
|
||||
assert "read_error" in result["rows"][0] and "secret" not in str(result["rows"])
|
||||
(run / "summary.json").unlink()
|
||||
(run / "cases").symlink_to(tmp_path, target_is_directory=True)
|
||||
assert not (await app.run_job("home_inspect_validation", validation_path=str(run), case_id="case~a")).success
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hidden_case_scope_is_enforced_by_agent_job_injection(app, prepared):
|
||||
run = write_run(prepared)
|
||||
job = app.context.jobs["home_inspect_validation"]
|
||||
tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"validation_path": str(run), "case_id": "case~a"})
|
||||
assert "validation_path" not in tool.input_schema["properties"] and "case_id" not in tool.input_schema["properties"]
|
||||
await tool.call(query_id="q1")
|
||||
with pytest.raises(ValueError, match="injected"):
|
||||
await tool.call(case_id="missing-case")
|
||||
with pytest.raises(ValueError, match="injected"):
|
||||
await tool.call(validation_path="evaluations/other/commit/1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_job_drains_read_thread(app, prepared, monkeypatch):
|
||||
entered, release, finished = Event(), Event(), Event()
|
||||
|
||||
def read(*args):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
finished.set()
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(inspection, "_list_results", read)
|
||||
task = asyncio.create_task(app.run_job("home_list_validation_results"))
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
task.cancel()
|
||||
await asyncio.sleep(0)
|
||||
assert not task.done()
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert finished.is_set()
|
||||
230
plugins/home/tests/test_memory_inspection.py
Normal file
230
plugins/home/tests/test_memory_inspection.py
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
"""Memory evidence Jobs against isolated real Git repositories."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
|
||||
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from reme_home.evaluation import memory_inspection_step as memory
|
||||
from reme_home.preparation.code import git
|
||||
|
||||
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
||||
FakeFactory,
|
||||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
)
|
||||
from test_evaluation_inspection import call, snapshot, write_run
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def evidence(tmp_path):
|
||||
home = tmp_path / "home"
|
||||
run = write_run(home, case_ids=["case1"])
|
||||
repo = run / "cases/case1/memory_construction/reme_workspace"
|
||||
repo.mkdir(parents=True)
|
||||
git(repo, "init")
|
||||
git(repo, "config", "user.name", "test")
|
||||
git(repo, "config", "user.email", "test@localhost")
|
||||
(repo / "memory").mkdir()
|
||||
(repo / "memory/note.md").write_text("old fact\n")
|
||||
(repo / "removed.txt").write_text("removed\n")
|
||||
first = commit(repo, "session: session1")
|
||||
(repo / "memory/note.md").write_text("new fact\n")
|
||||
(repo / "removed.txt").unlink()
|
||||
(repo / "added.bin").write_bytes(b"\x00\xff")
|
||||
second = commit(repo, "session: session2")
|
||||
return home, run, repo, first, second
|
||||
|
||||
|
||||
def commit(repo, subject):
|
||||
git(repo, "add", "--all")
|
||||
git(repo, "commit", "-m", subject)
|
||||
return git(repo, "rev-parse", "HEAD")
|
||||
|
||||
|
||||
def history(run, **kwargs):
|
||||
return memory.MemoryHistoryInput(validation_path=str(run), case_id="case1", **kwargs)
|
||||
|
||||
|
||||
def comparison(run, **kwargs):
|
||||
return memory.CompareMemoryInput(validation_path=str(run), case_id="case1", **kwargs)
|
||||
|
||||
|
||||
def test_history_paging_filter_and_no_writes(evidence):
|
||||
home, run, repo, first, second = evidence
|
||||
(repo / "memory/note.md").write_text("dirty checkout ignored\n")
|
||||
before = snapshot(home)
|
||||
result = memory.inspect_history(home, history(run, limit=1))
|
||||
assert result["head"] == second and result["row_count"] == 2
|
||||
assert result["rows"][0]["session_id"] == "session2" and result["next_offset"] == 1
|
||||
result = memory.inspect_history(home, history(run, offset=1, limit=1))
|
||||
assert result["rows"][0]["commit_sha"] == first and result["next_offset"] is None
|
||||
assert memory.inspect_history(home, history(run, path="added.bin"))["row_count"] == 1
|
||||
assert memory.inspect_history(home, history(run, path="memory/*"))["rows"] == []
|
||||
assert snapshot(home) == before
|
||||
|
||||
|
||||
def test_same_repository_diff_and_unique_prefix(evidence):
|
||||
home, run, repo, first, second = evidence
|
||||
(repo / "memory/note.md").write_text("dirty checkout ignored\n")
|
||||
before = snapshot(home)
|
||||
result = memory.compare_memory(home, comparison(run, base_ref=first[:7].upper()))
|
||||
assert result["base_commit"] == first and result["target_commit"] == second
|
||||
assert result["added"] == ["added.bin"] and result["removed"] == ["removed.txt"]
|
||||
assert result["modified"] == ["memory/note.md"]
|
||||
assert "-old fact" in result["diff"] and "+new fact" in result["diff"]
|
||||
assert "Binary files differ" in result["diff"] and not result["truncated"]
|
||||
assert "dirty checkout" not in result["diff"]
|
||||
assert memory.compare_memory(home, comparison(run, base_ref="HEAD~1", path="memory"))["added"] == []
|
||||
assert snapshot(home) == before
|
||||
|
||||
|
||||
def test_cross_evaluation_final_snapshots(evidence):
|
||||
home, run, _repo, _first, second = evidence
|
||||
other = write_run(home, "2", case_ids=["case1"], commit="b" * 40)
|
||||
target = other / "cases/case1/memory_construction/reme_workspace"
|
||||
target.mkdir(parents=True)
|
||||
git(target, "init")
|
||||
git(target, "config", "user.name", "test")
|
||||
git(target, "config", "user.email", "test@localhost")
|
||||
(target / "target.txt").write_text("target\n")
|
||||
target_head = commit(target, "new history")
|
||||
before = snapshot(home)
|
||||
result = memory.compare_memory(home, comparison(run, target_validation_path=str(other)))
|
||||
assert result["base_commit"] == second and result["target_commit"] == target_head
|
||||
assert result["added"] == ["target.txt"]
|
||||
assert result["removed"] == ["added.bin", "memory/note.md"]
|
||||
assert result["base_tree_sha256"] != result["target_tree_sha256"]
|
||||
assert snapshot(home) == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["workspace_missing", "history_missing", "history_empty"])
|
||||
def test_missing_and_empty_history(tmp_path, state):
|
||||
home = tmp_path / "home"
|
||||
run = write_run(home, case_ids=["case1"])
|
||||
repo = run / "cases/case1/memory_construction/reme_workspace"
|
||||
if state != "workspace_missing":
|
||||
repo.mkdir(parents=True)
|
||||
if state == "history_empty":
|
||||
git(repo, "init")
|
||||
result = memory.inspect_history(home, history(run))
|
||||
assert result["status"] == state and result["rows"] == [] and not result["has_git_history"]
|
||||
with pytest.raises(ValueError, match="requires memory commits"):
|
||||
memory.compare_memory(home, comparison(run, base_ref="HEAD"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[
|
||||
{"offset": -1},
|
||||
{"limit": 101},
|
||||
{"limit": True},
|
||||
{"case_id": "../bad"},
|
||||
{"path": "../escape"},
|
||||
{"path": ".git/config"},
|
||||
],
|
||||
)
|
||||
def test_input_validation(kwargs):
|
||||
with pytest.raises(ValidationError):
|
||||
memory.MemoryHistoryInput.model_validate({"validation_path": "a", "case_id": "case1", **kwargs})
|
||||
|
||||
|
||||
def test_comparison_requires_base():
|
||||
with pytest.raises(ValidationError, match="base_ref is required"):
|
||||
comparison("a")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["gitfile", "symlink", "alternates", "corrupt"])
|
||||
def test_invalid_or_external_git_storage(evidence, tmp_path, kind):
|
||||
home, run, repo, _first, _second = evidence
|
||||
if kind == "gitfile":
|
||||
(repo / ".git").rename(repo / "saved-git")
|
||||
(repo / ".git").write_text("gitdir: saved-git\n")
|
||||
elif kind == "symlink":
|
||||
(repo / ".git/objects/external").symlink_to(tmp_path)
|
||||
elif kind == "alternates":
|
||||
(repo / ".git/objects/info/alternates").write_text(str(tmp_path))
|
||||
else:
|
||||
(repo / ".git/HEAD").write_text("broken\n")
|
||||
with pytest.raises((ValueError, memory.subprocess.CalledProcessError)):
|
||||
memory.inspect_history(home, history(run))
|
||||
|
||||
|
||||
def test_scope_and_limits(evidence, monkeypatch):
|
||||
home, run, _repo, first, _second = evidence
|
||||
with pytest.raises(ValueError, match="case_id"):
|
||||
memory.inspect_history(home, memory.MemoryHistoryInput(validation_path=str(run), case_id="other"))
|
||||
with pytest.raises(ValueError):
|
||||
memory.inspect_history(home, history(home))
|
||||
monkeypatch.setattr(memory, "MAX_OUTPUT", 20)
|
||||
result = memory.compare_memory(home, comparison(run, base_ref=first))
|
||||
assert result["truncated"] and len(result["diff"]) <= 20
|
||||
monkeypatch.setattr(memory, "MAX_TREE_FILES", 1)
|
||||
with pytest.raises(ValueError, match="2000 files"):
|
||||
memory.compare_memory(home, comparison(run, base_ref=first))
|
||||
|
||||
|
||||
def test_ambiguous_and_unreachable_revisions(evidence):
|
||||
home, run, repo, first, second = evidence
|
||||
# Publish real commits until two reachable SHAs share a one-character prefix.
|
||||
previous = second
|
||||
seen = {first[0]: first, second[0]: second}
|
||||
for index in range(100):
|
||||
sha = git(repo, "commit-tree", "HEAD^{tree}", "-p", previous, "-m", str(index))
|
||||
previous = sha
|
||||
if sha[0] in seen:
|
||||
prefix = sha[0]
|
||||
break
|
||||
seen[sha[0]] = sha
|
||||
git(repo, "update-ref", "HEAD", previous)
|
||||
with pytest.raises(ValueError, match="matches reachable"):
|
||||
memory.compare_memory(home, comparison(run, base_ref=prefix))
|
||||
with pytest.raises(ValueError, match="0 matches"):
|
||||
memory.compare_memory(home, comparison(run, base_ref="f" * 40))
|
||||
detached = git(repo, "commit-tree", "HEAD^{tree}", "-m", "unreachable")
|
||||
with pytest.raises(memory.subprocess.CalledProcessError):
|
||||
memory.compare_memory(home, comparison(run, base_ref=detached + "~0"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registered_jobs_and_restricted_scope(app, prepared, evidence):
|
||||
run = write_run(prepared, case_ids=["case1"])
|
||||
result = await call(app, "home_inspect_memory_history", validation_path=str(run), case_id="case1")
|
||||
assert result["status"] == "workspace_missing"
|
||||
response = await app.run_job("home_compare_memory", validation_path=str(run), case_id="case1")
|
||||
assert not response.success and "base_ref" in str(response.answer)
|
||||
response = await app.run_job(
|
||||
"home_inspect_memory_history",
|
||||
validation_path=str(run),
|
||||
case_id="case1",
|
||||
_allowed_paths=[str(run)],
|
||||
)
|
||||
assert not response.success and "unrestricted" in str(response.answer)
|
||||
|
||||
_home, _run, repository, first, second = evidence
|
||||
destination = run / "cases/case1/memory_construction/reme_workspace"
|
||||
shutil.copytree(repository, destination)
|
||||
result = await call(app, "home_compare_memory", validation_path=str(run), case_id="case1", base_ref=first)
|
||||
assert result["target_commit"] == second and result["modified"] == ["memory/note.md"]
|
||||
|
||||
|
||||
def test_unterminated_lines_and_snapshot_byte_limit(evidence, monkeypatch):
|
||||
home, run, repo, _first, second = evidence
|
||||
(repo / "memory/note.md").write_text("unterminated")
|
||||
commit(repo, "ordinary checkpoint")
|
||||
result = memory.compare_memory(home, comparison(run, base_ref=second, path="memory"))
|
||||
assert "+unterminated\n\\ No newline at end of file\n" in result["diff"]
|
||||
assert memory.inspect_history(home, history(run))["rows"][0]["session_id"] is None
|
||||
monkeypatch.setattr(memory, "MAX_TREE_BYTES", 1)
|
||||
with pytest.raises(ValueError, match="20 MB"):
|
||||
memory.compare_memory(home, comparison(run, base_ref=second))
|
||||
|
||||
|
||||
def test_tracked_symlinks_are_not_followed(evidence):
|
||||
home, run, repo, _first, second = evidence
|
||||
(repo / "memory/link").symlink_to("/etc/passwd")
|
||||
commit(repo, "link")
|
||||
with pytest.raises(ValueError, match="regular files"):
|
||||
memory.compare_memory(home, comparison(run, base_ref=second))
|
||||
412
plugins/home/tests/test_sampling.py
Normal file
412
plugins/home/tests/test_sampling.py
Normal file
|
|
@ -0,0 +1,412 @@
|
|||
"""Adaptive sampling parity and actual-start accounting through real Home Jobs."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from reme_home import sampling
|
||||
from reme_home.evaluation import history as history_module
|
||||
from reme_home.preparation.data.models import CaseSpec, QuerySpec
|
||||
from reme_home.workspace import _initialization_lock
|
||||
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
||||
FakeFactory,
|
||||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
)
|
||||
|
||||
|
||||
def cases():
|
||||
return [
|
||||
CaseSpec(
|
||||
case_id=key,
|
||||
sessions=[],
|
||||
queries=[
|
||||
QuerySpec(query_id=f"query-{key}-{index}", question="?", golden_answer="answer") for index in range(4)
|
||||
],
|
||||
)
|
||||
for key in "abc"
|
||||
]
|
||||
|
||||
|
||||
def test_plan_matches_dev_meta_golden_draw(monkeypatch):
|
||||
# Golden output computed with dev_meta b53048e4 sampling.py, seed=123456.
|
||||
monkeypatch.setattr(sampling.time, "time_ns", lambda: 123456)
|
||||
state = {
|
||||
"round": 2,
|
||||
"cases": {
|
||||
"a": {
|
||||
"observed_count": 2,
|
||||
"ewma_loss": 0.9,
|
||||
"last_selected_round": 2,
|
||||
"last_error_round": 1,
|
||||
},
|
||||
},
|
||||
"queries": {},
|
||||
}
|
||||
before = copy.deepcopy(state)
|
||||
policy = sampling.SamplingPolicy(max_cases=2, max_queries=3, temperature=0.8)
|
||||
plan = sampling.create_sampling_plan(cases(), state, policy)
|
||||
assert plan["case_ids"] == ["a", "b"]
|
||||
assert plan["query_ids"] == {"a": ["query-a-2", "query-a-0"], "b": ["query-b-3"]}
|
||||
assert plan["case_draw_weights"] == pytest.approx(
|
||||
{
|
||||
"a": 0.40033106216673214,
|
||||
"b": 0.29983446891663396,
|
||||
"c": 0.29983446891663396,
|
||||
},
|
||||
)
|
||||
assert plan["query_draw_weights"]["a"] == {f"query-a-{index}": 0.25 for index in range(4)}
|
||||
assert state == before
|
||||
assert sampling.create_sampling_plan(cases(), state, policy) == plan
|
||||
|
||||
|
||||
def test_replay_eligibility_and_unrestricted_next_draw(monkeypatch):
|
||||
monkeypatch.setattr(sampling.time, "time_ns", lambda: 100)
|
||||
policy = sampling.SamplingPolicy(max_cases=3, max_queries=3)
|
||||
eligible = {"b": ["query-b-1", "query-b-3"], "c": ["query-c-2"]}
|
||||
plan = sampling.create_sampling_plan(cases(), {}, policy, eligible_query_ids=eligible)
|
||||
assert set(plan["case_ids"]) == {"b", "c"}
|
||||
assert all(set(values) <= set(eligible[key]) for key, values in plan["query_ids"].items())
|
||||
full = sampling.create_sampling_plan(cases(), {}, policy)
|
||||
assert set(full["case_draw_weights"]) == {"a", "b", "c"}
|
||||
assert full["eligibility"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options",
|
||||
[
|
||||
{"max_cases": 0},
|
||||
{"max_queries": 0},
|
||||
{"max_cases": True},
|
||||
{"max_queries": True},
|
||||
{"temperature": 0},
|
||||
{"temperature": float("nan")},
|
||||
{"temperature": float("inf")},
|
||||
{"ewma_alpha": 0},
|
||||
{"ewma_alpha": 1.1},
|
||||
{"uniform_exploration": -0.1},
|
||||
{"uniform_exploration": 1.1},
|
||||
{"unexpected": 1},
|
||||
],
|
||||
)
|
||||
def test_policy_rejects_invalid_limits(options):
|
||||
with pytest.raises(ValueError):
|
||||
sampling.SamplingPolicy(**options)
|
||||
|
||||
|
||||
def test_uniform_mass_and_unobserved_priority():
|
||||
records = {
|
||||
"easy": {"ewma_loss": 0.0, "observed_count": 100, "last_selected_round": 10},
|
||||
"hard": {"ewma_loss": 1.0, "observed_count": 1, "last_selected_round": 1, "last_error_round": 1},
|
||||
"unseen": {},
|
||||
}
|
||||
weights = sampling._mixed_weights(records, 11, sampling.SamplingPolicy(uniform_exploration=0.3))
|
||||
assert sum(weights.values()) == pytest.approx(1)
|
||||
assert all(weight >= 0.1 for weight in weights.values())
|
||||
assert weights["hard"] > weights["easy"] and weights["unseen"] > weights["easy"]
|
||||
uniform = sampling._mixed_weights(records, 11, sampling.SamplingPolicy(uniform_exploration=1))
|
||||
assert set(uniform.values()) == {1 / 3}
|
||||
|
||||
|
||||
def state(home):
|
||||
return history_module.read_state(home)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_baseline_then_adaptive_job_updates_only_selected_cases(app, prepared):
|
||||
baseline = await app.run_job("home_evaluate")
|
||||
assert baseline.success, baseline.answer
|
||||
assert state(prepared)["round"] == 1
|
||||
assert all(record["ewma_loss"] == 0.5 for record in state(prepared)["cases"].values())
|
||||
evaluation = app.context.components["home.evaluation"]["default"]
|
||||
evaluation.sampling_policy = sampling.SamplingPolicy(max_cases=1, max_queries=1, ewma_alpha=0.5)
|
||||
app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": '{"score": 1}'}
|
||||
before = copy.deepcopy(state(prepared))
|
||||
response = await app.run_job("home_evaluate", sampling=True)
|
||||
assert response.success, response.answer
|
||||
run = Path(response.answer["result_dir"])
|
||||
manifest = json.loads((run / "manifest.json").read_text())
|
||||
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
|
||||
selected = manifest["case_ids"][0]
|
||||
unselected = ({"1", "2"} - {selected}).pop()
|
||||
assert manifest["case_ids"] == plan["case_ids"] and manifest["query_ids"] == plan["query_ids"]
|
||||
assert plan["mode"] == "sampled" and plan["round"] == 2
|
||||
assert response.answer["case_count"] == response.answer["scored_query_count"] == 1
|
||||
after = state(prepared)
|
||||
assert after["cases"][selected]["ewma_loss"] == 0.25
|
||||
assert after["cases"][selected]["observed_count"] == 2
|
||||
assert after["cases"][selected]["successful_observations_after_last_error"] == 1
|
||||
assert after["cases"][unselected] == before["cases"][unselected]
|
||||
evaluation._finish_sampling(run)
|
||||
assert state(prepared) == after # A summary is consumed once, including after reopening.
|
||||
assert "home_validate" not in app.context.jobs
|
||||
assert app.context.jobs["home_evaluate"].parameters["properties"]["sampling"]["type"] == "boolean"
|
||||
assert "sampling_policy" not in app.context.jobs["home_evaluate"].parameters["properties"]
|
||||
assert "case_ids" not in app.context.jobs["home_evaluate"].parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_debug_never_reads_or_changes_sampling_state(app, prepared):
|
||||
before = prepared / "sampling/state.json"
|
||||
for enabled in (False, True):
|
||||
response = await app.run_job("home_evaluate", sampling=enabled, debug=True)
|
||||
assert response.success, response.answer
|
||||
assert not before.exists()
|
||||
assert not list((prepared / "sampling/plans").rglob("*.json"))
|
||||
# Corrupt sampling state cannot prevent an unrelated debug smoke test.
|
||||
before.write_text("corrupt cached state")
|
||||
response = await app.run_job("home_evaluate", sampling=True, debug=True)
|
||||
assert response.success and before.read_text() == "corrupt cached state"
|
||||
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
|
||||
assert "sandbox_started_at" not in manifest and "sampling" not in manifest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["dirty", "factory_constructor", "startup", "selection", "replay"])
|
||||
async def test_prestart_failure_never_counts(app, prepared, monkeypatch, failure):
|
||||
await app.run_job("home_evaluate")
|
||||
previous = (prepared / "sampling/state.json").read_bytes()
|
||||
plans = sorted((prepared / "sampling/plans").rglob("*.json"))
|
||||
options = {}
|
||||
if failure == "dirty":
|
||||
(prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py").write_text("dirty")
|
||||
elif failure == "selection":
|
||||
options["case_ids"] = ["1"]
|
||||
elif failure == "replay":
|
||||
options["reuse_construction_from"] = "evaluations/not-a-run"
|
||||
elif failure == "factory_constructor":
|
||||
|
||||
def fail(*args, **kwargs):
|
||||
raise RuntimeError("factory never created")
|
||||
|
||||
monkeypatch.setattr("reme_home.evaluation.component.DockerReMeSandboxFactory", fail)
|
||||
else:
|
||||
|
||||
async def fail(*args, **kwargs):
|
||||
raise RuntimeError("no sandbox initialized")
|
||||
|
||||
monkeypatch.setattr(FakeFactory, "create_cases", fail)
|
||||
response = await app.run_job("home_evaluate", sampling=True, **options)
|
||||
assert not response.success
|
||||
assert (prepared / "sampling/state.json").read_bytes() == previous
|
||||
assert sorted((prepared / "sampling/plans").rglob("*.json")) == plans
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_started_infrastructure_failure_is_not_a_quality_zero(app, prepared, monkeypatch):
|
||||
async def fail(*args, **kwargs):
|
||||
raise ConnectionError("construction provider unavailable")
|
||||
|
||||
monkeypatch.setattr("test_evaluation.FakeWorker.run_build", fail)
|
||||
response = await app.run_job("home_evaluate", case_ids=["1"])
|
||||
assert not response.success
|
||||
observed = state(prepared)
|
||||
assert observed["round"] == 1
|
||||
assert not observed["cases"] and not observed["queries"]
|
||||
assert app.context.components["home.evaluation_history"]["default"].list_results() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_started_candidate_failure_does_not_fabricate_a_score(app, prepared, monkeypatch):
|
||||
factory = FakeFactory.create_cases
|
||||
|
||||
async def create(self, *args, **kwargs):
|
||||
workers = await factory(self, *args, **kwargs)
|
||||
|
||||
async def empty(_queries):
|
||||
return {"success": True, "queries": [{"query_id": "fact:1", "answer": "", "error": None}]}
|
||||
|
||||
for worker in workers:
|
||||
worker.run_queries = empty
|
||||
return workers
|
||||
|
||||
monkeypatch.setattr(FakeFactory, "create_cases", create)
|
||||
response = await app.run_job("home_evaluate", case_ids=["1"])
|
||||
assert response.success, response.answer
|
||||
observed = state(prepared)
|
||||
assert observed["round"] == 1
|
||||
record = observed["cases"]["1"]
|
||||
assert record["candidate_failure_count"] == 1
|
||||
assert record["observed_count"] == 0 and record["ewma_loss"] is None
|
||||
query = observed["queries"][sampling._query_key("1", "fact:1")]
|
||||
assert query["candidate_failure_count"] == 1 and query["observed_count"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_started_cancellation_retains_round_but_no_partial_quality(app, prepared, monkeypatch):
|
||||
monkeypatch.setattr(FakeFactory, "block", asyncio.Event())
|
||||
task = asyncio.create_task(app.run_job("home_evaluate", sampling=True, concurrency=1))
|
||||
for _ in range(200):
|
||||
if FakeFactory.instances and FakeFactory.instances[0].entered.is_set():
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
assert FakeFactory.instances[0].entered.is_set()
|
||||
assert state(prepared)["round"] == 1
|
||||
concurrent = await app.run_job("home_evaluate", sampling=True)
|
||||
assert not concurrent.success # Formal validations follow dev_meta's serialization boundary.
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
observed = state(prepared)
|
||||
assert observed["round"] == 1 and not observed["cases"] and not observed["queries"]
|
||||
assert not observed["processed_validations"]
|
||||
with _initialization_lock(prepared / "metadata/validation.lock"):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replay_sampling_is_limited_to_donor_cases(app, prepared):
|
||||
baseline = await app.run_job("home_evaluate", case_ids=["1"])
|
||||
assert baseline.success, baseline.answer
|
||||
app.context.components["home.evaluation"]["default"].sampling_policy = sampling.SamplingPolicy(max_cases=1)
|
||||
response = await app.run_job("home_evaluate", sampling=True, reuse_construction_from=baseline.answer["result_dir"])
|
||||
assert response.success, response.answer
|
||||
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
|
||||
assert manifest["case_ids"] == ["1"]
|
||||
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
|
||||
assert plan["eligibility"] == {"1": ["fact:1"]}
|
||||
# The next full rebuild must see the entire training set again.
|
||||
response = await app.run_job("home_evaluate", sampling=True)
|
||||
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
|
||||
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
|
||||
assert set(plan["case_draw_weights"]) == {"1", "2"} and plan["eligibility"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sampling_cache_rebuild_and_new_round_after_reopen(app, prepared):
|
||||
baseline = await app.run_job("home_evaluate")
|
||||
assert baseline.success
|
||||
expected = state(prepared)
|
||||
(prepared / "sampling/state.json").unlink()
|
||||
assert state(prepared) == expected
|
||||
assert not (prepared / "sampling/state.json").exists() # Rebuilding reads never claim a new round.
|
||||
with _initialization_lock(prepared / "metadata/workspace.lock"):
|
||||
assert history_module.rebuild_sampling_state(prepared) == expected
|
||||
result = await app.run_job("home_evaluate", sampling=True)
|
||||
assert result.success, result.answer
|
||||
assert state(prepared)["round"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fail_fast_started_round_without_summary_is_not_quality_feedback(app, prepared):
|
||||
app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": "invalid judge output"}
|
||||
response = await app.run_job("home_evaluate", sampling=True, fail_fast=True)
|
||||
assert not response.success
|
||||
observed = state(prepared)
|
||||
assert observed["round"] == 1
|
||||
assert not observed["processed_validations"] and not observed["cases"]
|
||||
assert not observed["queries"]
|
||||
(prepared / "sampling/state.json").unlink()
|
||||
assert state(prepared) == observed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_pool_start_failure_counts_started_round(app, prepared, monkeypatch):
|
||||
original = FakeFactory.create_cases
|
||||
|
||||
async def fail_after_first_start(factory, *args, **kwargs):
|
||||
await original(factory, *args, **kwargs)
|
||||
raise ConnectionError("a later worker failed after first initialized")
|
||||
|
||||
monkeypatch.setattr(FakeFactory, "create_cases", fail_after_first_start)
|
||||
result = await app.run_job("home_evaluate", sampling=True)
|
||||
assert not result.success
|
||||
observed = state(prepared)
|
||||
assert observed["round"] == 1 and not observed["processed_validations"]
|
||||
assert not observed["cases"] and not observed["queries"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_started_callback_is_idempotent(app, prepared):
|
||||
response = await app.run_job("home_evaluate", sampling=True)
|
||||
assert response.success
|
||||
run = Path(response.answer["result_dir"])
|
||||
manifest = json.loads((run / "manifest.json").read_text())
|
||||
original = state(prepared)
|
||||
component = app.context.components["home.evaluation"]["default"]
|
||||
component._sampling_started(run, manifest, None)
|
||||
assert state(prepared) == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing", [1, 2])
|
||||
def test_replay_eligibility_uses_dev_meta_five_percent_threshold(tmp_path, missing):
|
||||
from reme_home.evaluation.inputs import EvaluationSpec, replay_sampling_eligibility
|
||||
from reme_home.evaluation.artifacts import write_json
|
||||
|
||||
home = tmp_path / "home"
|
||||
source = home / "evaluations/init/commit/1"
|
||||
selected = [
|
||||
CaseSpec(
|
||||
case_id=f"case-{index}",
|
||||
sessions=[],
|
||||
queries=[
|
||||
QuerySpec(query_id="q", question="?", golden_answer="answer"),
|
||||
],
|
||||
)
|
||||
for index in range(20)
|
||||
]
|
||||
write_json(
|
||||
source / "manifest.json",
|
||||
{
|
||||
"dataset_kind": "training-set",
|
||||
"mode": "scored",
|
||||
"status": "completed",
|
||||
"case_ids": [case.case_id for case in selected],
|
||||
},
|
||||
)
|
||||
write_json(source / "summary.json", {})
|
||||
for case in selected[missing:]:
|
||||
folder = source / "cases" / case.case_id
|
||||
write_json(folder / "memory_construction/result.json", {"success": True})
|
||||
(folder / "memory_construction/reme_workspace.tar.gz").write_bytes(b"fixture")
|
||||
write_json(folder / "queries/result.json", {"queries": [{"answer": "answer", "error": None}]})
|
||||
spec = EvaluationSpec(reuse_construction_from=str(source))
|
||||
if missing == 1:
|
||||
eligible = replay_sampling_eligibility(home, spec, selected)
|
||||
assert len(eligible) == 19 and "case-0" not in eligible
|
||||
else:
|
||||
with pytest.raises(ValueError, match="5%"):
|
||||
replay_sampling_eligibility(home, spec, selected)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_budget_and_draw_order_reach_actual_execution(app, inputs, monkeypatch):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
source = Path(inputs[0]["benchmark"]["source"])
|
||||
for path in source.glob("chats/*/*/probing_questions/probing_questions.json"):
|
||||
payload = json.loads(path.read_text())
|
||||
payload["fact"] = [
|
||||
dict(payload["fact"][0], question=f"question {index}", rubric=["says hello"]) for index in range(4)
|
||||
]
|
||||
path.write_text(json.dumps(payload))
|
||||
prepared = await app.run_job("home_prepare", spec=inputs[0])
|
||||
assert prepared.success
|
||||
monkeypatch.setattr("reme_home.evaluation.component.DockerReMeSandboxFactory", FakeFactory)
|
||||
monkeypatch.setattr(
|
||||
app.context.components["agent_wrapper"]["judge"],
|
||||
"reply",
|
||||
AsyncMock(return_value={"result": '{"score": 0.5}'}),
|
||||
)
|
||||
monkeypatch.setattr(sampling.time, "time_ns", lambda: 123456)
|
||||
component = app.context.components["home.evaluation"]["default"]
|
||||
component.sampling_policy = sampling.SamplingPolicy(max_cases=2, max_queries=3)
|
||||
response = await app.run_job("home_evaluate", sampling=True, concurrency=1)
|
||||
assert response.success, response.answer
|
||||
run = Path(response.answer["result_dir"])
|
||||
manifest = json.loads((run / "manifest.json").read_text())
|
||||
home = Path(app.config.workspace_dir) / "home"
|
||||
plan = json.loads((home / manifest["sampling"]["plan_path"]).read_text())
|
||||
assert manifest["query_ids"] == plan["query_ids"]
|
||||
assert len(manifest["case_ids"]) == 2 and sum(map(len, manifest["query_ids"].values())) == 3
|
||||
worker = FakeFactory.instances[-1].workers[0]
|
||||
assert [query.query_id for query in worker.queries] == manifest["query_ids"][worker.case_id]
|
||||
for case_id, query_ids in manifest["query_ids"].items():
|
||||
result = json.loads((run / "cases" / case_id / "queries/result.json").read_text())
|
||||
assert [row["query_id"] for row in result["queries"]] == query_ids
|
||||
assert response.answer["scored_query_count"] == 3
|
||||
|
|
@ -299,6 +299,11 @@ class AsAgentWrapper(BaseAgentWrapper):
|
|||
"""Resolve configured skill names to AgentScope local skill directories."""
|
||||
return [str(path) for path in self._resolve_project_skills(skills).values()]
|
||||
|
||||
def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> Agent:
|
||||
"""Construct the SDK agent while subclasses reuse ReMe's tool and session wiring."""
|
||||
del options
|
||||
return Agent(**kwargs)
|
||||
|
||||
async def _build_agent(self, inputs: Any, **kwargs) -> tuple[Agent, Any]:
|
||||
"""Build an Agent instance from kwargs. Returns (agent, processed_inputs)."""
|
||||
model = self.as_llm.model if self.as_llm else None
|
||||
|
|
@ -327,7 +332,8 @@ class AsAgentWrapper(BaseAgentWrapper):
|
|||
perm_mode = PermissionMode(kwargs.get("permission_mode", "bypass"))
|
||||
state = await self._load_state(kwargs, perm_mode)
|
||||
|
||||
agent = Agent(
|
||||
agent = self._create_agent(
|
||||
options=kwargs,
|
||||
name=self.name,
|
||||
system_prompt=system_prompt,
|
||||
model=model,
|
||||
|
|
|
|||
84
tests/unit/test_benchmark_docker_start_callback.py
Normal file
84
tests/unit/test_benchmark_docker_start_callback.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""Observe the first initialized sandbox without counting failed startup attempts."""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from deploy.benchmark_docker import DockerReMeSandboxFactory, ImageCandidate
|
||||
from deploy.benchmark_docker import docker_sandbox
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_callback_only_once_after_initialization(monkeypatch):
|
||||
"""Initialization is the boundary, including replacement workers and partial pool failure."""
|
||||
initialized, reported = [], []
|
||||
|
||||
async def initialize(case):
|
||||
await asyncio.sleep(0)
|
||||
initialized.append(case.case_id)
|
||||
if case.case_id == "bad":
|
||||
raise RuntimeError("startup failed")
|
||||
|
||||
monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", initialize)
|
||||
workspace = AsyncMock()
|
||||
factory = DockerReMeSandboxFactory(
|
||||
ImageCandidate("fixture"),
|
||||
workspace_builder=lambda *_: workspace,
|
||||
on_sandbox_started=lambda: reported.append(list(initialized)),
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="startup failed"):
|
||||
await factory.create_case("bad")
|
||||
assert not reported
|
||||
await factory.create_case("good")
|
||||
assert reported == [["bad", "good"]]
|
||||
await factory.create_case("replacement")
|
||||
assert len(reported) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_callback_failure_closes_initialized_resource(monkeypatch):
|
||||
"""Persisting a start marker must not orphan an already-created workspace on failure."""
|
||||
monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", AsyncMock())
|
||||
workspace = AsyncMock()
|
||||
|
||||
def fail():
|
||||
raise OSError("start marker unavailable")
|
||||
|
||||
factory = DockerReMeSandboxFactory(
|
||||
ImageCandidate("fixture"),
|
||||
workspace_builder=lambda *_: workspace,
|
||||
on_sandbox_started=fail,
|
||||
)
|
||||
with pytest.raises(OSError, match="start marker unavailable"):
|
||||
await factory.create_case("created")
|
||||
workspace.close.assert_awaited_once()
|
||||
assert not factory._sandbox_started_reported # pylint: disable=protected-access
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_pool_start_reports_first_success_before_failure(monkeypatch):
|
||||
"""An initialized worker counts even if a subsequent worker prevents pool startup."""
|
||||
workspaces, started = [], []
|
||||
|
||||
def workspace_builder(*_args):
|
||||
workspace = AsyncMock()
|
||||
workspaces.append(workspace)
|
||||
return workspace
|
||||
|
||||
async def initialize(case):
|
||||
if case.case_id == "bad":
|
||||
raise ConnectionError("second worker failed")
|
||||
|
||||
monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", initialize)
|
||||
factory = DockerReMeSandboxFactory(
|
||||
ImageCandidate("fixture"),
|
||||
workspace_builder=workspace_builder,
|
||||
on_sandbox_started=lambda: started.append(True),
|
||||
)
|
||||
with pytest.raises(ConnectionError, match="second worker failed"):
|
||||
await factory.create_cases(["good", "bad"], concurrency=1)
|
||||
assert started == [True]
|
||||
assert len(workspaces) == 2
|
||||
for workspace in workspaces:
|
||||
workspace.close.assert_awaited_once()
|
||||
Loading…
Add table
Reference in a new issue