refactor(home): extract shared review_report workflow helper and bump version to 0.4.2.2

This commit is contained in:
酱牛肉 2026-11-02 19:22:20 +08:00
parent 75ca3e2011
commit e89930f9c2
19 changed files with 476 additions and 451 deletions

View file

@ -96,8 +96,8 @@ jobs:
- name: Reject incompatible published ReMe for Home
run: |
HOME_WHEEL="$(pwd)/$(ls dist/home/reme_home-[0-9]*.whl)"
if python -m pip install --dry-run --ignore-installed "${HOME_WHEEL}" "reme-ai[as]==0.4.2.0" > "${RUNNER_TEMP}/home-incompatible.log" 2>&1; then
echo 'Home must not install with the published ReMe 0.4.2.0 wheel.'
if python -m pip install --dry-run --ignore-installed "${HOME_WHEEL}" "reme-ai[as]==0.4.2.1" > "${RUNNER_TEMP}/home-incompatible.log" 2>&1; then
echo 'Home must not install with the published ReMe 0.4.2.1 wheel.'
exit 1
fi
cat "${RUNNER_TEMP}/home-incompatible.log"

View file

@ -17,6 +17,19 @@ from typing import Any
from ._cleanup import finish_cleanup, format_error
from .docker_sandbox import DockerReMeSandbox, DockerReMeSandboxFactory
from .models import EvaluationQuery, JobRequest
from .worker import _exception_details
def _exception_classification(exc: BaseException) -> dict[str, Any]:
"""Retain only classification fields, omitting absent status codes and causes."""
details = _exception_details(exc)
result = {key: details[key] for key in ("type", "module")}
status = details.get("status_code")
if isinstance(status, int) and not isinstance(status, bool):
result["status_code"] = status
if details["causes"]:
result["causes"] = details["causes"]
return result
@dataclass(frozen=True, slots=True)
@ -144,7 +157,7 @@ class BenchmarkDockerPool:
except Exception as exc:
result.error = format_error(exc)
result.failure_stage = operation
result.exception = {"type": type(exc).__name__, "module": type(exc).__module__}
result.exception = _exception_classification(exc)
await self._retain_failed_build_log(worker, plan, result)
return result
accepted = accept_build(result.build) if accept_build else result.build.get("success")
@ -175,7 +188,7 @@ class BenchmarkDockerPool:
except Exception as exc: # Keep other cases running and release this worker.
result.error = format_error(exc)
result.failure_stage = operation
result.exception = {"type": type(exc).__name__, "module": type(exc).__module__}
result.exception = _exception_classification(exc)
return result
finally:
if result.error and plan.export_diagnostics_to is not None and worker.reusable:

View file

@ -144,15 +144,10 @@ All prepared outputs live under `workspace_dir/home/` (paths below are relative
manifest with case IDs and content fingerprints.
- `datasets/debug-cases/`: deterministic small smoke cases with source case IDs
in their metadata, or an empty `cases/` directory when debug derivation is disabled.
- `code/runtime/`: fixed committed ReMe sources, packaging files and configurations,
without plugin projects. Uncommitted source edits are excluded.
- `code/repo/`: an independent Git repository containing complete selected plugin
projects (including packaging and configuration) and any selected ReMe core paths,
retaining repository-relative locations. `candidate.paths` defines the editable
scope, not the complete file inventory. The initial commit preserves the baseline.
Future runners assemble an isolated evaluation tree from runtime plus repo,
excluding `.git`; `install_projects` paths are relative to that tree. Install
plugins from the candidate, and never edit the fixed runtime.
- `code/runtime/` and `code/repo/`: fixed committed runtime and candidate Git
repository. See the [workspace and modification boundary](src/reme_home/runtime_docs/architecture-guide.md#workspace-and-modification-boundary)
for their roles and editable scope. Source exports exclude uncommitted edits;
`install_projects` paths are relative to the assembled evaluation tree.
- `docs/architecture-guide.md` and `docs/validation-artifact-guide.md`: guides
shipped with the installed Home plugin, covering candidate development and
investigation of evaluation artifacts.
@ -236,60 +231,22 @@ For explicit `reuse_construction_from`, the caller must establish construction
equivalence. Automatic sampled/budgeted selection instead invokes the host's
equivalence reviewer and 60% replay policy. Runs record both commits.
Results use `home/evaluations/<branch>/<commit-prefix>/<ascending-run-id>/`:
```text
manifest.json
summary.json
cases/<case-id>/
memory_construction/
result.json
build.log
reme_workspace.tar.gz
reme_workspace/
queries/
result.json
<query-id>/answer.log
```
Structured scores in query results and summaries use at most four decimal places.
Raw judge text and reference answers are preserved.
The memory tar is exported **before** queries, retains session Git checkpoints,
and is independent of the query execution state. Replayed runs keep an
independent donor tar and inherited build log without charging construction
tokens again. Answers are persisted before host judging. Per-query results keep
reference/actual answers, scores, compact judge output, reported answer token usage
and failures. `queries/result.json` is the sole authoritative answer record;
reconstructable prompts and repeated payloads are removed using the reviewed
`dev_meta` rules. Construction results similarly compact redundant links, Job
counts, token records and cumulative daily-index snapshots. Summary
scores use the expected-query denominator; infrastructure/judge/artifact errors
make the comparable mean unavailable while retaining the observed-score mean.
The Job returns `success=false` for infrastructure/judge/artifact failures, while
preserving its structured answer and result directory; candidate failures remain
ordinary evaluation observations.
Base jobs preserve structured exception types, HTTP status codes and causes in
`Response.metadata["reme.job_error"]`; the Docker worker carries them into failure
artifacts. Network/provider failures anywhere in the cause chain invalidate the
comparison. Failed jobs without structured error details (including older
candidates) are conservatively treated as infrastructure errors, without parsing
their error text. Successful jobs returning empty answers still count as candidate
failures. The construction failure-rate tolerance applies only to candidate
errors: infrastructure or unclassified failures always reject construction, even
when another failed job was a candidate error.
The manifest retains run identity, selection, mode and execution status. The
summary retains only outcome counts, expected/scored query counts, score means,
low-score cases, question-type statistics and compact case/query mappings.
Candidate answer token summaries average only total tokens per model; detailed
construction costs remain in construction results. Temporary post-query archives
are discarded after extracting logs; separate execution, checksum and raw-answer
JSON copies are not saved. See the
[artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md) for the
exact fields, rebuild helper and migration from the previous verbose format.
Results use `home/evaluations/<branch>/<commit-prefix>/<ascending-run-id>/`.
The [artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md#saved-files)
is the reference for saved files, authoritative query records, compaction and
failure evidence. Its [summary reference](src/reme_home/runtime_docs/validation-artifact-guide.md#summary-fields)
covers score denominators, token accounting, debug fields and summary rebuilding.
Before comparing runs, verify completed scored execution, infrastructure outcomes,
candidate code and complete case/query coverage.
The Job returns `success=false` for infrastructure/judge/artifact failures while
preserving its structured answer and result directory; candidate failures remain
ordinary evaluation observations. Successful Jobs returning empty answers count
as candidate failures. Construction failure-rate tolerance applies only to
candidate errors; infrastructure or unclassified failures always reject construction.
See [failure interpretation](src/reme_home/runtime_docs/validation-artifact-guide.md#investigating-a-result)
for structured error classification.
Run/case/query `failure.json` files preserve diagnostics. Interrupted/fail-fast
runs publish `partial_summary.json`, not a completed summary. New executions
always allocate a new run; they never overwrite previous results. For scored runs, default
@ -305,10 +262,10 @@ Debug requires successful construction and nonempty, error-free answers, and
returns `success=false` for candidate failures as well as infrastructure failures.
It cannot reuse construction. Case/query selections refer to the selected mode's
dataset; omitted selections run all its cases/queries.
Debug results retain execution counts and artifacts, but all mean scores are
null. `scored_query_count` and `query_status_counts.unscored` are zero, while
`expected_query_count` still records the expected execution count. Debug/scored
mode is recorded in the manifest.
Debug results retain execution counts and artifacts, with null mean scores.
See the [summary reference](src/reme_home/runtime_docs/validation-artifact-guide.md#summary-fields)
for coverage fields and manifest mode.
The former `dataset_kind` input is rejected; use `debug` instead. Debug results
must not be used to assess answer correctness or candidate quality. No real
Docker/model evaluation is required by Home's isolated unit tests.
@ -581,22 +538,14 @@ checksum checks. A later full rebuild considers the full training set again.
### Successful evaluation history
`reme_home.evaluation.history.HomeEvaluationHistory` is the dedicated
`home.evaluation_history` Component, automatically bound by `HomeEvaluation`.
It owns successful-result registration, lookup, deduplication, cache rebuilding
and the observations used for the next draw. Steps remain invocation adapters;
they do not own cross-evaluation state.
A successful history entry requires `record_history=true` and a started, completed, scored training-set
run with a full summary and no infrastructure errors, following the existing
`Response.success` contract. Low or zero quality scores remain valid observations.
Classified candidate failures retain the existing response semantics; only their
actual numeric scores train EWMA. A run with any infrastructure error is entirely
excluded, including scores from its healthy cases. Debug, cancelled, fail-fast
and pre-start failures never enter successful history. A full summary is the durable
completion boundary: cancelling the caller during subsequent history bookkeeping
does not relabel that already-completed evaluation. Failed registration discards
only derived caches so they can rebuild from the final source run status.
The `home.evaluation_history` Component discovers successful scored training
results and rebuilds sampling observations from saved evidence. Low scores remain
valid observations; infrastructure failures, debug and incomplete runs are excluded.
See [history and sampling evidence](src/reme_home/runtime_docs/validation-artifact-guide.md#evaluation-history-and-adaptive-sampling-evidence)
for eligibility, started/successful rounds, saved plans and cache recovery.
The full summary is the durable completion boundary: cancellation during later
history bookkeeping does not relabel the completed evaluation. Failed registration
discards only derived caches, which can rebuild from saved run evidence.
```python
history = app.context.components["home.evaluation_history"]["default"]
@ -605,159 +554,47 @@ result = history.get_result(results[-1].run_key)
history.rebuild() # Rebuild sampling observations from saved artifacts.
```
The Component owns the workspace lock for these synchronous filesystem methods;
async host code can run them in a worker thread. Successful results are discovered
from each run's manifest and summary; no `history.json` index is written or read.
Legacy index files are ignored and left untouched. Each returned record points to
its run via `run_key`; questions, answers and memory files stay in that result directory.
The separate startup ledger advances only after the first successfully initialized
sandbox, including when a later worker or execution fails. Extra/replacement
workers do not count twice. `home/sampling/state.json` distinguishes started
`round` from `successful_round`. Only successful history advances the probability
clock or changes EWMA; failed runs cannot age cases or alter the next draw's
weights. Started plans live under
`home/sampling/plans/<branch>/<commit-prefix>/<run-id>.json`. Their `round` is the
startup ledger round; `selection_round` records the successful-history snapshot
used for an adaptive draw.
For `record_history=false`, full or explicit evaluation does not access the history
manager. Adaptive selection can still read existing observations, while leaving
them unchanged; its draw is saved locally as `manifest.selection_plan`, without
a startup-ledger round or shared sampling plan. `manifest.options.record_history`
persists the requested setting, and the response's `record_history` indicates
whether the run participates in history (always false for debug). Legacy manifests
without the setting retain the previous history behavior.
Formal runs remain serialized by `home/metadata/validation.lock`; debug bypasses
that lock. Missing caches reconstruct from started plans and successful scored
artifacts. Older sampling caches are rebuilt under the success-only rule.
`history.rebuild()` rewrites only the catalog and sampling cache. Preserve source
manifests, plans and result artifacts when rebuilding; candidate code, datasets
and memory files are never modified.
These synchronous filesystem methods own the workspace lock; async host code can
run them in a worker thread. Formal runs remain serialized by
`home/metadata/validation.lock`; debug bypasses that lock. Rebuilding changes only
derived state; preserve source manifests, plans and result artifacts.
## Evaluation result inspection
Two read-only Jobs navigate saved evaluations independently of successful
optimization history. They include debug, failed, cancelled, running and standalone
(`record_history=false`) runs. Reads never register results, update sampling,
rebuild summaries or write artifacts.
Read-only inspection includes debug, failed, cancelled, running and standalone
(`record_history=false`) runs, independently of successful optimization history.
It never registers results, updates sampling, rebuilds summaries or writes artifacts.
`home_list_validation_results` accepts optional `branch_name` and `commit_sha`
filters plus `offset=0`, `limit=20` (1–100). Without filters it lists all runs,
newest first, with a stable path tie-breaker. It returns `row_count`, `rows` and
`next_offset`, compact identity/options, saved summary counts and scores, failure
previews and artifact paths. A malformed run is reported as a `read_error` row
without stopping the unfiltered catalog.
`commit_sha` accepts a full SHA or hexadecimal prefix. A prefix must match exactly
one of **all commits reachable from the selected local branch**, including commits
that have never been evaluated. Omitting `branch_name` with a commit filter uses
the current candidate branch. Missing or ambiguous matches fail explicitly; the
response includes the resolved branch and full SHA. A branch-only filter can also
browse historical runs from a deleted branch.
`home_inspect_validation` requires `validation_path` (absolute or relative to
`home/`) and accepts optional `case_id`, `query_id`, `offset=0` and `limit=20`:
| Selection | Returned rows |
| Job | Use |
| --- | --- |
| Run only | Case summaries, missing scores first, then low scores. |
| Run and case | Query summaries in manifest order, plus independently paged failed construction Jobs. |
| Run, case and query | One query's question, answer, rubric, score and judge/error previews. |
`query_id` requires `case_id`; both must be selected by the run manifest. Cases or
queries without published evidence are explicitly marked `missing`, with null
scores. The tool uses a saved full summary, or an existing partial summary when
the full summary is absent. Missing summaries are reported without reconstruction.
Errors are previewed to 600 characters, and selected query fields to 2,000 each;
truncated fields carry a marker. `rows_source`, JSON `pointer` values, and log and
memory paths identify the original evidence. Failed construction Jobs have their
own `row_count`, `rows`, `next_offset` under `construction.failed_jobs`; their
pagination is independent of the query rows, using the supplied offset/limit.
| `home_list_validation_results` | Find runs, optionally filtered by `branch_name` and `commit_sha`. |
| `home_inspect_validation` | Inspect a `validation_path`, then narrow with `case_id` and `query_id`. |
| `home_trace_query_evidence` | Link a query to source sessions, memory checkpoints and saved evidence without a model call. |
| `home_inspect_memory_history` | Page through one case's exported memory Git checkpoints. |
| `home_compare_memory` | Compare committed memory within a run or across evaluations of the same case. |
With the configured service running, use a second terminal:
```bash
reme home_list_validation_results branch_name=candidate commit_sha=abc123 limit=10
reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1
reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1 case_id=case-a query_id=query-a
```
Host callers use the same Jobs through `app.run_job()`. For a case-scoped agent,
inject `validation_path` and `case_id` with `injected_job_kwargs`; the wrapper hides
and enforces these values. Both Jobs reject path escapes and symlinks, and fail
closed for restricted `_allowed_paths` contexts, like the other Home Jobs.
### Memory checkpoint inspection
Read-only Jobs investigate a case's exported memory Git repository. Locate
an evaluation and case with `home_list_validation_results` and
`home_inspect_validation` first. These Jobs never change artifacts, Git refs, the
index, evaluation history or sampling state.
- `home_trace_query_evidence(validation_path, case_id, query_id,
evidence_session_ids=null)` joins a selected query to its source sessions and
memory checkpoints without calling a model. Dataset evidence annotations take
precedence over optional inferred session IDs, including an empty annotation.
Returns `query` (question, expected/actual answer, score, error and
`result_missing` and `expected_kind`), `locations` with JSON Pointers into source
and saved query files, optional `criteria` previews of saved rubric judgments,
`evidence_source` (`independent_annotation`,
`diagnostician_inference`, or `unavailable`), `evidence_is_certain`,
`memory_status`, `sessions` (ID, source JSON Pointer, checkpoint and changed files), and `paths` to
the normalized case, memory repository, query results and query artifacts.
A null checkpoint means no matching `session: <id>` commit was found; missing
results keep scores null. Answer/question previews are limited to 2,000
characters and errors to 600. At most 50 sessions and 50 changed files per
session are returned, with omitted counts when needed. Criterion previews are
limited to 20 entries and 600 characters per requirement/reason; absent or
mismatched rubric judgments are not paired. Session text, diffs,
later history and full answer/judge results remain at the returned paths or
in the inspection/comparison Jobs below. Original transcripts and derived
memories can both be searchable; these coordinates alone do not prove a cause.
- `home_inspect_memory_history(validation_path, case_id, path=null, offset=0,
limit=20)` returns `status`, `workspace_exists`, `has_git_history`, `head`,
`repository`, and newest-first checkpoint `rows` with `row_count` and
`next_offset`. Limits are 1–100. Each row includes the full commit SHA, parents,
timestamp, subject (up to 2,000 characters) and a `session_id` when the subject
follows `session: <id>`. `path` follows a literal workspace-relative file or
directory. Missing exports, absent Git history and empty repositories have
distinct statuses; corrupt repositories fail explicitly.
- `home_compare_memory(validation_path, case_id, base_ref=null,
target_ref="HEAD", target_validation_path=null, path=null)` compares two memory
commits. Within one evaluation, `base_ref` is required. For another evaluation,
supply `target_validation_path`; both sides then default to `HEAD`. The case
must be selected in both manifests. Returns resolved commits, candidate commit
identities, manifest fingerprints, added/removed/modified paths, content-tree
hashes, a unified `diff`, and `truncated`.
References accept HEAD-relative expressions and hexadecimal SHA prefixes unique
among commits reachable from that memory repository's HEAD. Resolved expressions
must also be HEAD-reachable. This is independent of candidate-code branch history.
Comparisons read committed regular files from Git objects, ignoring dirty or
untracked checkout contents. File mode changes alone are not content differences.
Binary changes are identified without printing their bytes. Each snapshot is
limited to 2,000 files, 20 MB total and 5 MB per file; diff output is limited to
100,000 characters. Narrow `path` if a limit is reached. Tree hashes describe the
selected path scope. Linked Git storage, alternates, shallow/partial histories and
symbolic links in Git metadata are rejected.
After starting the service with the Home configuration, call from another terminal:
```bash
reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1
reme home_list_validation_results branch_name=candidate limit=10
reme home_inspect_validation validation_path=evaluations/candidate/abcdef0/1
reme home_inspect_validation validation_path=evaluations/candidate/abcdef0/1 case_id=case1 query_id=query1
reme home_trace_query_evidence validation_path=evaluations/candidate/abcdef0/1 case_id=case1 query_id=query1
reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1
reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 case_id=case1 base_ref=HEAD~1
reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 target_validation_path=evaluations/candidate/1234567/1 case_id=case1
```
`validation_path` is absolute or relative to `home/`; for example the relative
form is `evaluations/candidate/abcdef0/1`. For agent tools, host `injected_job_kwargs` can pin
`validation_path` and `case_id`; the wrapper hides injected fields and rejects
model overrides. Like the other Home inspection Jobs, these Jobs reject
`_allowed_paths` restrictions rather than bypassing them. Use these tools directly
or through the case-study batch Job below.
`validation_path` is absolute or relative to `home/`. See
[read-only evaluation inspection](src/reme_home/runtime_docs/validation-artifact-guide.md#read-only-evaluation-inspection)
for pagination, commit-prefix resolution, previews and source pointers, and
[memory checkpoint tools](src/reme_home/runtime_docs/validation-artifact-guide.md#memory-checkpoint-tools)
for evidence provenance, missing-history states, comparison references and limits.
Host callers use the same Jobs through `app.run_job()`. For case-scoped agent
tools, pin `validation_path` and `case_id` through `injected_job_kwargs`;
the wrapper hides those fields and rejects model overrides. These Jobs reject
path escapes, symlinks and restricted `_allowed_paths` contexts.
## Case-study batches
@ -794,19 +631,11 @@ available. No case scratch directory or intermediate analysis files are created.
still return a text report with per-case metadata. Do not supply both path names
or both case selectors. Other inspection Jobs retain their `validation_path` argument.
The toolkit includes validation inspection, compact query-evidence tracing,
memory history/comparison, and the three committed-code navigation Jobs.
Validation/case selectors and the evaluated code revision are hidden and enforced
by the host; memory comparison stays within the selected evaluation. General
Read/Grep/Glob, Write/Edit, validation and code-mutation tools are not exposed.
The case-study toolkit and mandatory read-only OS sandbox are described in the
[artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md#single-case-diagnostic-agent).
The host pins case/evaluation selectors and code revision. Direct code reads can
see current worktree changes; use revision-pinned Jobs to establish evaluated behavior.
`AutoBash` and `python_execute` share a required OS sandbox. Commands have no
static whitelist (`is_read_only=false`); filesystem permissions enforce access.
Read-only roots are the selected case artifact directory, normalized source-case
JSON, validation manifest, all of `home/code/`, and `home/docs/`. No host writable
roots are granted. System/interpreter runtime paths and disposable backend
temporary storage remain available. Direct code reads can see current worktree
changes; use the revision-pinned Jobs to establish the evaluated behavior.
`python_execute` accepts either `code` or an existing `file`, optional file-only
`args`, and a `timeout` in seconds (default 60, maximum 600). It uses `python -I -B`
and returns returncode/stdout/stderr/timed_out. Both executors reuse bounded output,

View file

@ -5,7 +5,7 @@ description = "Self-evolution workspace preparation and local command execution
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
"reme-ai[as]>=0.4.2.1",
"reme-ai[as]>=0.4.2.2",
"safecmd>=0.1.16,<0.2",
]

View file

@ -30,6 +30,8 @@ class TimeBudgetReminder:
class RunTimeBudgetTimer:
"""Track one agent run and emit advisory reminders without stopping it."""
reminder_thresholds = (0.90, 0.75, 0.60)
def __init__(
self,
time_budget_minutes: float,
@ -89,7 +91,7 @@ class RunTimeBudgetTimer:
if elapsed_ratio >= 1.0:
reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10
else:
for threshold in (0.90, 0.75, 0.60):
for threshold in self.reminder_thresholds:
if elapsed_ratio >= threshold:
reached_ratio = threshold
break
@ -98,17 +100,21 @@ class RunTimeBudgetTimer:
self._last_reminded_ratio = reached_ratio
persistent = self._next_reminder_is_persistent
self._next_reminder_is_persistent = not persistent
return TimeBudgetReminder(
content=self._reminder_content(elapsed_minutes, reached_ratio),
persistent=persistent,
)
def _reminder_content(self, elapsed_minutes: float, reached_ratio: float) -> str:
"""Describe this run without changing the shared reminder schedule."""
elapsed = _format_minutes(elapsed_minutes)
percentage = round(reached_ratio * 100)
return TimeBudgetReminder(
content=(
f"<system-reminder>The {self.agent_label} run has used {elapsed} ({percentage}% of its "
f"{_format_minutes(self.time_budget_minutes)} soft time budget). "
f"The run has completed {self.context_compression_count} context compressions. "
"This reminder does not terminate the run. Prioritize the highest-value remaining investigation and "
"produce the required final report as soon as the evidence is sufficient.</system-reminder>"
),
persistent=persistent,
return (
f"<system-reminder>The {self.agent_label} run has used {elapsed} ({percentage}% of its "
f"{_format_minutes(self.time_budget_minutes)} soft time budget). "
f"The run has completed {self.context_compression_count} context compressions. "
"This reminder does not terminate the run. Prioritize the highest-value remaining investigation and "
"produce the required final report as soon as the evidence is sufficient.</system-reminder>"
)

View file

@ -1,18 +1,12 @@
"""Early candidate admission using read-only, commit-pinned source exports."""
import json
from contextlib import asynccontextmanager
from pathlib import Path
import shutil
import sys
from typing import Literal
from uuid import uuid4
from agentscope.tool import Glob, Grep, Read, Toolkit
from pydantic import Field
import yaml
from ..auto_bash import AutoBash
from ..code.repository import current_branch, require_clean, resolve_revision, validate_repository, validate_tree
from ..code.snapshot import export_blob, tree
from ..evaluation.artifacts import contained, read_json
@ -20,7 +14,7 @@ from ..evaluation.component import blocking
from ..evaluation.repetition import previous_candidate_result
from ..preparation.code import git
from ..preparation.spec import SpecModel
from .workflow_common import WorkflowSandbox, save_record
from .workflow_common import review_report
class CandidateGateReport(SpecModel):
@ -92,51 +86,18 @@ async def review_candidate(wrapper, home, context, budget, *, memory_limit_mib=N
"""Fail closed on sandbox setup; model/report failures allow validation as in dev_meta."""
async with review_export(home, "candidate-gate") as root:
scope = await blocking(prepare_review_scope, home, context, root=root)
return await _review_candidate(wrapper, home, context, budget, scope, memory_limit_mib)
async def _review_candidate(wrapper, home, context, budget, scope, memory_limit_mib):
"""Run inside the caller-owned export lifetime; persist only the report."""
backend = WorkflowSandbox(scope, memory_limit_mib)
probe = await backend.exec_shell([sys.executable, "-I", "-B", "-c", "print('candidate gate sandbox')"], timeout=10)
if not probe.ok():
raise RuntimeError("Candidate gate sandbox unavailable: " + probe.stderr.decode(errors="replace"))
tools = [
Read(backend=backend),
Glob(backend=backend),
Grep(backend=backend),
AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=True),
]
for tool in tools:
tool.is_concurrency_safe = False
prompt = yaml.safe_load(Path(__file__).with_suffix(".yaml").read_text(encoding="utf-8"))
report = {**context, "validation_budget": budget, "source_path": scope["scratch"]}
try:
reply = await wrapper.reply(
return await review_report(
wrapper,
home,
scope,
f"Review admission from {context['base_commit']} to {context['candidate_commit']}.\n"
f"Source exports and complete diff.patch: {scope['scratch']}",
system_prompt=prompt["system_prompt"],
toolkit=Toolkit(tools=tools),
builtin_tools=[],
job_tools=[],
skills=[],
resume=None,
session_id=None,
fork_session=False,
output_schema=None,
_structured_report_model=CandidateGateReport,
permission_mode="bypass",
{**context, "validation_budget": budget, "source_path": scope["scratch"]},
role="candidate_gate",
sandbox_label="Candidate gate",
category="candidate-gates",
report_model=CandidateGateReport,
failure_decision="accept",
failure_reason="Candidate gate failed; validation is allowed",
memory_limit_mib=memory_limit_mib,
)
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})
text = reply["result"].strip()
if text.startswith("```") and text.endswith("```"):
text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
report.update(CandidateGateReport.model_validate(json.loads(text)).model_dump())
except Exception as exc:
report.update(
decision="accept",
reason=f"Candidate gate failed; validation is allowed: {type(exc).__name__}: {exc}",
review_failed=True,
)
path = await blocking(save_record, home, "candidate-gates", report)
return {**report, "review_path": path}

View file

@ -1,23 +1,17 @@
"""Construction equivalence review over isolated source and construction exports."""
import json
from pathlib import Path
import shutil
import sys
from typing import Literal
from agentscope.tool import Glob, Grep, Read, Toolkit
from pydantic import Field
import yaml
from ..auto_bash import AutoBash
from ..evaluation.artifacts import contained, file_sha256, inspect_archive, read_json, write_json
from ..evaluation.component import blocking
from ..evaluation.inputs import reuse_inputs
from ..preparation.code import tree_fingerprint
from ..preparation.spec import SpecModel
from .candidate_gate import prepare_review_scope, review_export
from .workflow_common import WorkflowSandbox, save_record
from .workflow_common import review_report
class ConstructionEquivalenceReport(SpecModel):
@ -65,54 +59,18 @@ async def review_construction(wrapper, home, context, spec, cases, fingerprints,
"""Stop on sandbox setup errors; review errors conservatively require rebuilding."""
async with review_export(home, "construction-equivalence") as root:
scope = await blocking(prepare_equivalence_scope, home, context, spec, cases, fingerprints, root=root)
return await _review_construction(wrapper, home, context, scope, memory_limit_mib)
async def _review_construction(wrapper, home, context, scope, memory_limit_mib):
"""Run inside the caller-owned export lifetime; persist only the report."""
backend = WorkflowSandbox(scope, memory_limit_mib)
probe = await backend.exec_shell(
[sys.executable, "-I", "-B", "-c", "print('construction review sandbox')"],
timeout=10,
)
if not probe.ok():
raise RuntimeError("Construction review sandbox unavailable: " + probe.stderr.decode(errors="replace"))
tools = [
Read(backend=backend),
Glob(backend=backend),
Grep(backend=backend),
AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=True),
]
for tool in tools:
tool.is_concurrency_safe = False
prompt = yaml.safe_load(Path(__file__).with_suffix(".yaml").read_text(encoding="utf-8"))
report = {**context, "source_path": scope["scratch"]}
try:
reply = await wrapper.reply(
return await review_report(
wrapper,
home,
scope,
f"Compare construction from {context['base_commit']} to {context['candidate_commit']}.\n"
f"Read-only source, diff and construction evidence: {scope['scratch']}",
system_prompt=prompt["system_prompt"],
toolkit=Toolkit(tools=tools),
builtin_tools=[],
job_tools=[],
skills=[],
resume=None,
session_id=None,
fork_session=False,
output_schema=None,
_structured_report_model=ConstructionEquivalenceReport,
permission_mode="bypass",
{**context, "source_path": scope["scratch"]},
role="construction_equivalence",
sandbox_label="Construction review",
category="construction-equivalence",
report_model=ConstructionEquivalenceReport,
failure_decision="construction_changed",
failure_reason="Equivalence review failed; rebuild construction",
memory_limit_mib=memory_limit_mib,
)
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})
text = reply["result"].strip()
if text.startswith("```") and text.endswith("```"):
text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
report.update(ConstructionEquivalenceReport.model_validate(json.loads(text)).model_dump())
except Exception as exc:
report.update(
decision="construction_changed",
reason=f"Equivalence review failed; rebuild construction: {type(exc).__name__}: {exc}",
review_failed=True,
)
path = await blocking(save_record, home, "construction-equivalence", report)
return {**report, "review_path": path}

View file

@ -2,7 +2,6 @@
import asyncio
import json
import math
import time
from pathlib import Path
from typing import Callable
@ -23,7 +22,7 @@ from ..evaluation.budget import validate_max_evaluations, validation_budget_stat
from ..preparation.spec import SpecModel
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from .diagnostic import diagnostic_tool, evaluation_paths
from .bounded import TimeBudgetReminder, _format_minutes, _remove_hint_by_identity
from .bounded import RunTimeBudgetTimer, _format_minutes, _remove_hint_by_identity
from .compression import RecoverableCompressionAgent
from .workflow_common import required_job, save_record, workflow_reply
from .wrapper import CaseStudyAgentWrapper
@ -37,55 +36,23 @@ MUTATION_JOBS = (
PENDING_CYCLE_TRANSITIONS = "home.optimizer.pending_cycle_transitions"
class ValidationCycleTimer:
class ValidationCycleTimer(RunTimeBudgetTimer):
"""Emit escalating reminders as a scored-validation cycle ages."""
reminder_thresholds = (0.90, 0.75, 0.50)
def __init__(self, time_budget_minutes: float, *, clock: Callable[[], float] = time.monotonic) -> None:
if (
not isinstance(time_budget_minutes, (int, float))
or isinstance(time_budget_minutes, bool)
or not math.isfinite(time_budget_minutes)
or time_budget_minutes <= 0
):
raise ValueError("time_budget_minutes must be a positive number")
self.time_budget_minutes = float(time_budget_minutes)
self._clock = clock
self._started_at = self._clock()
self._last_reminded_ratio = 0.0
self._next_reminder_is_persistent = False
self.context_compression_count = 0
super().__init__(time_budget_minutes, agent_label="optimizer", clock=clock)
self.start()
def reset(self) -> None:
"""Start a fresh cycle after a scored validation finishes."""
self._started_at = self._clock()
self._last_reminded_ratio = 0.0
self._next_reminder_is_persistent = False
self.context_compression_count = 0
self.start()
def record_context_compression(self) -> None:
"""Record one completed compression since the last scored validation."""
self.context_compression_count += 1
def next_reminder(self) -> TimeBudgetReminder | None:
"""Return one reminder for the latest newly reached deadline."""
elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0)
def _reminder_content(self, elapsed_minutes: float, reached_ratio: float) -> str:
"""Keep optimizer guidance tied to the last scored validation."""
elapsed_ratio = elapsed_minutes / self.time_budget_minutes
reached_ratio: float | None = None
if elapsed_ratio >= 1.0:
reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10
else:
for threshold in (0.9, 0.75, 0.5):
if elapsed_ratio >= threshold:
reached_ratio = threshold
break
if reached_ratio is None or reached_ratio <= self._last_reminded_ratio:
return None
self._last_reminded_ratio = reached_ratio
persistent = self._next_reminder_is_persistent
self._next_reminder_is_persistent = not persistent
elapsed = _format_minutes(elapsed_minutes)
percentage = round(reached_ratio * 100)
timing = (
@ -97,22 +64,16 @@ class ValidationCycleTimer:
"scored validation. "
)
if elapsed_ratio >= 1.0:
return TimeBudgetReminder(
content=(
f"<system-reminder>{timing}{compression}"
"Prioritize completing the code changes, committing them, passing the smoke test, and starting "
"validation as soon as possible.</system-reminder>"
),
persistent=persistent,
return (
f"<system-reminder>{timing}{compression}"
"Prioritize completing the code changes, committing them, passing the smoke test, and starting "
"validation as soon as possible.</system-reminder>"
)
remaining = _format_minutes(self.time_budget_minutes - elapsed_minutes)
return TimeBudgetReminder(
content=(
f"<system-reminder>{timing}{compression}"
f"Complete the code changes, commit them, pass the smoke test, and start validation within {remaining}."
"</system-reminder>"
),
persistent=persistent,
return (
f"<system-reminder>{timing}{compression}"
f"Complete the code changes, commit them, pass the smoke test, and start validation within {remaining}."
"</system-reminder>"
)

View file

@ -150,6 +150,72 @@ def save_record(home, category, payload):
return str(path)
async def review_report(
wrapper,
home,
scope,
task,
report,
*,
role,
sandbox_label,
category,
report_model,
failure_decision,
failure_reason,
memory_limit_mib=None,
):
"""Share read-only review mechanics while callers choose their failure policy.
Sandbox and prompt setup and report persistence deliberately stay outside
the model/report recovery boundary. Cancellation propagates to the export owner.
"""
backend = WorkflowSandbox(scope, memory_limit_mib)
probe = await backend.exec_shell(
[sys.executable, "-I", "-B", "-c", f"print('{sandbox_label.lower()} sandbox')"],
timeout=10,
)
if not probe.ok():
raise RuntimeError(f"{sandbox_label} sandbox unavailable: " + probe.stderr.decode(errors="replace"))
tools = [
Read(backend=backend),
Glob(backend=backend),
Grep(backend=backend),
AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=True),
]
for tool in tools:
tool.is_concurrency_safe = False
prompt = yaml.safe_load(Path(__file__).with_name(f"{role}.yaml").read_text(encoding="utf-8"))
try:
reply = await wrapper.reply(
task,
system_prompt=prompt["system_prompt"],
toolkit=Toolkit(tools=tools),
builtin_tools=[],
job_tools=[],
skills=[],
resume=None,
session_id=None,
fork_session=False,
output_schema=None,
_structured_report_model=report_model,
permission_mode="bypass",
)
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})
text = reply["result"].strip()
if text.startswith("```") and text.endswith("```"):
text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
report.update(report_model.model_validate(json.loads(text)).model_dump())
except Exception as exc:
report.update(
decision=failure_decision,
reason=f"{failure_reason}: {type(exc).__name__}: {exc}",
review_failed=True,
)
path = await blocking(save_record, home, category, report)
return {**report, "review_path": path}
_INSPECT = """\
import json, pathlib, sys
path = pathlib.Path(sys.argv[2])

View file

@ -331,10 +331,12 @@ def rebuild_summary(run: Path) -> dict:
INFRA_TYPES = {
"APIConnectionError",
"APITimeoutError",
"AuthenticationError",
"ConnectError",
"ConnectionError",
"ConnectTimeout",
"NetworkError",
"PermissionDeniedError",
"PoolTimeout",
"RateLimitError",
"ReadError",
@ -350,7 +352,7 @@ INFRA_TYPES = {
def failure_kind(exception, stage: str) -> str:
"""Only candidate execution errors count against candidate scores."""
"""Only candidate execution errors count against scores; provider access errors are infrastructure failures."""
if stage not in {"construction", "answer"}:
return "infra_error"
exception = exception if isinstance(exception, dict) else {}
@ -360,7 +362,7 @@ def failure_kind(exception, stage: str) -> str:
if isinstance(causes, list) and any(failure_kind(cause, stage) == "infra_error" for cause in causes):
return "infra_error"
status = exception.get("status_code")
if isinstance(status, int) and (status in {408, 429} or status >= 500):
if isinstance(status, int) and (status in {401, 403, 408, 429} or status >= 500):
return "infra_error"
name = str(exception.get("type", ""))
module = str(exception.get("module", ""))

View file

@ -235,8 +235,12 @@ class HomeEvaluation(BaseComponent):
manifest["selection_plan"] = plan
write_json(run / "manifest.json", manifest)
return
self.history.record_started(run, policy if policy is not None else self.sampling_policy, plan)
manifest.update(read_json(run / "manifest.json"))
try:
self.history.record_started(run, policy if policy is not None else self.sampling_policy, plan)
finally:
# Cache publication can fail after the durable start was written.
# Preserve those fields when the caller later saves failure status.
manifest.update(read_json(run / "manifest.json"))
def _finish_sampling(self, run):
self.history.record(run)

View file

@ -114,35 +114,45 @@ def start_validation(home: Path, run: Path, policy: SamplingPolicy, plan: dict |
return {}
key = run.relative_to(home / "evaluations").as_posix()
state = read_state(home)
if key in state["started_validations"]:
return read_json(contained(home, manifest["sampling"]["plan_path"]))
plan = (
dict(plan)
if plan is not None
else {
"mode": "explicit",
"case_ids": manifest["case_ids"],
"query_ids": manifest["query_ids"],
"policy": policy.model_dump(),
}
)
# Preserve the draw's state snapshot separately from the committed start round.
if "round" in plan:
plan["selection_round"] = plan["round"]
plan.update(
round=state["round"] + 1,
validation_id=manifest["validation_id"],
validation_key=key,
sandbox_started_at=utc_now().isoformat(),
)
relative = f"sampling/plans/{key}.json"
write_json(contained(home, relative), plan)
if key in state["started_validations"]:
plan = read_json(contained(home, relative))
else:
plan = (
dict(plan)
if plan is not None
else {
"mode": "explicit",
"case_ids": manifest["case_ids"],
"query_ids": manifest["query_ids"],
"policy": policy.model_dump(),
}
)
# Preserve the draw's state snapshot separately from the committed start round.
if "round" in plan:
plan["selection_round"] = plan["round"]
plan.update(
round=state["round"] + 1,
validation_id=manifest["validation_id"],
validation_key=key,
sandbox_started_at=utc_now().isoformat(),
)
# Invalidate before publishing evidence: interruption at either following
# write must rebuild from the started plan rather than trust an old cache.
cache = contained(home, "sampling/state.json")
cache.unlink(missing_ok=True)
if key not in state["started_validations"]:
write_json(contained(home, relative), plan)
manifest["sampling"] = {"round": plan["round"], "mode": plan["mode"], "plan_path": relative}
manifest["sandbox_started_at"] = plan["sandbox_started_at"]
write_json(run / "manifest.json", manifest)
state["round"] = plan["round"]
state["round"] = max(state["round"], plan["round"])
state["started_validations"][key] = plan["round"]
write_json(contained(home, "sampling/state.json"), state)
try:
write_json(cache, state)
except BaseException:
cache.unlink(missing_ok=True)
raise
return plan
@ -231,10 +241,17 @@ def rebuild_sampling_state(home: Path, *, persist: bool = True) -> dict:
manifest.get("mode") == "debug"
or manifest.get("dataset_kind") != "training-set"
or manifest.get("options", {}).get("record_history") is False
or not (manifest.get("sandbox_started_at"))
):
continue
plan = read_json(contained(home, manifest["sampling"]["plan_path"]))
key = path.parent.relative_to(home / "evaluations").as_posix()
plan_path = contained(home, manifest.get("sampling", {}).get("plan_path", f"sampling/plans/{key}.json"))
if not manifest.get("sandbox_started_at") and not plan_path.is_file():
continue
plan = read_json(plan_path)
# A started plan is independent evidence even if the process stopped
# before publishing the manifest's start fields.
if not plan.get("sandbox_started_at"):
continue
runs.append((plan["round"], path.parent, plan["validation_key"]))
for round_id, run, key in sorted(runs):
state["round"] = max(state["round"], round_id)

View file

@ -86,12 +86,12 @@ application_defaults:
thinking_enable: false
agent_wrapper:
home_construction_equivalence:
backend: home_workflow_agent
backend: home_case_study_agent
as_llm: home_construction_equivalence
react_config:
max_iters: 100
home_candidate_gate:
backend: home_workflow_agent
backend: home_case_study_agent
as_llm: home_candidate_gate
react_config:
max_iters: 60
@ -106,7 +106,7 @@ application_defaults:
react_config:
max_iters: 3000
home_diagnostic:
backend: home_workflow_agent
backend: home_case_study_agent
as_llm: home_diagnostic
time_budget_minutes: 60
react_config:

View file

@ -434,12 +434,11 @@ async def test_partial_export_and_report_failures_clean_review_files(
assert (await app.run_job("home_evaluate")).success
commit_candidate(prepared)
if stage == "report":
module = candidate_gate if kind == "candidate-gate" else construction_equivalence
def fail_report(*args, **kwargs):
raise OSError("report write failed")
monkeypatch.setattr(module, "save_record", fail_report)
monkeypatch.setattr(workflow_common, "save_record", fail_report)
elif kind == "candidate-gate":
original = candidate_gate.export_blob

View file

@ -680,13 +680,31 @@ async def test_startup_rejects_disabled_or_replaced_judge_job(app, tmp_path, ove
@pytest.mark.asyncio
@pytest.mark.parametrize("phase", ["answer", "construction"])
@pytest.mark.parametrize("kind", ["timeout", "wrapped_timeout", "http_429", "http_503", "code", "legacy"])
@pytest.mark.parametrize(
"kind",
[
"timeout",
"wrapped_timeout",
"http_401",
"http_403",
"sdk_auth",
"sdk_permission",
"http_429",
"http_503",
"code",
"legacy",
],
)
async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monkeypatch, phase, kind):
from types import SimpleNamespace
import httpx
from deploy.benchmark_docker.worker import _run_job_on_app
from reme.components.job.base_job import BaseJob
authentication_failure = kind in {"http_401", "http_403", "sdk_auth", "sdk_permission"}
method = "run_queries" if phase == "answer" else "run_build"
original = getattr(FakeWorker, method)
async def failing_step(_context):
if kind == "timeout":
raise TimeoutError("model timed out")
@ -695,6 +713,14 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk
raise TimeoutError("model timed out")
except TimeoutError as error:
raise RuntimeError("model invocation failed") from error
if kind.startswith("sdk_"):
import openai
request = httpx.Request("POST", "https://example.test")
status = 401 if kind == "sdk_auth" else 403
response = httpx.Response(status, request=request)
error_type = openai.AuthenticationError if kind == "sdk_auth" else openai.PermissionDeniedError
raise error_type("provider access denied", response=response, body=None)
if kind.startswith("http_"):
request = httpx.Request("POST", "https://example.test")
response = httpx.Response(int(kind.split("_")[1]), request=request)
@ -761,6 +787,74 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk
if kind != "legacy":
assert failure["exception"] == result["exception"]
assert all(worker.closed for worker in FakeFactory.instances[-1].workers)
if authentication_failure:
history = app.context.components["home.evaluation_history"]["default"]
assert history.list_results() == []
# Correcting the credentials must allow the same commit to be assessed.
monkeypatch.setattr(FakeWorker, method, original)
retry = await app.run_job("home_evaluate", case_ids=["1"])
assert retry.success, retry.answer
assert retry.answer["status"] == "completed"
assert retry.answer["commit_sha"] == summary["commit_sha"]
assert Path(retry.answer["result_dir"]).name == "2"
assert retry.answer["scored_query_count"] == 1
assert len(history.list_results()) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("phase", ["construction", "answer"])
@pytest.mark.parametrize("kind", ["http_503", "wrapped_http_503", "wrapped_timeout", "code"])
async def test_pool_exceptions_preserve_classification_and_retry_policy(app, prepared, monkeypatch, phase, kind):
import httpx
async def fail(worker, inputs):
if kind == "code":
raise ValueError("candidate bug")
try:
if kind == "wrapped_timeout":
raise TimeoutError("transport timed out")
request = httpx.Request("POST", "https://example.test")
response = httpx.Response(503, request=request)
raise httpx.HTTPStatusError("service unavailable", request=request, response=response)
except (TimeoutError, httpx.HTTPStatusError) as error:
if kind.startswith("wrapped_"):
raise RuntimeError("worker invocation failed") from error
raise
monkeypatch.setattr(FakeWorker, "run_build" if phase == "construction" else "run_queries", fail)
response = await app.run_job("home_evaluate", case_ids=["1"])
infrastructure = kind != "code"
assert response.success is (not infrastructure)
summary = response.answer
failure_kind = "infra_error" if infrastructure else "candidate_failure"
assert summary["case_status_counts"][failure_kind] == 1
assert summary["mean_query_score"] == (None if infrastructure else 0)
run = Path(summary["result_dir"])
failure = json.loads((run / "cases/1/failure.json").read_text())
assert failure["stage"] == phase
assert failure["failure_kind"] == failure_kind
if kind == "code":
expected = {"type": "ValueError", "module": "builtins"}
else:
cause = (
{"type": "TimeoutError", "module": "builtins"}
if kind == "wrapped_timeout"
else {"type": "HTTPStatusError", "module": "httpx", "status_code": 503}
)
expected = (
{"type": "RuntimeError", "module": "builtins", "causes": [cause]} if kind.startswith("wrapped_") else cause
)
assert failure["exception"] == expected
retry = await app.run_job("home_evaluate", case_ids=["1"])
if infrastructure:
assert retry.answer["status"] == "completed"
assert Path(retry.answer["result_dir"]).name == "2"
assert len(FakeFactory.instances) == 2
else:
assert retry.answer["error_code"] == "UNCHANGED_CANDIDATE"
assert len(FakeFactory.instances) == 1
assert all(worker.closed for factory in FakeFactory.instances for worker in factory.workers)
@pytest.mark.asyncio

View file

@ -8,11 +8,37 @@ import json
import pytest
from reme_home.evaluation.artifacts import average_tokens, read_json, rebuild_summary, summarize, write_json
from reme_home.evaluation.artifacts import (
average_tokens,
failure_kind,
read_json,
rebuild_summary,
summarize,
write_json,
)
from reme_home.evaluation.compaction import compact_construction, compact_query
from reme_home.preparation.data.models import CaseSpec, QuerySpec
@pytest.mark.parametrize("stage", ["construction", "answer"])
@pytest.mark.parametrize("wrapped", [False, True])
@pytest.mark.parametrize(
"exception, expected",
[
({"type": "HTTPStatusError", "module": "httpx", "status_code": 401}, "infra_error"),
({"type": "HTTPStatusError", "module": "httpx", "status_code": 403}, "infra_error"),
({"type": "AuthenticationError", "module": "openai"}, "infra_error"),
({"type": "PermissionDeniedError", "module": "openai"}, "infra_error"),
({"type": "HTTPStatusError", "module": "httpx", "status_code": 400}, "candidate_failure"),
({"type": "ValueError", "module": "builtins"}, "candidate_failure"),
],
)
def test_provider_access_errors_are_infrastructure_failures(stage, wrapped, exception, expected):
if wrapped:
exception = {"type": "RuntimeError", "module": "builtins", "causes": [exception]}
assert failure_kind(exception, stage) == expected
def test_scores_are_rounded_without_changing_reference_text_or_token_metrics(tmp_path):
raw = {
"query_id": "q",

View file

@ -7,7 +7,15 @@ from pathlib import Path
import pytest
from agentscope.message import ToolResultState
from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state
from reme_home.evaluation.history import (
HomeEvaluationHistory,
EvaluationRecord,
read_state,
rebuild_sampling_state,
start_validation,
)
from reme_home.evaluation.budget import validation_budget_status
from reme_home.evaluation import history as history_module
from reme_home.sampling import SamplingPolicy, create_sampling_plan
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
@ -31,6 +39,86 @@ def manifest(response):
return json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
@pytest.mark.asyncio
async def test_start_cache_failure_keeps_start_evidence_and_exhausts_budget(app, prepared, monkeypatch):
commit_candidate(prepared)
manager = history(app)
manager.rebuild() # Start with a valid but empty cache, not just a missing cache.
original = history_module.write_json
def fail_cache(path, value, **kwargs):
if path == prepared / "sampling/state.json":
raise OSError("injected start cache publication failure")
return original(path, value, **kwargs)
monkeypatch.setattr(history_module, "write_json", fail_cache)
response = await app.run_job("home_evaluate", max_evaluations=1, sampling=True)
assert not response.success
paths = list((prepared / "evaluations").glob("*/*/*/manifest.json"))
assert len(paths) == 1
payload = json.loads(paths[0].read_text())
assert payload["status"] == "failed" and payload["sandbox_started_at"]
plan = json.loads((prepared / payload["sampling"]["plan_path"]).read_text())
assert payload["sandbox_started_at"] == plan["sandbox_started_at"]
assert payload["sampling"]["round"] == plan["round"] == 1
assert not (prepared / "sampling/state.json").exists()
expected = {plan["validation_key"]: 1}
assert read_state(prepared)["started_validations"] == expected
assert rebuild_sampling_state(prepared, persist=False)["started_validations"] == expected
budget = validation_budget_status(prepared, 1)
assert budget["used_validations"] == 1 and budget["remaining_validations"] == 0
rejected = await app.run_job("home_evaluate", max_evaluations=1)
assert rejected.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED"
assert len(FakeFactory.instances) == 1
assert manager.list_results() == []
@pytest.mark.parametrize("interrupted_write", ["manifest.json", "state.json"])
def test_interrupted_start_rebuilds_from_plan_and_retry_is_idempotent(tmp_path, monkeypatch, interrupted_write):
class Interrupted(BaseException):
"""Model process interruption outside ordinary exception handling."""
home = tmp_path / "home"
run = home / "evaluations/candidate/abcdef0/1"
run.mkdir(parents=True)
original = history_module.write_json
original(
run / "manifest.json",
{
"mode": "scored",
"dataset_kind": "training-set",
"status": "running",
"validation_id": "1",
"case_ids": ["1"],
"query_ids": {"1": ["q1"]},
},
)
rebuild_sampling_state(home) # An old cache must not hide newly published evidence.
def interrupt(path, value, **kwargs):
if path.name == interrupted_write:
raise Interrupted()
return original(path, value, **kwargs)
monkeypatch.setattr(history_module, "write_json", interrupt)
with pytest.raises(Interrupted):
start_validation(home, run, SamplingPolicy())
plan_path = home / "sampling/plans/candidate/abcdef0/1.json"
evidence = plan_path.read_bytes()
plan = json.loads(evidence)
assert not (home / "sampling/state.json").exists()
if interrupted_write == "manifest.json":
assert "sandbox_started_at" not in json.loads((run / "manifest.json").read_text())
assert read_state(home)["started_validations"] == {"candidate/abcdef0/1": 1}
assert validation_budget_status(home, 1)["remaining_validations"] == 0
monkeypatch.setattr(history_module, "write_json", original)
assert start_validation(home, run, SamplingPolicy()) == plan
assert plan_path.read_bytes() == evidence
assert json.loads((run / "manifest.json").read_text())["sandbox_started_at"] == plan["sandbox_started_at"]
assert read_state(home)["round"] == 1
assert rebuild_sampling_state(home)["started_validations"] == {"candidate/abcdef0/1": 1}
@pytest.mark.asyncio
async def test_history_catalog_keeps_successful_low_scores_and_deduplicates(app, prepared):
manager = history(app)

View file

@ -1,6 +1,6 @@
"""ReMe CLI package."""
__version__ = "0.4.2.1"
__version__ = "0.4.2.2"
from . import config
from . import constants

View file

@ -27,15 +27,16 @@ bump_version = _load_script("bump_version")
package_studio = _load_script("package_studio")
def test_home_requires_first_sandbox_release() -> None:
"""Home must reject published ReMe wheels that predate its sandbox imports."""
def test_home_requires_workflow_interfaces_release() -> None:
"""Home must reject ReMe wheels that predate its workflow interfaces."""
config = tomllib.loads((REPOSITORY / "plugins" / "home" / "pyproject.toml").read_text(encoding="utf-8"))
requirement = next(Requirement(value) for value in config["project"]["dependencies"] if value.startswith("reme-ai"))
assert requirement.extras == {"as"}
assert Version("0.4.1.14") not in requirement.specifier
assert Version("0.4.2.0") not in requirement.specifier
assert Version("0.4.2.1") in requirement.specifier
assert Version("0.4.2.1") not in requirement.specifier
assert Version("0.4.2.2") in requirement.specifier
assert Version(bump_version.read_version()) in requirement.specifier