mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-15 09:11:24 +00:00
refactor(home): extract shared review_report workflow helper and bump version to 0.4.2.2
This commit is contained in:
parent
75ca3e2011
commit
e89930f9c2
19 changed files with 476 additions and 451 deletions
4
.github/workflows/_build-python-packages.yml
vendored
4
.github/workflows/_build-python-packages.yml
vendored
|
|
@ -96,8 +96,8 @@ jobs:
|
|||
- name: Reject incompatible published ReMe for Home
|
||||
run: |
|
||||
HOME_WHEEL="$(pwd)/$(ls dist/home/reme_home-[0-9]*.whl)"
|
||||
if python -m pip install --dry-run --ignore-installed "${HOME_WHEEL}" "reme-ai[as]==0.4.2.0" > "${RUNNER_TEMP}/home-incompatible.log" 2>&1; then
|
||||
echo 'Home must not install with the published ReMe 0.4.2.0 wheel.'
|
||||
if python -m pip install --dry-run --ignore-installed "${HOME_WHEEL}" "reme-ai[as]==0.4.2.1" > "${RUNNER_TEMP}/home-incompatible.log" 2>&1; then
|
||||
echo 'Home must not install with the published ReMe 0.4.2.1 wheel.'
|
||||
exit 1
|
||||
fi
|
||||
cat "${RUNNER_TEMP}/home-incompatible.log"
|
||||
|
|
|
|||
|
|
@ -17,6 +17,19 @@ from typing import Any
|
|||
from ._cleanup import finish_cleanup, format_error
|
||||
from .docker_sandbox import DockerReMeSandbox, DockerReMeSandboxFactory
|
||||
from .models import EvaluationQuery, JobRequest
|
||||
from .worker import _exception_details
|
||||
|
||||
|
||||
def _exception_classification(exc: BaseException) -> dict[str, Any]:
|
||||
"""Retain only classification fields, omitting absent status codes and causes."""
|
||||
details = _exception_details(exc)
|
||||
result = {key: details[key] for key in ("type", "module")}
|
||||
status = details.get("status_code")
|
||||
if isinstance(status, int) and not isinstance(status, bool):
|
||||
result["status_code"] = status
|
||||
if details["causes"]:
|
||||
result["causes"] = details["causes"]
|
||||
return result
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -144,7 +157,7 @@ class BenchmarkDockerPool:
|
|||
except Exception as exc:
|
||||
result.error = format_error(exc)
|
||||
result.failure_stage = operation
|
||||
result.exception = {"type": type(exc).__name__, "module": type(exc).__module__}
|
||||
result.exception = _exception_classification(exc)
|
||||
await self._retain_failed_build_log(worker, plan, result)
|
||||
return result
|
||||
accepted = accept_build(result.build) if accept_build else result.build.get("success")
|
||||
|
|
@ -175,7 +188,7 @@ class BenchmarkDockerPool:
|
|||
except Exception as exc: # Keep other cases running and release this worker.
|
||||
result.error = format_error(exc)
|
||||
result.failure_stage = operation
|
||||
result.exception = {"type": type(exc).__name__, "module": type(exc).__module__}
|
||||
result.exception = _exception_classification(exc)
|
||||
return result
|
||||
finally:
|
||||
if result.error and plan.export_diagnostics_to is not None and worker.reusable:
|
||||
|
|
|
|||
|
|
@ -144,15 +144,10 @@ All prepared outputs live under `workspace_dir/home/` (paths below are relative
|
|||
manifest with case IDs and content fingerprints.
|
||||
- `datasets/debug-cases/`: deterministic small smoke cases with source case IDs
|
||||
in their metadata, or an empty `cases/` directory when debug derivation is disabled.
|
||||
- `code/runtime/`: fixed committed ReMe sources, packaging files and configurations,
|
||||
without plugin projects. Uncommitted source edits are excluded.
|
||||
- `code/repo/`: an independent Git repository containing complete selected plugin
|
||||
projects (including packaging and configuration) and any selected ReMe core paths,
|
||||
retaining repository-relative locations. `candidate.paths` defines the editable
|
||||
scope, not the complete file inventory. The initial commit preserves the baseline.
|
||||
Future runners assemble an isolated evaluation tree from runtime plus repo,
|
||||
excluding `.git`; `install_projects` paths are relative to that tree. Install
|
||||
plugins from the candidate, and never edit the fixed runtime.
|
||||
- `code/runtime/` and `code/repo/`: fixed committed runtime and candidate Git
|
||||
repository. See the [workspace and modification boundary](src/reme_home/runtime_docs/architecture-guide.md#workspace-and-modification-boundary)
|
||||
for their roles and editable scope. Source exports exclude uncommitted edits;
|
||||
`install_projects` paths are relative to the assembled evaluation tree.
|
||||
- `docs/architecture-guide.md` and `docs/validation-artifact-guide.md`: guides
|
||||
shipped with the installed Home plugin, covering candidate development and
|
||||
investigation of evaluation artifacts.
|
||||
|
|
@ -236,60 +231,22 @@ For explicit `reuse_construction_from`, the caller must establish construction
|
|||
equivalence. Automatic sampled/budgeted selection instead invokes the host's
|
||||
equivalence reviewer and 60% replay policy. Runs record both commits.
|
||||
|
||||
Results use `home/evaluations/<branch>/<commit-prefix>/<ascending-run-id>/`:
|
||||
|
||||
```text
|
||||
manifest.json
|
||||
summary.json
|
||||
cases/<case-id>/
|
||||
memory_construction/
|
||||
result.json
|
||||
build.log
|
||||
reme_workspace.tar.gz
|
||||
reme_workspace/
|
||||
queries/
|
||||
result.json
|
||||
<query-id>/answer.log
|
||||
```
|
||||
|
||||
Structured scores in query results and summaries use at most four decimal places.
|
||||
Raw judge text and reference answers are preserved.
|
||||
|
||||
The memory tar is exported **before** queries, retains session Git checkpoints,
|
||||
and is independent of the query execution state. Replayed runs keep an
|
||||
independent donor tar and inherited build log without charging construction
|
||||
tokens again. Answers are persisted before host judging. Per-query results keep
|
||||
reference/actual answers, scores, compact judge output, reported answer token usage
|
||||
and failures. `queries/result.json` is the sole authoritative answer record;
|
||||
reconstructable prompts and repeated payloads are removed using the reviewed
|
||||
`dev_meta` rules. Construction results similarly compact redundant links, Job
|
||||
counts, token records and cumulative daily-index snapshots. Summary
|
||||
scores use the expected-query denominator; infrastructure/judge/artifact errors
|
||||
make the comparable mean unavailable while retaining the observed-score mean.
|
||||
The Job returns `success=false` for infrastructure/judge/artifact failures, while
|
||||
preserving its structured answer and result directory; candidate failures remain
|
||||
ordinary evaluation observations.
|
||||
Base jobs preserve structured exception types, HTTP status codes and causes in
|
||||
`Response.metadata["reme.job_error"]`; the Docker worker carries them into failure
|
||||
artifacts. Network/provider failures anywhere in the cause chain invalidate the
|
||||
comparison. Failed jobs without structured error details (including older
|
||||
candidates) are conservatively treated as infrastructure errors, without parsing
|
||||
their error text. Successful jobs returning empty answers still count as candidate
|
||||
failures. The construction failure-rate tolerance applies only to candidate
|
||||
errors: infrastructure or unclassified failures always reject construction, even
|
||||
when another failed job was a candidate error.
|
||||
The manifest retains run identity, selection, mode and execution status. The
|
||||
summary retains only outcome counts, expected/scored query counts, score means,
|
||||
low-score cases, question-type statistics and compact case/query mappings.
|
||||
Candidate answer token summaries average only total tokens per model; detailed
|
||||
construction costs remain in construction results. Temporary post-query archives
|
||||
are discarded after extracting logs; separate execution, checksum and raw-answer
|
||||
JSON copies are not saved. See the
|
||||
[artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md) for the
|
||||
exact fields, rebuild helper and migration from the previous verbose format.
|
||||
Results use `home/evaluations/<branch>/<commit-prefix>/<ascending-run-id>/`.
|
||||
The [artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md#saved-files)
|
||||
is the reference for saved files, authoritative query records, compaction and
|
||||
failure evidence. Its [summary reference](src/reme_home/runtime_docs/validation-artifact-guide.md#summary-fields)
|
||||
covers score denominators, token accounting, debug fields and summary rebuilding.
|
||||
Before comparing runs, verify completed scored execution, infrastructure outcomes,
|
||||
candidate code and complete case/query coverage.
|
||||
|
||||
The Job returns `success=false` for infrastructure/judge/artifact failures while
|
||||
preserving its structured answer and result directory; candidate failures remain
|
||||
ordinary evaluation observations. Successful Jobs returning empty answers count
|
||||
as candidate failures. Construction failure-rate tolerance applies only to
|
||||
candidate errors; infrastructure or unclassified failures always reject construction.
|
||||
See [failure interpretation](src/reme_home/runtime_docs/validation-artifact-guide.md#investigating-a-result)
|
||||
for structured error classification.
|
||||
|
||||
Run/case/query `failure.json` files preserve diagnostics. Interrupted/fail-fast
|
||||
runs publish `partial_summary.json`, not a completed summary. New executions
|
||||
always allocate a new run; they never overwrite previous results. For scored runs, default
|
||||
|
|
@ -305,10 +262,10 @@ Debug requires successful construction and nonempty, error-free answers, and
|
|||
returns `success=false` for candidate failures as well as infrastructure failures.
|
||||
It cannot reuse construction. Case/query selections refer to the selected mode's
|
||||
dataset; omitted selections run all its cases/queries.
|
||||
Debug results retain execution counts and artifacts, but all mean scores are
|
||||
null. `scored_query_count` and `query_status_counts.unscored` are zero, while
|
||||
`expected_query_count` still records the expected execution count. Debug/scored
|
||||
mode is recorded in the manifest.
|
||||
Debug results retain execution counts and artifacts, with null mean scores.
|
||||
See the [summary reference](src/reme_home/runtime_docs/validation-artifact-guide.md#summary-fields)
|
||||
for coverage fields and manifest mode.
|
||||
|
||||
The former `dataset_kind` input is rejected; use `debug` instead. Debug results
|
||||
must not be used to assess answer correctness or candidate quality. No real
|
||||
Docker/model evaluation is required by Home's isolated unit tests.
|
||||
|
|
@ -581,22 +538,14 @@ checksum checks. A later full rebuild considers the full training set again.
|
|||
|
||||
### Successful evaluation history
|
||||
|
||||
`reme_home.evaluation.history.HomeEvaluationHistory` is the dedicated
|
||||
`home.evaluation_history` Component, automatically bound by `HomeEvaluation`.
|
||||
It owns successful-result registration, lookup, deduplication, cache rebuilding
|
||||
and the observations used for the next draw. Steps remain invocation adapters;
|
||||
they do not own cross-evaluation state.
|
||||
|
||||
A successful history entry requires `record_history=true` and a started, completed, scored training-set
|
||||
run with a full summary and no infrastructure errors, following the existing
|
||||
`Response.success` contract. Low or zero quality scores remain valid observations.
|
||||
Classified candidate failures retain the existing response semantics; only their
|
||||
actual numeric scores train EWMA. A run with any infrastructure error is entirely
|
||||
excluded, including scores from its healthy cases. Debug, cancelled, fail-fast
|
||||
and pre-start failures never enter successful history. A full summary is the durable
|
||||
completion boundary: cancelling the caller during subsequent history bookkeeping
|
||||
does not relabel that already-completed evaluation. Failed registration discards
|
||||
only derived caches so they can rebuild from the final source run status.
|
||||
The `home.evaluation_history` Component discovers successful scored training
|
||||
results and rebuilds sampling observations from saved evidence. Low scores remain
|
||||
valid observations; infrastructure failures, debug and incomplete runs are excluded.
|
||||
See [history and sampling evidence](src/reme_home/runtime_docs/validation-artifact-guide.md#evaluation-history-and-adaptive-sampling-evidence)
|
||||
for eligibility, started/successful rounds, saved plans and cache recovery.
|
||||
The full summary is the durable completion boundary: cancellation during later
|
||||
history bookkeeping does not relabel the completed evaluation. Failed registration
|
||||
discards only derived caches, which can rebuild from saved run evidence.
|
||||
|
||||
```python
|
||||
history = app.context.components["home.evaluation_history"]["default"]
|
||||
|
|
@ -605,159 +554,47 @@ result = history.get_result(results[-1].run_key)
|
|||
history.rebuild() # Rebuild sampling observations from saved artifacts.
|
||||
```
|
||||
|
||||
The Component owns the workspace lock for these synchronous filesystem methods;
|
||||
async host code can run them in a worker thread. Successful results are discovered
|
||||
from each run's manifest and summary; no `history.json` index is written or read.
|
||||
Legacy index files are ignored and left untouched. Each returned record points to
|
||||
its run via `run_key`; questions, answers and memory files stay in that result directory.
|
||||
|
||||
The separate startup ledger advances only after the first successfully initialized
|
||||
sandbox, including when a later worker or execution fails. Extra/replacement
|
||||
workers do not count twice. `home/sampling/state.json` distinguishes started
|
||||
`round` from `successful_round`. Only successful history advances the probability
|
||||
clock or changes EWMA; failed runs cannot age cases or alter the next draw's
|
||||
weights. Started plans live under
|
||||
`home/sampling/plans/<branch>/<commit-prefix>/<run-id>.json`. Their `round` is the
|
||||
startup ledger round; `selection_round` records the successful-history snapshot
|
||||
used for an adaptive draw.
|
||||
|
||||
For `record_history=false`, full or explicit evaluation does not access the history
|
||||
manager. Adaptive selection can still read existing observations, while leaving
|
||||
them unchanged; its draw is saved locally as `manifest.selection_plan`, without
|
||||
a startup-ledger round or shared sampling plan. `manifest.options.record_history`
|
||||
persists the requested setting, and the response's `record_history` indicates
|
||||
whether the run participates in history (always false for debug). Legacy manifests
|
||||
without the setting retain the previous history behavior.
|
||||
|
||||
Formal runs remain serialized by `home/metadata/validation.lock`; debug bypasses
|
||||
that lock. Missing caches reconstruct from started plans and successful scored
|
||||
artifacts. Older sampling caches are rebuilt under the success-only rule.
|
||||
`history.rebuild()` rewrites only the catalog and sampling cache. Preserve source
|
||||
manifests, plans and result artifacts when rebuilding; candidate code, datasets
|
||||
and memory files are never modified.
|
||||
These synchronous filesystem methods own the workspace lock; async host code can
|
||||
run them in a worker thread. Formal runs remain serialized by
|
||||
`home/metadata/validation.lock`; debug bypasses that lock. Rebuilding changes only
|
||||
derived state; preserve source manifests, plans and result artifacts.
|
||||
|
||||
## Evaluation result inspection
|
||||
|
||||
Two read-only Jobs navigate saved evaluations independently of successful
|
||||
optimization history. They include debug, failed, cancelled, running and standalone
|
||||
(`record_history=false`) runs. Reads never register results, update sampling,
|
||||
rebuild summaries or write artifacts.
|
||||
Read-only inspection includes debug, failed, cancelled, running and standalone
|
||||
(`record_history=false`) runs, independently of successful optimization history.
|
||||
It never registers results, updates sampling, rebuilds summaries or writes artifacts.
|
||||
|
||||
`home_list_validation_results` accepts optional `branch_name` and `commit_sha`
|
||||
filters plus `offset=0`, `limit=20` (1–100). Without filters it lists all runs,
|
||||
newest first, with a stable path tie-breaker. It returns `row_count`, `rows` and
|
||||
`next_offset`, compact identity/options, saved summary counts and scores, failure
|
||||
previews and artifact paths. A malformed run is reported as a `read_error` row
|
||||
without stopping the unfiltered catalog.
|
||||
|
||||
`commit_sha` accepts a full SHA or hexadecimal prefix. A prefix must match exactly
|
||||
one of **all commits reachable from the selected local branch**, including commits
|
||||
that have never been evaluated. Omitting `branch_name` with a commit filter uses
|
||||
the current candidate branch. Missing or ambiguous matches fail explicitly; the
|
||||
response includes the resolved branch and full SHA. A branch-only filter can also
|
||||
browse historical runs from a deleted branch.
|
||||
|
||||
`home_inspect_validation` requires `validation_path` (absolute or relative to
|
||||
`home/`) and accepts optional `case_id`, `query_id`, `offset=0` and `limit=20`:
|
||||
|
||||
| Selection | Returned rows |
|
||||
| Job | Use |
|
||||
| --- | --- |
|
||||
| Run only | Case summaries, missing scores first, then low scores. |
|
||||
| Run and case | Query summaries in manifest order, plus independently paged failed construction Jobs. |
|
||||
| Run, case and query | One query's question, answer, rubric, score and judge/error previews. |
|
||||
|
||||
`query_id` requires `case_id`; both must be selected by the run manifest. Cases or
|
||||
queries without published evidence are explicitly marked `missing`, with null
|
||||
scores. The tool uses a saved full summary, or an existing partial summary when
|
||||
the full summary is absent. Missing summaries are reported without reconstruction.
|
||||
Errors are previewed to 600 characters, and selected query fields to 2,000 each;
|
||||
truncated fields carry a marker. `rows_source`, JSON `pointer` values, and log and
|
||||
memory paths identify the original evidence. Failed construction Jobs have their
|
||||
own `row_count`, `rows`, `next_offset` under `construction.failed_jobs`; their
|
||||
pagination is independent of the query rows, using the supplied offset/limit.
|
||||
| `home_list_validation_results` | Find runs, optionally filtered by `branch_name` and `commit_sha`. |
|
||||
| `home_inspect_validation` | Inspect a `validation_path`, then narrow with `case_id` and `query_id`. |
|
||||
| `home_trace_query_evidence` | Link a query to source sessions, memory checkpoints and saved evidence without a model call. |
|
||||
| `home_inspect_memory_history` | Page through one case's exported memory Git checkpoints. |
|
||||
| `home_compare_memory` | Compare committed memory within a run or across evaluations of the same case. |
|
||||
|
||||
With the configured service running, use a second terminal:
|
||||
|
||||
```bash
|
||||
reme home_list_validation_results branch_name=candidate commit_sha=abc123 limit=10
|
||||
reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1
|
||||
reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1 case_id=case-a query_id=query-a
|
||||
```
|
||||
|
||||
Host callers use the same Jobs through `app.run_job()`. For a case-scoped agent,
|
||||
inject `validation_path` and `case_id` with `injected_job_kwargs`; the wrapper hides
|
||||
and enforces these values. Both Jobs reject path escapes and symlinks, and fail
|
||||
closed for restricted `_allowed_paths` contexts, like the other Home Jobs.
|
||||
|
||||
### Memory checkpoint inspection
|
||||
|
||||
Read-only Jobs investigate a case's exported memory Git repository. Locate
|
||||
an evaluation and case with `home_list_validation_results` and
|
||||
`home_inspect_validation` first. These Jobs never change artifacts, Git refs, the
|
||||
index, evaluation history or sampling state.
|
||||
|
||||
- `home_trace_query_evidence(validation_path, case_id, query_id,
|
||||
evidence_session_ids=null)` joins a selected query to its source sessions and
|
||||
memory checkpoints without calling a model. Dataset evidence annotations take
|
||||
precedence over optional inferred session IDs, including an empty annotation.
|
||||
Returns `query` (question, expected/actual answer, score, error and
|
||||
`result_missing` and `expected_kind`), `locations` with JSON Pointers into source
|
||||
and saved query files, optional `criteria` previews of saved rubric judgments,
|
||||
`evidence_source` (`independent_annotation`,
|
||||
`diagnostician_inference`, or `unavailable`), `evidence_is_certain`,
|
||||
`memory_status`, `sessions` (ID, source JSON Pointer, checkpoint and changed files), and `paths` to
|
||||
the normalized case, memory repository, query results and query artifacts.
|
||||
A null checkpoint means no matching `session: <id>` commit was found; missing
|
||||
results keep scores null. Answer/question previews are limited to 2,000
|
||||
characters and errors to 600. At most 50 sessions and 50 changed files per
|
||||
session are returned, with omitted counts when needed. Criterion previews are
|
||||
limited to 20 entries and 600 characters per requirement/reason; absent or
|
||||
mismatched rubric judgments are not paired. Session text, diffs,
|
||||
later history and full answer/judge results remain at the returned paths or
|
||||
in the inspection/comparison Jobs below. Original transcripts and derived
|
||||
memories can both be searchable; these coordinates alone do not prove a cause.
|
||||
- `home_inspect_memory_history(validation_path, case_id, path=null, offset=0,
|
||||
limit=20)` returns `status`, `workspace_exists`, `has_git_history`, `head`,
|
||||
`repository`, and newest-first checkpoint `rows` with `row_count` and
|
||||
`next_offset`. Limits are 1–100. Each row includes the full commit SHA, parents,
|
||||
timestamp, subject (up to 2,000 characters) and a `session_id` when the subject
|
||||
follows `session: <id>`. `path` follows a literal workspace-relative file or
|
||||
directory. Missing exports, absent Git history and empty repositories have
|
||||
distinct statuses; corrupt repositories fail explicitly.
|
||||
- `home_compare_memory(validation_path, case_id, base_ref=null,
|
||||
target_ref="HEAD", target_validation_path=null, path=null)` compares two memory
|
||||
commits. Within one evaluation, `base_ref` is required. For another evaluation,
|
||||
supply `target_validation_path`; both sides then default to `HEAD`. The case
|
||||
must be selected in both manifests. Returns resolved commits, candidate commit
|
||||
identities, manifest fingerprints, added/removed/modified paths, content-tree
|
||||
hashes, a unified `diff`, and `truncated`.
|
||||
|
||||
References accept HEAD-relative expressions and hexadecimal SHA prefixes unique
|
||||
among commits reachable from that memory repository's HEAD. Resolved expressions
|
||||
must also be HEAD-reachable. This is independent of candidate-code branch history.
|
||||
Comparisons read committed regular files from Git objects, ignoring dirty or
|
||||
untracked checkout contents. File mode changes alone are not content differences.
|
||||
Binary changes are identified without printing their bytes. Each snapshot is
|
||||
limited to 2,000 files, 20 MB total and 5 MB per file; diff output is limited to
|
||||
100,000 characters. Narrow `path` if a limit is reached. Tree hashes describe the
|
||||
selected path scope. Linked Git storage, alternates, shallow/partial histories and
|
||||
symbolic links in Git metadata are rejected.
|
||||
|
||||
After starting the service with the Home configuration, call from another terminal:
|
||||
|
||||
```bash
|
||||
reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1
|
||||
reme home_list_validation_results branch_name=candidate limit=10
|
||||
reme home_inspect_validation validation_path=evaluations/candidate/abcdef0/1
|
||||
reme home_inspect_validation validation_path=evaluations/candidate/abcdef0/1 case_id=case1 query_id=query1
|
||||
reme home_trace_query_evidence validation_path=evaluations/candidate/abcdef0/1 case_id=case1 query_id=query1
|
||||
reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1
|
||||
reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 case_id=case1 base_ref=HEAD~1
|
||||
reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 target_validation_path=evaluations/candidate/1234567/1 case_id=case1
|
||||
```
|
||||
|
||||
`validation_path` is absolute or relative to `home/`; for example the relative
|
||||
form is `evaluations/candidate/abcdef0/1`. For agent tools, host `injected_job_kwargs` can pin
|
||||
`validation_path` and `case_id`; the wrapper hides injected fields and rejects
|
||||
model overrides. Like the other Home inspection Jobs, these Jobs reject
|
||||
`_allowed_paths` restrictions rather than bypassing them. Use these tools directly
|
||||
or through the case-study batch Job below.
|
||||
`validation_path` is absolute or relative to `home/`. See
|
||||
[read-only evaluation inspection](src/reme_home/runtime_docs/validation-artifact-guide.md#read-only-evaluation-inspection)
|
||||
for pagination, commit-prefix resolution, previews and source pointers, and
|
||||
[memory checkpoint tools](src/reme_home/runtime_docs/validation-artifact-guide.md#memory-checkpoint-tools)
|
||||
for evidence provenance, missing-history states, comparison references and limits.
|
||||
|
||||
Host callers use the same Jobs through `app.run_job()`. For case-scoped agent
|
||||
tools, pin `validation_path` and `case_id` through `injected_job_kwargs`;
|
||||
the wrapper hides those fields and rejects model overrides. These Jobs reject
|
||||
path escapes, symlinks and restricted `_allowed_paths` contexts.
|
||||
|
||||
## Case-study batches
|
||||
|
||||
|
|
@ -794,19 +631,11 @@ available. No case scratch directory or intermediate analysis files are created.
|
|||
still return a text report with per-case metadata. Do not supply both path names
|
||||
or both case selectors. Other inspection Jobs retain their `validation_path` argument.
|
||||
|
||||
The toolkit includes validation inspection, compact query-evidence tracing,
|
||||
memory history/comparison, and the three committed-code navigation Jobs.
|
||||
Validation/case selectors and the evaluated code revision are hidden and enforced
|
||||
by the host; memory comparison stays within the selected evaluation. General
|
||||
Read/Grep/Glob, Write/Edit, validation and code-mutation tools are not exposed.
|
||||
The case-study toolkit and mandatory read-only OS sandbox are described in the
|
||||
[artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md#single-case-diagnostic-agent).
|
||||
The host pins case/evaluation selectors and code revision. Direct code reads can
|
||||
see current worktree changes; use revision-pinned Jobs to establish evaluated behavior.
|
||||
|
||||
`AutoBash` and `python_execute` share a required OS sandbox. Commands have no
|
||||
static whitelist (`is_read_only=false`); filesystem permissions enforce access.
|
||||
Read-only roots are the selected case artifact directory, normalized source-case
|
||||
JSON, validation manifest, all of `home/code/`, and `home/docs/`. No host writable
|
||||
roots are granted. System/interpreter runtime paths and disposable backend
|
||||
temporary storage remain available. Direct code reads can see current worktree
|
||||
changes; use the revision-pinned Jobs to establish the evaluated behavior.
|
||||
`python_execute` accepts either `code` or an existing `file`, optional file-only
|
||||
`args`, and a `timeout` in seconds (default 60, maximum 600). It uses `python -I -B`
|
||||
and returns returncode/stdout/stderr/timed_out. Both executors reuse bounded output,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ description = "Self-evolution workspace preparation and local command execution
|
|||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"reme-ai[as]>=0.4.2.1",
|
||||
"reme-ai[as]>=0.4.2.2",
|
||||
"safecmd>=0.1.16,<0.2",
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ class TimeBudgetReminder:
|
|||
class RunTimeBudgetTimer:
|
||||
"""Track one agent run and emit advisory reminders without stopping it."""
|
||||
|
||||
reminder_thresholds = (0.90, 0.75, 0.60)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
time_budget_minutes: float,
|
||||
|
|
@ -89,7 +91,7 @@ class RunTimeBudgetTimer:
|
|||
if elapsed_ratio >= 1.0:
|
||||
reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10
|
||||
else:
|
||||
for threshold in (0.90, 0.75, 0.60):
|
||||
for threshold in self.reminder_thresholds:
|
||||
if elapsed_ratio >= threshold:
|
||||
reached_ratio = threshold
|
||||
break
|
||||
|
|
@ -98,17 +100,21 @@ class RunTimeBudgetTimer:
|
|||
self._last_reminded_ratio = reached_ratio
|
||||
persistent = self._next_reminder_is_persistent
|
||||
self._next_reminder_is_persistent = not persistent
|
||||
return TimeBudgetReminder(
|
||||
content=self._reminder_content(elapsed_minutes, reached_ratio),
|
||||
persistent=persistent,
|
||||
)
|
||||
|
||||
def _reminder_content(self, elapsed_minutes: float, reached_ratio: float) -> str:
|
||||
"""Describe this run without changing the shared reminder schedule."""
|
||||
elapsed = _format_minutes(elapsed_minutes)
|
||||
percentage = round(reached_ratio * 100)
|
||||
return TimeBudgetReminder(
|
||||
content=(
|
||||
f"<system-reminder>The {self.agent_label} run has used {elapsed} ({percentage}% of its "
|
||||
f"{_format_minutes(self.time_budget_minutes)} soft time budget). "
|
||||
f"The run has completed {self.context_compression_count} context compressions. "
|
||||
"This reminder does not terminate the run. Prioritize the highest-value remaining investigation and "
|
||||
"produce the required final report as soon as the evidence is sufficient.</system-reminder>"
|
||||
),
|
||||
persistent=persistent,
|
||||
return (
|
||||
f"<system-reminder>The {self.agent_label} run has used {elapsed} ({percentage}% of its "
|
||||
f"{_format_minutes(self.time_budget_minutes)} soft time budget). "
|
||||
f"The run has completed {self.context_compression_count} context compressions. "
|
||||
"This reminder does not terminate the run. Prioritize the highest-value remaining investigation and "
|
||||
"produce the required final report as soon as the evidence is sufficient.</system-reminder>"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,12 @@
|
|||
"""Early candidate admission using read-only, commit-pinned source exports."""
|
||||
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Literal
|
||||
from uuid import uuid4
|
||||
|
||||
from agentscope.tool import Glob, Grep, Read, Toolkit
|
||||
from pydantic import Field
|
||||
import yaml
|
||||
|
||||
from ..auto_bash import AutoBash
|
||||
from ..code.repository import current_branch, require_clean, resolve_revision, validate_repository, validate_tree
|
||||
from ..code.snapshot import export_blob, tree
|
||||
from ..evaluation.artifacts import contained, read_json
|
||||
|
|
@ -20,7 +14,7 @@ from ..evaluation.component import blocking
|
|||
from ..evaluation.repetition import previous_candidate_result
|
||||
from ..preparation.code import git
|
||||
from ..preparation.spec import SpecModel
|
||||
from .workflow_common import WorkflowSandbox, save_record
|
||||
from .workflow_common import review_report
|
||||
|
||||
|
||||
class CandidateGateReport(SpecModel):
|
||||
|
|
@ -92,51 +86,18 @@ async def review_candidate(wrapper, home, context, budget, *, memory_limit_mib=N
|
|||
"""Fail closed on sandbox setup; model/report failures allow validation as in dev_meta."""
|
||||
async with review_export(home, "candidate-gate") as root:
|
||||
scope = await blocking(prepare_review_scope, home, context, root=root)
|
||||
return await _review_candidate(wrapper, home, context, budget, scope, memory_limit_mib)
|
||||
|
||||
|
||||
async def _review_candidate(wrapper, home, context, budget, scope, memory_limit_mib):
|
||||
"""Run inside the caller-owned export lifetime; persist only the report."""
|
||||
backend = WorkflowSandbox(scope, memory_limit_mib)
|
||||
probe = await backend.exec_shell([sys.executable, "-I", "-B", "-c", "print('candidate gate sandbox')"], timeout=10)
|
||||
if not probe.ok():
|
||||
raise RuntimeError("Candidate gate sandbox unavailable: " + probe.stderr.decode(errors="replace"))
|
||||
tools = [
|
||||
Read(backend=backend),
|
||||
Glob(backend=backend),
|
||||
Grep(backend=backend),
|
||||
AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=True),
|
||||
]
|
||||
for tool in tools:
|
||||
tool.is_concurrency_safe = False
|
||||
prompt = yaml.safe_load(Path(__file__).with_suffix(".yaml").read_text(encoding="utf-8"))
|
||||
report = {**context, "validation_budget": budget, "source_path": scope["scratch"]}
|
||||
try:
|
||||
reply = await wrapper.reply(
|
||||
return await review_report(
|
||||
wrapper,
|
||||
home,
|
||||
scope,
|
||||
f"Review admission from {context['base_commit']} to {context['candidate_commit']}.\n"
|
||||
f"Source exports and complete diff.patch: {scope['scratch']}",
|
||||
system_prompt=prompt["system_prompt"],
|
||||
toolkit=Toolkit(tools=tools),
|
||||
builtin_tools=[],
|
||||
job_tools=[],
|
||||
skills=[],
|
||||
resume=None,
|
||||
session_id=None,
|
||||
fork_session=False,
|
||||
output_schema=None,
|
||||
_structured_report_model=CandidateGateReport,
|
||||
permission_mode="bypass",
|
||||
{**context, "validation_budget": budget, "source_path": scope["scratch"]},
|
||||
role="candidate_gate",
|
||||
sandbox_label="Candidate gate",
|
||||
category="candidate-gates",
|
||||
report_model=CandidateGateReport,
|
||||
failure_decision="accept",
|
||||
failure_reason="Candidate gate failed; validation is allowed",
|
||||
memory_limit_mib=memory_limit_mib,
|
||||
)
|
||||
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})
|
||||
text = reply["result"].strip()
|
||||
if text.startswith("```") and text.endswith("```"):
|
||||
text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
|
||||
report.update(CandidateGateReport.model_validate(json.loads(text)).model_dump())
|
||||
except Exception as exc:
|
||||
report.update(
|
||||
decision="accept",
|
||||
reason=f"Candidate gate failed; validation is allowed: {type(exc).__name__}: {exc}",
|
||||
review_failed=True,
|
||||
)
|
||||
path = await blocking(save_record, home, "candidate-gates", report)
|
||||
return {**report, "review_path": path}
|
||||
|
|
|
|||
|
|
@ -1,23 +1,17 @@
|
|||
"""Construction equivalence review over isolated source and construction exports."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Literal
|
||||
|
||||
from agentscope.tool import Glob, Grep, Read, Toolkit
|
||||
from pydantic import Field
|
||||
import yaml
|
||||
|
||||
from ..auto_bash import AutoBash
|
||||
from ..evaluation.artifacts import contained, file_sha256, inspect_archive, read_json, write_json
|
||||
from ..evaluation.component import blocking
|
||||
from ..evaluation.inputs import reuse_inputs
|
||||
from ..preparation.code import tree_fingerprint
|
||||
from ..preparation.spec import SpecModel
|
||||
from .candidate_gate import prepare_review_scope, review_export
|
||||
from .workflow_common import WorkflowSandbox, save_record
|
||||
from .workflow_common import review_report
|
||||
|
||||
|
||||
class ConstructionEquivalenceReport(SpecModel):
|
||||
|
|
@ -65,54 +59,18 @@ async def review_construction(wrapper, home, context, spec, cases, fingerprints,
|
|||
"""Stop on sandbox setup errors; review errors conservatively require rebuilding."""
|
||||
async with review_export(home, "construction-equivalence") as root:
|
||||
scope = await blocking(prepare_equivalence_scope, home, context, spec, cases, fingerprints, root=root)
|
||||
return await _review_construction(wrapper, home, context, scope, memory_limit_mib)
|
||||
|
||||
|
||||
async def _review_construction(wrapper, home, context, scope, memory_limit_mib):
|
||||
"""Run inside the caller-owned export lifetime; persist only the report."""
|
||||
backend = WorkflowSandbox(scope, memory_limit_mib)
|
||||
probe = await backend.exec_shell(
|
||||
[sys.executable, "-I", "-B", "-c", "print('construction review sandbox')"],
|
||||
timeout=10,
|
||||
)
|
||||
if not probe.ok():
|
||||
raise RuntimeError("Construction review sandbox unavailable: " + probe.stderr.decode(errors="replace"))
|
||||
tools = [
|
||||
Read(backend=backend),
|
||||
Glob(backend=backend),
|
||||
Grep(backend=backend),
|
||||
AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=True),
|
||||
]
|
||||
for tool in tools:
|
||||
tool.is_concurrency_safe = False
|
||||
prompt = yaml.safe_load(Path(__file__).with_suffix(".yaml").read_text(encoding="utf-8"))
|
||||
report = {**context, "source_path": scope["scratch"]}
|
||||
try:
|
||||
reply = await wrapper.reply(
|
||||
return await review_report(
|
||||
wrapper,
|
||||
home,
|
||||
scope,
|
||||
f"Compare construction from {context['base_commit']} to {context['candidate_commit']}.\n"
|
||||
f"Read-only source, diff and construction evidence: {scope['scratch']}",
|
||||
system_prompt=prompt["system_prompt"],
|
||||
toolkit=Toolkit(tools=tools),
|
||||
builtin_tools=[],
|
||||
job_tools=[],
|
||||
skills=[],
|
||||
resume=None,
|
||||
session_id=None,
|
||||
fork_session=False,
|
||||
output_schema=None,
|
||||
_structured_report_model=ConstructionEquivalenceReport,
|
||||
permission_mode="bypass",
|
||||
{**context, "source_path": scope["scratch"]},
|
||||
role="construction_equivalence",
|
||||
sandbox_label="Construction review",
|
||||
category="construction-equivalence",
|
||||
report_model=ConstructionEquivalenceReport,
|
||||
failure_decision="construction_changed",
|
||||
failure_reason="Equivalence review failed; rebuild construction",
|
||||
memory_limit_mib=memory_limit_mib,
|
||||
)
|
||||
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})
|
||||
text = reply["result"].strip()
|
||||
if text.startswith("```") and text.endswith("```"):
|
||||
text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
|
||||
report.update(ConstructionEquivalenceReport.model_validate(json.loads(text)).model_dump())
|
||||
except Exception as exc:
|
||||
report.update(
|
||||
decision="construction_changed",
|
||||
reason=f"Equivalence review failed; rebuild construction: {type(exc).__name__}: {exc}",
|
||||
review_failed=True,
|
||||
)
|
||||
path = await blocking(save_record, home, "construction-equivalence", report)
|
||||
return {**report, "review_path": path}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
|
@ -23,7 +22,7 @@ from ..evaluation.budget import validate_max_evaluations, validation_budget_stat
|
|||
from ..preparation.spec import SpecModel
|
||||
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
|
||||
from .diagnostic import diagnostic_tool, evaluation_paths
|
||||
from .bounded import TimeBudgetReminder, _format_minutes, _remove_hint_by_identity
|
||||
from .bounded import RunTimeBudgetTimer, _format_minutes, _remove_hint_by_identity
|
||||
from .compression import RecoverableCompressionAgent
|
||||
from .workflow_common import required_job, save_record, workflow_reply
|
||||
from .wrapper import CaseStudyAgentWrapper
|
||||
|
|
@ -37,55 +36,23 @@ MUTATION_JOBS = (
|
|||
PENDING_CYCLE_TRANSITIONS = "home.optimizer.pending_cycle_transitions"
|
||||
|
||||
|
||||
class ValidationCycleTimer:
|
||||
class ValidationCycleTimer(RunTimeBudgetTimer):
|
||||
"""Emit escalating reminders as a scored-validation cycle ages."""
|
||||
|
||||
reminder_thresholds = (0.90, 0.75, 0.50)
|
||||
|
||||
def __init__(self, time_budget_minutes: float, *, clock: Callable[[], float] = time.monotonic) -> None:
|
||||
if (
|
||||
not isinstance(time_budget_minutes, (int, float))
|
||||
or isinstance(time_budget_minutes, bool)
|
||||
or not math.isfinite(time_budget_minutes)
|
||||
or time_budget_minutes <= 0
|
||||
):
|
||||
raise ValueError("time_budget_minutes must be a positive number")
|
||||
self.time_budget_minutes = float(time_budget_minutes)
|
||||
self._clock = clock
|
||||
self._started_at = self._clock()
|
||||
self._last_reminded_ratio = 0.0
|
||||
self._next_reminder_is_persistent = False
|
||||
self.context_compression_count = 0
|
||||
super().__init__(time_budget_minutes, agent_label="optimizer", clock=clock)
|
||||
self.start()
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Start a fresh cycle after a scored validation finishes."""
|
||||
|
||||
self._started_at = self._clock()
|
||||
self._last_reminded_ratio = 0.0
|
||||
self._next_reminder_is_persistent = False
|
||||
self.context_compression_count = 0
|
||||
self.start()
|
||||
|
||||
def record_context_compression(self) -> None:
|
||||
"""Record one completed compression since the last scored validation."""
|
||||
|
||||
self.context_compression_count += 1
|
||||
|
||||
def next_reminder(self) -> TimeBudgetReminder | None:
|
||||
"""Return one reminder for the latest newly reached deadline."""
|
||||
|
||||
elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0)
|
||||
def _reminder_content(self, elapsed_minutes: float, reached_ratio: float) -> str:
|
||||
"""Keep optimizer guidance tied to the last scored validation."""
|
||||
elapsed_ratio = elapsed_minutes / self.time_budget_minutes
|
||||
reached_ratio: float | None = None
|
||||
if elapsed_ratio >= 1.0:
|
||||
reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10
|
||||
else:
|
||||
for threshold in (0.9, 0.75, 0.5):
|
||||
if elapsed_ratio >= threshold:
|
||||
reached_ratio = threshold
|
||||
break
|
||||
if reached_ratio is None or reached_ratio <= self._last_reminded_ratio:
|
||||
return None
|
||||
self._last_reminded_ratio = reached_ratio
|
||||
persistent = self._next_reminder_is_persistent
|
||||
self._next_reminder_is_persistent = not persistent
|
||||
elapsed = _format_minutes(elapsed_minutes)
|
||||
percentage = round(reached_ratio * 100)
|
||||
timing = (
|
||||
|
|
@ -97,22 +64,16 @@ class ValidationCycleTimer:
|
|||
"scored validation. "
|
||||
)
|
||||
if elapsed_ratio >= 1.0:
|
||||
return TimeBudgetReminder(
|
||||
content=(
|
||||
f"<system-reminder>{timing}{compression}"
|
||||
"Prioritize completing the code changes, committing them, passing the smoke test, and starting "
|
||||
"validation as soon as possible.</system-reminder>"
|
||||
),
|
||||
persistent=persistent,
|
||||
return (
|
||||
f"<system-reminder>{timing}{compression}"
|
||||
"Prioritize completing the code changes, committing them, passing the smoke test, and starting "
|
||||
"validation as soon as possible.</system-reminder>"
|
||||
)
|
||||
remaining = _format_minutes(self.time_budget_minutes - elapsed_minutes)
|
||||
return TimeBudgetReminder(
|
||||
content=(
|
||||
f"<system-reminder>{timing}{compression}"
|
||||
f"Complete the code changes, commit them, pass the smoke test, and start validation within {remaining}."
|
||||
"</system-reminder>"
|
||||
),
|
||||
persistent=persistent,
|
||||
return (
|
||||
f"<system-reminder>{timing}{compression}"
|
||||
f"Complete the code changes, commit them, pass the smoke test, and start validation within {remaining}."
|
||||
"</system-reminder>"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -150,6 +150,72 @@ def save_record(home, category, payload):
|
|||
return str(path)
|
||||
|
||||
|
||||
async def review_report(
|
||||
wrapper,
|
||||
home,
|
||||
scope,
|
||||
task,
|
||||
report,
|
||||
*,
|
||||
role,
|
||||
sandbox_label,
|
||||
category,
|
||||
report_model,
|
||||
failure_decision,
|
||||
failure_reason,
|
||||
memory_limit_mib=None,
|
||||
):
|
||||
"""Share read-only review mechanics while callers choose their failure policy.
|
||||
|
||||
Sandbox and prompt setup and report persistence deliberately stay outside
|
||||
the model/report recovery boundary. Cancellation propagates to the export owner.
|
||||
"""
|
||||
backend = WorkflowSandbox(scope, memory_limit_mib)
|
||||
probe = await backend.exec_shell(
|
||||
[sys.executable, "-I", "-B", "-c", f"print('{sandbox_label.lower()} sandbox')"],
|
||||
timeout=10,
|
||||
)
|
||||
if not probe.ok():
|
||||
raise RuntimeError(f"{sandbox_label} sandbox unavailable: " + probe.stderr.decode(errors="replace"))
|
||||
tools = [
|
||||
Read(backend=backend),
|
||||
Glob(backend=backend),
|
||||
Grep(backend=backend),
|
||||
AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=True),
|
||||
]
|
||||
for tool in tools:
|
||||
tool.is_concurrency_safe = False
|
||||
prompt = yaml.safe_load(Path(__file__).with_name(f"{role}.yaml").read_text(encoding="utf-8"))
|
||||
try:
|
||||
reply = await wrapper.reply(
|
||||
task,
|
||||
system_prompt=prompt["system_prompt"],
|
||||
toolkit=Toolkit(tools=tools),
|
||||
builtin_tools=[],
|
||||
job_tools=[],
|
||||
skills=[],
|
||||
resume=None,
|
||||
session_id=None,
|
||||
fork_session=False,
|
||||
output_schema=None,
|
||||
_structured_report_model=report_model,
|
||||
permission_mode="bypass",
|
||||
)
|
||||
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})
|
||||
text = reply["result"].strip()
|
||||
if text.startswith("```") and text.endswith("```"):
|
||||
text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
|
||||
report.update(report_model.model_validate(json.loads(text)).model_dump())
|
||||
except Exception as exc:
|
||||
report.update(
|
||||
decision=failure_decision,
|
||||
reason=f"{failure_reason}: {type(exc).__name__}: {exc}",
|
||||
review_failed=True,
|
||||
)
|
||||
path = await blocking(save_record, home, category, report)
|
||||
return {**report, "review_path": path}
|
||||
|
||||
|
||||
_INSPECT = """\
|
||||
import json, pathlib, sys
|
||||
path = pathlib.Path(sys.argv[2])
|
||||
|
|
|
|||
|
|
@ -331,10 +331,12 @@ def rebuild_summary(run: Path) -> dict:
|
|||
INFRA_TYPES = {
|
||||
"APIConnectionError",
|
||||
"APITimeoutError",
|
||||
"AuthenticationError",
|
||||
"ConnectError",
|
||||
"ConnectionError",
|
||||
"ConnectTimeout",
|
||||
"NetworkError",
|
||||
"PermissionDeniedError",
|
||||
"PoolTimeout",
|
||||
"RateLimitError",
|
||||
"ReadError",
|
||||
|
|
@ -350,7 +352,7 @@ INFRA_TYPES = {
|
|||
|
||||
|
||||
def failure_kind(exception, stage: str) -> str:
|
||||
"""Only candidate execution errors count against candidate scores."""
|
||||
"""Only candidate execution errors count against scores; provider access errors are infrastructure failures."""
|
||||
if stage not in {"construction", "answer"}:
|
||||
return "infra_error"
|
||||
exception = exception if isinstance(exception, dict) else {}
|
||||
|
|
@ -360,7 +362,7 @@ def failure_kind(exception, stage: str) -> str:
|
|||
if isinstance(causes, list) and any(failure_kind(cause, stage) == "infra_error" for cause in causes):
|
||||
return "infra_error"
|
||||
status = exception.get("status_code")
|
||||
if isinstance(status, int) and (status in {408, 429} or status >= 500):
|
||||
if isinstance(status, int) and (status in {401, 403, 408, 429} or status >= 500):
|
||||
return "infra_error"
|
||||
name = str(exception.get("type", ""))
|
||||
module = str(exception.get("module", ""))
|
||||
|
|
|
|||
|
|
@ -235,8 +235,12 @@ class HomeEvaluation(BaseComponent):
|
|||
manifest["selection_plan"] = plan
|
||||
write_json(run / "manifest.json", manifest)
|
||||
return
|
||||
self.history.record_started(run, policy if policy is not None else self.sampling_policy, plan)
|
||||
manifest.update(read_json(run / "manifest.json"))
|
||||
try:
|
||||
self.history.record_started(run, policy if policy is not None else self.sampling_policy, plan)
|
||||
finally:
|
||||
# Cache publication can fail after the durable start was written.
|
||||
# Preserve those fields when the caller later saves failure status.
|
||||
manifest.update(read_json(run / "manifest.json"))
|
||||
|
||||
def _finish_sampling(self, run):
|
||||
self.history.record(run)
|
||||
|
|
|
|||
|
|
@ -114,35 +114,45 @@ def start_validation(home: Path, run: Path, policy: SamplingPolicy, plan: dict |
|
|||
return {}
|
||||
key = run.relative_to(home / "evaluations").as_posix()
|
||||
state = read_state(home)
|
||||
if key in state["started_validations"]:
|
||||
return read_json(contained(home, manifest["sampling"]["plan_path"]))
|
||||
plan = (
|
||||
dict(plan)
|
||||
if plan is not None
|
||||
else {
|
||||
"mode": "explicit",
|
||||
"case_ids": manifest["case_ids"],
|
||||
"query_ids": manifest["query_ids"],
|
||||
"policy": policy.model_dump(),
|
||||
}
|
||||
)
|
||||
# Preserve the draw's state snapshot separately from the committed start round.
|
||||
if "round" in plan:
|
||||
plan["selection_round"] = plan["round"]
|
||||
plan.update(
|
||||
round=state["round"] + 1,
|
||||
validation_id=manifest["validation_id"],
|
||||
validation_key=key,
|
||||
sandbox_started_at=utc_now().isoformat(),
|
||||
)
|
||||
relative = f"sampling/plans/{key}.json"
|
||||
write_json(contained(home, relative), plan)
|
||||
if key in state["started_validations"]:
|
||||
plan = read_json(contained(home, relative))
|
||||
else:
|
||||
plan = (
|
||||
dict(plan)
|
||||
if plan is not None
|
||||
else {
|
||||
"mode": "explicit",
|
||||
"case_ids": manifest["case_ids"],
|
||||
"query_ids": manifest["query_ids"],
|
||||
"policy": policy.model_dump(),
|
||||
}
|
||||
)
|
||||
# Preserve the draw's state snapshot separately from the committed start round.
|
||||
if "round" in plan:
|
||||
plan["selection_round"] = plan["round"]
|
||||
plan.update(
|
||||
round=state["round"] + 1,
|
||||
validation_id=manifest["validation_id"],
|
||||
validation_key=key,
|
||||
sandbox_started_at=utc_now().isoformat(),
|
||||
)
|
||||
# Invalidate before publishing evidence: interruption at either following
|
||||
# write must rebuild from the started plan rather than trust an old cache.
|
||||
cache = contained(home, "sampling/state.json")
|
||||
cache.unlink(missing_ok=True)
|
||||
if key not in state["started_validations"]:
|
||||
write_json(contained(home, relative), plan)
|
||||
manifest["sampling"] = {"round": plan["round"], "mode": plan["mode"], "plan_path": relative}
|
||||
manifest["sandbox_started_at"] = plan["sandbox_started_at"]
|
||||
write_json(run / "manifest.json", manifest)
|
||||
state["round"] = plan["round"]
|
||||
state["round"] = max(state["round"], plan["round"])
|
||||
state["started_validations"][key] = plan["round"]
|
||||
write_json(contained(home, "sampling/state.json"), state)
|
||||
try:
|
||||
write_json(cache, state)
|
||||
except BaseException:
|
||||
cache.unlink(missing_ok=True)
|
||||
raise
|
||||
return plan
|
||||
|
||||
|
||||
|
|
@ -231,10 +241,17 @@ def rebuild_sampling_state(home: Path, *, persist: bool = True) -> dict:
|
|||
manifest.get("mode") == "debug"
|
||||
or manifest.get("dataset_kind") != "training-set"
|
||||
or manifest.get("options", {}).get("record_history") is False
|
||||
or not (manifest.get("sandbox_started_at"))
|
||||
):
|
||||
continue
|
||||
plan = read_json(contained(home, manifest["sampling"]["plan_path"]))
|
||||
key = path.parent.relative_to(home / "evaluations").as_posix()
|
||||
plan_path = contained(home, manifest.get("sampling", {}).get("plan_path", f"sampling/plans/{key}.json"))
|
||||
if not manifest.get("sandbox_started_at") and not plan_path.is_file():
|
||||
continue
|
||||
plan = read_json(plan_path)
|
||||
# A started plan is independent evidence even if the process stopped
|
||||
# before publishing the manifest's start fields.
|
||||
if not plan.get("sandbox_started_at"):
|
||||
continue
|
||||
runs.append((plan["round"], path.parent, plan["validation_key"]))
|
||||
for round_id, run, key in sorted(runs):
|
||||
state["round"] = max(state["round"], round_id)
|
||||
|
|
|
|||
|
|
@ -86,12 +86,12 @@ application_defaults:
|
|||
thinking_enable: false
|
||||
agent_wrapper:
|
||||
home_construction_equivalence:
|
||||
backend: home_workflow_agent
|
||||
backend: home_case_study_agent
|
||||
as_llm: home_construction_equivalence
|
||||
react_config:
|
||||
max_iters: 100
|
||||
home_candidate_gate:
|
||||
backend: home_workflow_agent
|
||||
backend: home_case_study_agent
|
||||
as_llm: home_candidate_gate
|
||||
react_config:
|
||||
max_iters: 60
|
||||
|
|
@ -106,7 +106,7 @@ application_defaults:
|
|||
react_config:
|
||||
max_iters: 3000
|
||||
home_diagnostic:
|
||||
backend: home_workflow_agent
|
||||
backend: home_case_study_agent
|
||||
as_llm: home_diagnostic
|
||||
time_budget_minutes: 60
|
||||
react_config:
|
||||
|
|
|
|||
|
|
@ -434,12 +434,11 @@ async def test_partial_export_and_report_failures_clean_review_files(
|
|||
assert (await app.run_job("home_evaluate")).success
|
||||
commit_candidate(prepared)
|
||||
if stage == "report":
|
||||
module = candidate_gate if kind == "candidate-gate" else construction_equivalence
|
||||
|
||||
def fail_report(*args, **kwargs):
|
||||
raise OSError("report write failed")
|
||||
|
||||
monkeypatch.setattr(module, "save_record", fail_report)
|
||||
monkeypatch.setattr(workflow_common, "save_record", fail_report)
|
||||
elif kind == "candidate-gate":
|
||||
original = candidate_gate.export_blob
|
||||
|
||||
|
|
|
|||
|
|
@ -680,13 +680,31 @@ async def test_startup_rejects_disabled_or_replaced_judge_job(app, tmp_path, ove
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("phase", ["answer", "construction"])
|
||||
@pytest.mark.parametrize("kind", ["timeout", "wrapped_timeout", "http_429", "http_503", "code", "legacy"])
|
||||
@pytest.mark.parametrize(
|
||||
"kind",
|
||||
[
|
||||
"timeout",
|
||||
"wrapped_timeout",
|
||||
"http_401",
|
||||
"http_403",
|
||||
"sdk_auth",
|
||||
"sdk_permission",
|
||||
"http_429",
|
||||
"http_503",
|
||||
"code",
|
||||
"legacy",
|
||||
],
|
||||
)
|
||||
async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monkeypatch, phase, kind):
|
||||
from types import SimpleNamespace
|
||||
import httpx
|
||||
from deploy.benchmark_docker.worker import _run_job_on_app
|
||||
from reme.components.job.base_job import BaseJob
|
||||
|
||||
authentication_failure = kind in {"http_401", "http_403", "sdk_auth", "sdk_permission"}
|
||||
method = "run_queries" if phase == "answer" else "run_build"
|
||||
original = getattr(FakeWorker, method)
|
||||
|
||||
async def failing_step(_context):
|
||||
if kind == "timeout":
|
||||
raise TimeoutError("model timed out")
|
||||
|
|
@ -695,6 +713,14 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk
|
|||
raise TimeoutError("model timed out")
|
||||
except TimeoutError as error:
|
||||
raise RuntimeError("model invocation failed") from error
|
||||
if kind.startswith("sdk_"):
|
||||
import openai
|
||||
|
||||
request = httpx.Request("POST", "https://example.test")
|
||||
status = 401 if kind == "sdk_auth" else 403
|
||||
response = httpx.Response(status, request=request)
|
||||
error_type = openai.AuthenticationError if kind == "sdk_auth" else openai.PermissionDeniedError
|
||||
raise error_type("provider access denied", response=response, body=None)
|
||||
if kind.startswith("http_"):
|
||||
request = httpx.Request("POST", "https://example.test")
|
||||
response = httpx.Response(int(kind.split("_")[1]), request=request)
|
||||
|
|
@ -761,6 +787,74 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk
|
|||
if kind != "legacy":
|
||||
assert failure["exception"] == result["exception"]
|
||||
assert all(worker.closed for worker in FakeFactory.instances[-1].workers)
|
||||
if authentication_failure:
|
||||
history = app.context.components["home.evaluation_history"]["default"]
|
||||
assert history.list_results() == []
|
||||
# Correcting the credentials must allow the same commit to be assessed.
|
||||
monkeypatch.setattr(FakeWorker, method, original)
|
||||
retry = await app.run_job("home_evaluate", case_ids=["1"])
|
||||
assert retry.success, retry.answer
|
||||
assert retry.answer["status"] == "completed"
|
||||
assert retry.answer["commit_sha"] == summary["commit_sha"]
|
||||
assert Path(retry.answer["result_dir"]).name == "2"
|
||||
assert retry.answer["scored_query_count"] == 1
|
||||
assert len(history.list_results()) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("phase", ["construction", "answer"])
|
||||
@pytest.mark.parametrize("kind", ["http_503", "wrapped_http_503", "wrapped_timeout", "code"])
|
||||
async def test_pool_exceptions_preserve_classification_and_retry_policy(app, prepared, monkeypatch, phase, kind):
|
||||
import httpx
|
||||
|
||||
async def fail(worker, inputs):
|
||||
if kind == "code":
|
||||
raise ValueError("candidate bug")
|
||||
try:
|
||||
if kind == "wrapped_timeout":
|
||||
raise TimeoutError("transport timed out")
|
||||
request = httpx.Request("POST", "https://example.test")
|
||||
response = httpx.Response(503, request=request)
|
||||
raise httpx.HTTPStatusError("service unavailable", request=request, response=response)
|
||||
except (TimeoutError, httpx.HTTPStatusError) as error:
|
||||
if kind.startswith("wrapped_"):
|
||||
raise RuntimeError("worker invocation failed") from error
|
||||
raise
|
||||
|
||||
monkeypatch.setattr(FakeWorker, "run_build" if phase == "construction" else "run_queries", fail)
|
||||
response = await app.run_job("home_evaluate", case_ids=["1"])
|
||||
infrastructure = kind != "code"
|
||||
assert response.success is (not infrastructure)
|
||||
summary = response.answer
|
||||
failure_kind = "infra_error" if infrastructure else "candidate_failure"
|
||||
assert summary["case_status_counts"][failure_kind] == 1
|
||||
assert summary["mean_query_score"] == (None if infrastructure else 0)
|
||||
run = Path(summary["result_dir"])
|
||||
failure = json.loads((run / "cases/1/failure.json").read_text())
|
||||
assert failure["stage"] == phase
|
||||
assert failure["failure_kind"] == failure_kind
|
||||
if kind == "code":
|
||||
expected = {"type": "ValueError", "module": "builtins"}
|
||||
else:
|
||||
cause = (
|
||||
{"type": "TimeoutError", "module": "builtins"}
|
||||
if kind == "wrapped_timeout"
|
||||
else {"type": "HTTPStatusError", "module": "httpx", "status_code": 503}
|
||||
)
|
||||
expected = (
|
||||
{"type": "RuntimeError", "module": "builtins", "causes": [cause]} if kind.startswith("wrapped_") else cause
|
||||
)
|
||||
assert failure["exception"] == expected
|
||||
|
||||
retry = await app.run_job("home_evaluate", case_ids=["1"])
|
||||
if infrastructure:
|
||||
assert retry.answer["status"] == "completed"
|
||||
assert Path(retry.answer["result_dir"]).name == "2"
|
||||
assert len(FakeFactory.instances) == 2
|
||||
else:
|
||||
assert retry.answer["error_code"] == "UNCHANGED_CANDIDATE"
|
||||
assert len(FakeFactory.instances) == 1
|
||||
assert all(worker.closed for factory in FakeFactory.instances for worker in factory.workers)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -8,11 +8,37 @@ import json
|
|||
|
||||
import pytest
|
||||
|
||||
from reme_home.evaluation.artifacts import average_tokens, read_json, rebuild_summary, summarize, write_json
|
||||
from reme_home.evaluation.artifacts import (
|
||||
average_tokens,
|
||||
failure_kind,
|
||||
read_json,
|
||||
rebuild_summary,
|
||||
summarize,
|
||||
write_json,
|
||||
)
|
||||
from reme_home.evaluation.compaction import compact_construction, compact_query
|
||||
from reme_home.preparation.data.models import CaseSpec, QuerySpec
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["construction", "answer"])
|
||||
@pytest.mark.parametrize("wrapped", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"exception, expected",
|
||||
[
|
||||
({"type": "HTTPStatusError", "module": "httpx", "status_code": 401}, "infra_error"),
|
||||
({"type": "HTTPStatusError", "module": "httpx", "status_code": 403}, "infra_error"),
|
||||
({"type": "AuthenticationError", "module": "openai"}, "infra_error"),
|
||||
({"type": "PermissionDeniedError", "module": "openai"}, "infra_error"),
|
||||
({"type": "HTTPStatusError", "module": "httpx", "status_code": 400}, "candidate_failure"),
|
||||
({"type": "ValueError", "module": "builtins"}, "candidate_failure"),
|
||||
],
|
||||
)
|
||||
def test_provider_access_errors_are_infrastructure_failures(stage, wrapped, exception, expected):
|
||||
if wrapped:
|
||||
exception = {"type": "RuntimeError", "module": "builtins", "causes": [exception]}
|
||||
assert failure_kind(exception, stage) == expected
|
||||
|
||||
|
||||
def test_scores_are_rounded_without_changing_reference_text_or_token_metrics(tmp_path):
|
||||
raw = {
|
||||
"query_id": "q",
|
||||
|
|
|
|||
|
|
@ -7,7 +7,15 @@ from pathlib import Path
|
|||
|
||||
import pytest
|
||||
from agentscope.message import ToolResultState
|
||||
from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state
|
||||
from reme_home.evaluation.history import (
|
||||
HomeEvaluationHistory,
|
||||
EvaluationRecord,
|
||||
read_state,
|
||||
rebuild_sampling_state,
|
||||
start_validation,
|
||||
)
|
||||
from reme_home.evaluation.budget import validation_budget_status
|
||||
from reme_home.evaluation import history as history_module
|
||||
from reme_home.sampling import SamplingPolicy, create_sampling_plan
|
||||
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
||||
FakeFactory,
|
||||
|
|
@ -31,6 +39,86 @@ def manifest(response):
|
|||
return json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_cache_failure_keeps_start_evidence_and_exhausts_budget(app, prepared, monkeypatch):
|
||||
commit_candidate(prepared)
|
||||
manager = history(app)
|
||||
manager.rebuild() # Start with a valid but empty cache, not just a missing cache.
|
||||
original = history_module.write_json
|
||||
|
||||
def fail_cache(path, value, **kwargs):
|
||||
if path == prepared / "sampling/state.json":
|
||||
raise OSError("injected start cache publication failure")
|
||||
return original(path, value, **kwargs)
|
||||
|
||||
monkeypatch.setattr(history_module, "write_json", fail_cache)
|
||||
response = await app.run_job("home_evaluate", max_evaluations=1, sampling=True)
|
||||
assert not response.success
|
||||
paths = list((prepared / "evaluations").glob("*/*/*/manifest.json"))
|
||||
assert len(paths) == 1
|
||||
payload = json.loads(paths[0].read_text())
|
||||
assert payload["status"] == "failed" and payload["sandbox_started_at"]
|
||||
plan = json.loads((prepared / payload["sampling"]["plan_path"]).read_text())
|
||||
assert payload["sandbox_started_at"] == plan["sandbox_started_at"]
|
||||
assert payload["sampling"]["round"] == plan["round"] == 1
|
||||
assert not (prepared / "sampling/state.json").exists()
|
||||
expected = {plan["validation_key"]: 1}
|
||||
assert read_state(prepared)["started_validations"] == expected
|
||||
assert rebuild_sampling_state(prepared, persist=False)["started_validations"] == expected
|
||||
budget = validation_budget_status(prepared, 1)
|
||||
assert budget["used_validations"] == 1 and budget["remaining_validations"] == 0
|
||||
rejected = await app.run_job("home_evaluate", max_evaluations=1)
|
||||
assert rejected.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED"
|
||||
assert len(FakeFactory.instances) == 1
|
||||
assert manager.list_results() == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interrupted_write", ["manifest.json", "state.json"])
|
||||
def test_interrupted_start_rebuilds_from_plan_and_retry_is_idempotent(tmp_path, monkeypatch, interrupted_write):
|
||||
class Interrupted(BaseException):
|
||||
"""Model process interruption outside ordinary exception handling."""
|
||||
|
||||
home = tmp_path / "home"
|
||||
run = home / "evaluations/candidate/abcdef0/1"
|
||||
run.mkdir(parents=True)
|
||||
original = history_module.write_json
|
||||
original(
|
||||
run / "manifest.json",
|
||||
{
|
||||
"mode": "scored",
|
||||
"dataset_kind": "training-set",
|
||||
"status": "running",
|
||||
"validation_id": "1",
|
||||
"case_ids": ["1"],
|
||||
"query_ids": {"1": ["q1"]},
|
||||
},
|
||||
)
|
||||
rebuild_sampling_state(home) # An old cache must not hide newly published evidence.
|
||||
|
||||
def interrupt(path, value, **kwargs):
|
||||
if path.name == interrupted_write:
|
||||
raise Interrupted()
|
||||
return original(path, value, **kwargs)
|
||||
|
||||
monkeypatch.setattr(history_module, "write_json", interrupt)
|
||||
with pytest.raises(Interrupted):
|
||||
start_validation(home, run, SamplingPolicy())
|
||||
plan_path = home / "sampling/plans/candidate/abcdef0/1.json"
|
||||
evidence = plan_path.read_bytes()
|
||||
plan = json.loads(evidence)
|
||||
assert not (home / "sampling/state.json").exists()
|
||||
if interrupted_write == "manifest.json":
|
||||
assert "sandbox_started_at" not in json.loads((run / "manifest.json").read_text())
|
||||
assert read_state(home)["started_validations"] == {"candidate/abcdef0/1": 1}
|
||||
assert validation_budget_status(home, 1)["remaining_validations"] == 0
|
||||
monkeypatch.setattr(history_module, "write_json", original)
|
||||
assert start_validation(home, run, SamplingPolicy()) == plan
|
||||
assert plan_path.read_bytes() == evidence
|
||||
assert json.loads((run / "manifest.json").read_text())["sandbox_started_at"] == plan["sandbox_started_at"]
|
||||
assert read_state(home)["round"] == 1
|
||||
assert rebuild_sampling_state(home)["started_validations"] == {"candidate/abcdef0/1": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_catalog_keeps_successful_low_scores_and_deduplicates(app, prepared):
|
||||
manager = history(app)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""ReMe CLI package."""
|
||||
|
||||
__version__ = "0.4.2.1"
|
||||
__version__ = "0.4.2.2"
|
||||
|
||||
from . import config
|
||||
from . import constants
|
||||
|
|
|
|||
|
|
@ -27,15 +27,16 @@ bump_version = _load_script("bump_version")
|
|||
package_studio = _load_script("package_studio")
|
||||
|
||||
|
||||
def test_home_requires_first_sandbox_release() -> None:
|
||||
"""Home must reject published ReMe wheels that predate its sandbox imports."""
|
||||
def test_home_requires_workflow_interfaces_release() -> None:
|
||||
"""Home must reject ReMe wheels that predate its workflow interfaces."""
|
||||
config = tomllib.loads((REPOSITORY / "plugins" / "home" / "pyproject.toml").read_text(encoding="utf-8"))
|
||||
requirement = next(Requirement(value) for value in config["project"]["dependencies"] if value.startswith("reme-ai"))
|
||||
|
||||
assert requirement.extras == {"as"}
|
||||
assert Version("0.4.1.14") not in requirement.specifier
|
||||
assert Version("0.4.2.0") not in requirement.specifier
|
||||
assert Version("0.4.2.1") in requirement.specifier
|
||||
assert Version("0.4.2.1") not in requirement.specifier
|
||||
assert Version("0.4.2.2") in requirement.specifier
|
||||
assert Version(bump_version.read_version()) in requirement.specifier
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue