diff --git a/.github/workflows/_build-python-packages.yml b/.github/workflows/_build-python-packages.yml index b4bfcf85..cdf90c93 100644 --- a/.github/workflows/_build-python-packages.yml +++ b/.github/workflows/_build-python-packages.yml @@ -96,8 +96,8 @@ jobs: - name: Reject incompatible published ReMe for Home run: | HOME_WHEEL="$(pwd)/$(ls dist/home/reme_home-[0-9]*.whl)" - if python -m pip install --dry-run --ignore-installed "${HOME_WHEEL}" "reme-ai[as]==0.4.2.0" > "${RUNNER_TEMP}/home-incompatible.log" 2>&1; then - echo 'Home must not install with the published ReMe 0.4.2.0 wheel.' + if python -m pip install --dry-run --ignore-installed "${HOME_WHEEL}" "reme-ai[as]==0.4.2.1" > "${RUNNER_TEMP}/home-incompatible.log" 2>&1; then + echo 'Home must not install with the published ReMe 0.4.2.1 wheel.' exit 1 fi cat "${RUNNER_TEMP}/home-incompatible.log" diff --git a/deploy/benchmark_docker/pool.py b/deploy/benchmark_docker/pool.py index d16aae63..09f867b0 100644 --- a/deploy/benchmark_docker/pool.py +++ b/deploy/benchmark_docker/pool.py @@ -17,6 +17,19 @@ from typing import Any from ._cleanup import finish_cleanup, format_error from .docker_sandbox import DockerReMeSandbox, DockerReMeSandboxFactory from .models import EvaluationQuery, JobRequest +from .worker import _exception_details + + +def _exception_classification(exc: BaseException) -> dict[str, Any]: + """Retain only classification fields, omitting absent status codes and causes.""" + details = _exception_details(exc) + result = {key: details[key] for key in ("type", "module")} + status = details.get("status_code") + if isinstance(status, int) and not isinstance(status, bool): + result["status_code"] = status + if details["causes"]: + result["causes"] = details["causes"] + return result @dataclass(frozen=True, slots=True) @@ -144,7 +157,7 @@ class BenchmarkDockerPool: except Exception as exc: result.error = format_error(exc) result.failure_stage = operation - result.exception = {"type": type(exc).__name__, "module": type(exc).__module__} + result.exception = _exception_classification(exc) await self._retain_failed_build_log(worker, plan, result) return result accepted = accept_build(result.build) if accept_build else result.build.get("success") @@ -175,7 +188,7 @@ class BenchmarkDockerPool: except Exception as exc: # Keep other cases running and release this worker. result.error = format_error(exc) result.failure_stage = operation - result.exception = {"type": type(exc).__name__, "module": type(exc).__module__} + result.exception = _exception_classification(exc) return result finally: if result.error and plan.export_diagnostics_to is not None and worker.reusable: diff --git a/plugins/home/README.md b/plugins/home/README.md index f74df8d3..f45ed398 100644 --- a/plugins/home/README.md +++ b/plugins/home/README.md @@ -144,15 +144,10 @@ All prepared outputs live under `workspace_dir/home/` (paths below are relative manifest with case IDs and content fingerprints. - `datasets/debug-cases/`: deterministic small smoke cases with source case IDs in their metadata, or an empty `cases/` directory when debug derivation is disabled. -- `code/runtime/`: fixed committed ReMe sources, packaging files and configurations, - without plugin projects. Uncommitted source edits are excluded. -- `code/repo/`: an independent Git repository containing complete selected plugin - projects (including packaging and configuration) and any selected ReMe core paths, - retaining repository-relative locations. `candidate.paths` defines the editable - scope, not the complete file inventory. The initial commit preserves the baseline. - Future runners assemble an isolated evaluation tree from runtime plus repo, - excluding `.git`; `install_projects` paths are relative to that tree. Install - plugins from the candidate, and never edit the fixed runtime. +- `code/runtime/` and `code/repo/`: fixed committed runtime and candidate Git + repository. See the [workspace and modification boundary](src/reme_home/runtime_docs/architecture-guide.md#workspace-and-modification-boundary) + for their roles and editable scope. Source exports exclude uncommitted edits; + `install_projects` paths are relative to the assembled evaluation tree. - `docs/architecture-guide.md` and `docs/validation-artifact-guide.md`: guides shipped with the installed Home plugin, covering candidate development and investigation of evaluation artifacts. @@ -236,60 +231,22 @@ For explicit `reuse_construction_from`, the caller must establish construction equivalence. Automatic sampled/budgeted selection instead invokes the host's equivalence reviewer and 60% replay policy. Runs record both commits. -Results use `home/evaluations////`: - -```text -manifest.json -summary.json -cases// - memory_construction/ - result.json - build.log - reme_workspace.tar.gz - reme_workspace/ - queries/ - result.json - /answer.log -``` - -Structured scores in query results and summaries use at most four decimal places. -Raw judge text and reference answers are preserved. - -The memory tar is exported **before** queries, retains session Git checkpoints, -and is independent of the query execution state. Replayed runs keep an -independent donor tar and inherited build log without charging construction -tokens again. Answers are persisted before host judging. Per-query results keep -reference/actual answers, scores, compact judge output, reported answer token usage -and failures. `queries/result.json` is the sole authoritative answer record; -reconstructable prompts and repeated payloads are removed using the reviewed -`dev_meta` rules. Construction results similarly compact redundant links, Job -counts, token records and cumulative daily-index snapshots. Summary -scores use the expected-query denominator; infrastructure/judge/artifact errors -make the comparable mean unavailable while retaining the observed-score mean. -The Job returns `success=false` for infrastructure/judge/artifact failures, while -preserving its structured answer and result directory; candidate failures remain -ordinary evaluation observations. -Base jobs preserve structured exception types, HTTP status codes and causes in -`Response.metadata["reme.job_error"]`; the Docker worker carries them into failure -artifacts. Network/provider failures anywhere in the cause chain invalidate the -comparison. Failed jobs without structured error details (including older -candidates) are conservatively treated as infrastructure errors, without parsing -their error text. Successful jobs returning empty answers still count as candidate -failures. The construction failure-rate tolerance applies only to candidate -errors: infrastructure or unclassified failures always reject construction, even -when another failed job was a candidate error. -The manifest retains run identity, selection, mode and execution status. The -summary retains only outcome counts, expected/scored query counts, score means, -low-score cases, question-type statistics and compact case/query mappings. -Candidate answer token summaries average only total tokens per model; detailed -construction costs remain in construction results. Temporary post-query archives -are discarded after extracting logs; separate execution, checksum and raw-answer -JSON copies are not saved. See the -[artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md) for the -exact fields, rebuild helper and migration from the previous verbose format. +Results use `home/evaluations////`. +The [artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md#saved-files) +is the reference for saved files, authoritative query records, compaction and +failure evidence. Its [summary reference](src/reme_home/runtime_docs/validation-artifact-guide.md#summary-fields) +covers score denominators, token accounting, debug fields and summary rebuilding. Before comparing runs, verify completed scored execution, infrastructure outcomes, candidate code and complete case/query coverage. +The Job returns `success=false` for infrastructure/judge/artifact failures while +preserving its structured answer and result directory; candidate failures remain +ordinary evaluation observations. Successful Jobs returning empty answers count +as candidate failures. Construction failure-rate tolerance applies only to +candidate errors; infrastructure or unclassified failures always reject construction. +See [failure interpretation](src/reme_home/runtime_docs/validation-artifact-guide.md#investigating-a-result) +for structured error classification. + Run/case/query `failure.json` files preserve diagnostics. Interrupted/fail-fast runs publish `partial_summary.json`, not a completed summary. New executions always allocate a new run; they never overwrite previous results. For scored runs, default @@ -305,10 +262,10 @@ Debug requires successful construction and nonempty, error-free answers, and returns `success=false` for candidate failures as well as infrastructure failures. It cannot reuse construction. Case/query selections refer to the selected mode's dataset; omitted selections run all its cases/queries. -Debug results retain execution counts and artifacts, but all mean scores are -null. `scored_query_count` and `query_status_counts.unscored` are zero, while -`expected_query_count` still records the expected execution count. Debug/scored -mode is recorded in the manifest. +Debug results retain execution counts and artifacts, with null mean scores. +See the [summary reference](src/reme_home/runtime_docs/validation-artifact-guide.md#summary-fields) +for coverage fields and manifest mode. + The former `dataset_kind` input is rejected; use `debug` instead. Debug results must not be used to assess answer correctness or candidate quality. No real Docker/model evaluation is required by Home's isolated unit tests. @@ -581,22 +538,14 @@ checksum checks. A later full rebuild considers the full training set again. ### Successful evaluation history -`reme_home.evaluation.history.HomeEvaluationHistory` is the dedicated -`home.evaluation_history` Component, automatically bound by `HomeEvaluation`. -It owns successful-result registration, lookup, deduplication, cache rebuilding -and the observations used for the next draw. Steps remain invocation adapters; -they do not own cross-evaluation state. - -A successful history entry requires `record_history=true` and a started, completed, scored training-set -run with a full summary and no infrastructure errors, following the existing -`Response.success` contract. Low or zero quality scores remain valid observations. -Classified candidate failures retain the existing response semantics; only their -actual numeric scores train EWMA. A run with any infrastructure error is entirely -excluded, including scores from its healthy cases. Debug, cancelled, fail-fast -and pre-start failures never enter successful history. A full summary is the durable -completion boundary: cancelling the caller during subsequent history bookkeeping -does not relabel that already-completed evaluation. Failed registration discards -only derived caches so they can rebuild from the final source run status. +The `home.evaluation_history` Component discovers successful scored training +results and rebuilds sampling observations from saved evidence. Low scores remain +valid observations; infrastructure failures, debug and incomplete runs are excluded. +See [history and sampling evidence](src/reme_home/runtime_docs/validation-artifact-guide.md#evaluation-history-and-adaptive-sampling-evidence) +for eligibility, started/successful rounds, saved plans and cache recovery. +The full summary is the durable completion boundary: cancellation during later +history bookkeeping does not relabel the completed evaluation. Failed registration +discards only derived caches, which can rebuild from saved run evidence. ```python history = app.context.components["home.evaluation_history"]["default"] @@ -605,159 +554,47 @@ result = history.get_result(results[-1].run_key) history.rebuild() # Rebuild sampling observations from saved artifacts. ``` -The Component owns the workspace lock for these synchronous filesystem methods; -async host code can run them in a worker thread. Successful results are discovered -from each run's manifest and summary; no `history.json` index is written or read. -Legacy index files are ignored and left untouched. Each returned record points to -its run via `run_key`; questions, answers and memory files stay in that result directory. - -The separate startup ledger advances only after the first successfully initialized -sandbox, including when a later worker or execution fails. Extra/replacement -workers do not count twice. `home/sampling/state.json` distinguishes started -`round` from `successful_round`. Only successful history advances the probability -clock or changes EWMA; failed runs cannot age cases or alter the next draw's -weights. Started plans live under -`home/sampling/plans///.json`. Their `round` is the -startup ledger round; `selection_round` records the successful-history snapshot -used for an adaptive draw. - -For `record_history=false`, full or explicit evaluation does not access the history -manager. Adaptive selection can still read existing observations, while leaving -them unchanged; its draw is saved locally as `manifest.selection_plan`, without -a startup-ledger round or shared sampling plan. `manifest.options.record_history` -persists the requested setting, and the response's `record_history` indicates -whether the run participates in history (always false for debug). Legacy manifests -without the setting retain the previous history behavior. - -Formal runs remain serialized by `home/metadata/validation.lock`; debug bypasses -that lock. Missing caches reconstruct from started plans and successful scored -artifacts. Older sampling caches are rebuilt under the success-only rule. -`history.rebuild()` rewrites only the catalog and sampling cache. Preserve source -manifests, plans and result artifacts when rebuilding; candidate code, datasets -and memory files are never modified. +These synchronous filesystem methods own the workspace lock; async host code can +run them in a worker thread. Formal runs remain serialized by +`home/metadata/validation.lock`; debug bypasses that lock. Rebuilding changes only +derived state; preserve source manifests, plans and result artifacts. ## Evaluation result inspection -Two read-only Jobs navigate saved evaluations independently of successful -optimization history. They include debug, failed, cancelled, running and standalone -(`record_history=false`) runs. Reads never register results, update sampling, -rebuild summaries or write artifacts. +Read-only inspection includes debug, failed, cancelled, running and standalone +(`record_history=false`) runs, independently of successful optimization history. +It never registers results, updates sampling, rebuilds summaries or writes artifacts. -`home_list_validation_results` accepts optional `branch_name` and `commit_sha` -filters plus `offset=0`, `limit=20` (1–100). Without filters it lists all runs, -newest first, with a stable path tie-breaker. It returns `row_count`, `rows` and -`next_offset`, compact identity/options, saved summary counts and scores, failure -previews and artifact paths. A malformed run is reported as a `read_error` row -without stopping the unfiltered catalog. - -`commit_sha` accepts a full SHA or hexadecimal prefix. A prefix must match exactly -one of **all commits reachable from the selected local branch**, including commits -that have never been evaluated. Omitting `branch_name` with a commit filter uses -the current candidate branch. Missing or ambiguous matches fail explicitly; the -response includes the resolved branch and full SHA. A branch-only filter can also -browse historical runs from a deleted branch. - -`home_inspect_validation` requires `validation_path` (absolute or relative to -`home/`) and accepts optional `case_id`, `query_id`, `offset=0` and `limit=20`: - -| Selection | Returned rows | +| Job | Use | | --- | --- | -| Run only | Case summaries, missing scores first, then low scores. | -| Run and case | Query summaries in manifest order, plus independently paged failed construction Jobs. | -| Run, case and query | One query's question, answer, rubric, score and judge/error previews. | - -`query_id` requires `case_id`; both must be selected by the run manifest. Cases or -queries without published evidence are explicitly marked `missing`, with null -scores. The tool uses a saved full summary, or an existing partial summary when -the full summary is absent. Missing summaries are reported without reconstruction. -Errors are previewed to 600 characters, and selected query fields to 2,000 each; -truncated fields carry a marker. `rows_source`, JSON `pointer` values, and log and -memory paths identify the original evidence. Failed construction Jobs have their -own `row_count`, `rows`, `next_offset` under `construction.failed_jobs`; their -pagination is independent of the query rows, using the supplied offset/limit. +| `home_list_validation_results` | Find runs, optionally filtered by `branch_name` and `commit_sha`. | +| `home_inspect_validation` | Inspect a `validation_path`, then narrow with `case_id` and `query_id`. | +| `home_trace_query_evidence` | Link a query to source sessions, memory checkpoints and saved evidence without a model call. | +| `home_inspect_memory_history` | Page through one case's exported memory Git checkpoints. | +| `home_compare_memory` | Compare committed memory within a run or across evaluations of the same case. | With the configured service running, use a second terminal: ```bash -reme home_list_validation_results branch_name=candidate commit_sha=abc123 limit=10 -reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1 -reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1 case_id=case-a query_id=query-a -``` - -Host callers use the same Jobs through `app.run_job()`. For a case-scoped agent, -inject `validation_path` and `case_id` with `injected_job_kwargs`; the wrapper hides -and enforces these values. Both Jobs reject path escapes and symlinks, and fail -closed for restricted `_allowed_paths` contexts, like the other Home Jobs. - -### Memory checkpoint inspection - -Read-only Jobs investigate a case's exported memory Git repository. Locate -an evaluation and case with `home_list_validation_results` and -`home_inspect_validation` first. These Jobs never change artifacts, Git refs, the -index, evaluation history or sampling state. - -- `home_trace_query_evidence(validation_path, case_id, query_id, - evidence_session_ids=null)` joins a selected query to its source sessions and - memory checkpoints without calling a model. Dataset evidence annotations take - precedence over optional inferred session IDs, including an empty annotation. - Returns `query` (question, expected/actual answer, score, error and - `result_missing` and `expected_kind`), `locations` with JSON Pointers into source - and saved query files, optional `criteria` previews of saved rubric judgments, - `evidence_source` (`independent_annotation`, - `diagnostician_inference`, or `unavailable`), `evidence_is_certain`, - `memory_status`, `sessions` (ID, source JSON Pointer, checkpoint and changed files), and `paths` to - the normalized case, memory repository, query results and query artifacts. - A null checkpoint means no matching `session: ` commit was found; missing - results keep scores null. Answer/question previews are limited to 2,000 - characters and errors to 600. At most 50 sessions and 50 changed files per - session are returned, with omitted counts when needed. Criterion previews are - limited to 20 entries and 600 characters per requirement/reason; absent or - mismatched rubric judgments are not paired. Session text, diffs, - later history and full answer/judge results remain at the returned paths or - in the inspection/comparison Jobs below. Original transcripts and derived - memories can both be searchable; these coordinates alone do not prove a cause. -- `home_inspect_memory_history(validation_path, case_id, path=null, offset=0, - limit=20)` returns `status`, `workspace_exists`, `has_git_history`, `head`, - `repository`, and newest-first checkpoint `rows` with `row_count` and - `next_offset`. Limits are 1–100. Each row includes the full commit SHA, parents, - timestamp, subject (up to 2,000 characters) and a `session_id` when the subject - follows `session: `. `path` follows a literal workspace-relative file or - directory. Missing exports, absent Git history and empty repositories have - distinct statuses; corrupt repositories fail explicitly. -- `home_compare_memory(validation_path, case_id, base_ref=null, - target_ref="HEAD", target_validation_path=null, path=null)` compares two memory - commits. Within one evaluation, `base_ref` is required. For another evaluation, - supply `target_validation_path`; both sides then default to `HEAD`. The case - must be selected in both manifests. Returns resolved commits, candidate commit - identities, manifest fingerprints, added/removed/modified paths, content-tree - hashes, a unified `diff`, and `truncated`. - -References accept HEAD-relative expressions and hexadecimal SHA prefixes unique -among commits reachable from that memory repository's HEAD. Resolved expressions -must also be HEAD-reachable. This is independent of candidate-code branch history. -Comparisons read committed regular files from Git objects, ignoring dirty or -untracked checkout contents. File mode changes alone are not content differences. -Binary changes are identified without printing their bytes. Each snapshot is -limited to 2,000 files, 20 MB total and 5 MB per file; diff output is limited to -100,000 characters. Narrow `path` if a limit is reached. Tree hashes describe the -selected path scope. Linked Git storage, alternates, shallow/partial histories and -symbolic links in Git metadata are rejected. - -After starting the service with the Home configuration, call from another terminal: - -```bash -reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1 +reme home_list_validation_results branch_name=candidate limit=10 +reme home_inspect_validation validation_path=evaluations/candidate/abcdef0/1 +reme home_inspect_validation validation_path=evaluations/candidate/abcdef0/1 case_id=case1 query_id=query1 reme home_trace_query_evidence validation_path=evaluations/candidate/abcdef0/1 case_id=case1 query_id=query1 +reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1 reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 case_id=case1 base_ref=HEAD~1 reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 target_validation_path=evaluations/candidate/1234567/1 case_id=case1 ``` -`validation_path` is absolute or relative to `home/`; for example the relative -form is `evaluations/candidate/abcdef0/1`. For agent tools, host `injected_job_kwargs` can pin -`validation_path` and `case_id`; the wrapper hides injected fields and rejects -model overrides. Like the other Home inspection Jobs, these Jobs reject -`_allowed_paths` restrictions rather than bypassing them. Use these tools directly -or through the case-study batch Job below. +`validation_path` is absolute or relative to `home/`. See +[read-only evaluation inspection](src/reme_home/runtime_docs/validation-artifact-guide.md#read-only-evaluation-inspection) +for pagination, commit-prefix resolution, previews and source pointers, and +[memory checkpoint tools](src/reme_home/runtime_docs/validation-artifact-guide.md#memory-checkpoint-tools) +for evidence provenance, missing-history states, comparison references and limits. + +Host callers use the same Jobs through `app.run_job()`. For case-scoped agent +tools, pin `validation_path` and `case_id` through `injected_job_kwargs`; +the wrapper hides those fields and rejects model overrides. These Jobs reject +path escapes, symlinks and restricted `_allowed_paths` contexts. ## Case-study batches @@ -794,19 +631,11 @@ available. No case scratch directory or intermediate analysis files are created. still return a text report with per-case metadata. Do not supply both path names or both case selectors. Other inspection Jobs retain their `validation_path` argument. -The toolkit includes validation inspection, compact query-evidence tracing, -memory history/comparison, and the three committed-code navigation Jobs. -Validation/case selectors and the evaluated code revision are hidden and enforced -by the host; memory comparison stays within the selected evaluation. General -Read/Grep/Glob, Write/Edit, validation and code-mutation tools are not exposed. +The case-study toolkit and mandatory read-only OS sandbox are described in the +[artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md#single-case-diagnostic-agent). +The host pins case/evaluation selectors and code revision. Direct code reads can +see current worktree changes; use revision-pinned Jobs to establish evaluated behavior. -`AutoBash` and `python_execute` share a required OS sandbox. Commands have no -static whitelist (`is_read_only=false`); filesystem permissions enforce access. -Read-only roots are the selected case artifact directory, normalized source-case -JSON, validation manifest, all of `home/code/`, and `home/docs/`. No host writable -roots are granted. System/interpreter runtime paths and disposable backend -temporary storage remain available. Direct code reads can see current worktree -changes; use the revision-pinned Jobs to establish the evaluated behavior. `python_execute` accepts either `code` or an existing `file`, optional file-only `args`, and a `timeout` in seconds (default 60, maximum 600). It uses `python -I -B` and returns returncode/stdout/stderr/timed_out. Both executors reuse bounded output, diff --git a/plugins/home/pyproject.toml b/plugins/home/pyproject.toml index e549f13c..59ac7578 100644 --- a/plugins/home/pyproject.toml +++ b/plugins/home/pyproject.toml @@ -5,7 +5,7 @@ description = "Self-evolution workspace preparation and local command execution readme = "README.md" requires-python = ">=3.11" dependencies = [ - "reme-ai[as]>=0.4.2.1", + "reme-ai[as]>=0.4.2.2", "safecmd>=0.1.16,<0.2", ] diff --git a/plugins/home/src/reme_home/agent/bounded.py b/plugins/home/src/reme_home/agent/bounded.py index 7db6a5a9..72af380f 100644 --- a/plugins/home/src/reme_home/agent/bounded.py +++ b/plugins/home/src/reme_home/agent/bounded.py @@ -30,6 +30,8 @@ class TimeBudgetReminder: class RunTimeBudgetTimer: """Track one agent run and emit advisory reminders without stopping it.""" + reminder_thresholds = (0.90, 0.75, 0.60) + def __init__( self, time_budget_minutes: float, @@ -89,7 +91,7 @@ class RunTimeBudgetTimer: if elapsed_ratio >= 1.0: reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10 else: - for threshold in (0.90, 0.75, 0.60): + for threshold in self.reminder_thresholds: if elapsed_ratio >= threshold: reached_ratio = threshold break @@ -98,17 +100,21 @@ class RunTimeBudgetTimer: self._last_reminded_ratio = reached_ratio persistent = self._next_reminder_is_persistent self._next_reminder_is_persistent = not persistent + return TimeBudgetReminder( + content=self._reminder_content(elapsed_minutes, reached_ratio), + persistent=persistent, + ) + + def _reminder_content(self, elapsed_minutes: float, reached_ratio: float) -> str: + """Describe this run without changing the shared reminder schedule.""" elapsed = _format_minutes(elapsed_minutes) percentage = round(reached_ratio * 100) - return TimeBudgetReminder( - content=( - f"The {self.agent_label} run has used {elapsed} ({percentage}% of its " - f"{_format_minutes(self.time_budget_minutes)} soft time budget). " - f"The run has completed {self.context_compression_count} context compressions. " - "This reminder does not terminate the run. Prioritize the highest-value remaining investigation and " - "produce the required final report as soon as the evidence is sufficient." - ), - persistent=persistent, + return ( + f"The {self.agent_label} run has used {elapsed} ({percentage}% of its " + f"{_format_minutes(self.time_budget_minutes)} soft time budget). " + f"The run has completed {self.context_compression_count} context compressions. " + "This reminder does not terminate the run. Prioritize the highest-value remaining investigation and " + "produce the required final report as soon as the evidence is sufficient." ) diff --git a/plugins/home/src/reme_home/agent/candidate_gate.py b/plugins/home/src/reme_home/agent/candidate_gate.py index 4951f243..485525fe 100644 --- a/plugins/home/src/reme_home/agent/candidate_gate.py +++ b/plugins/home/src/reme_home/agent/candidate_gate.py @@ -1,18 +1,12 @@ """Early candidate admission using read-only, commit-pinned source exports.""" -import json from contextlib import asynccontextmanager -from pathlib import Path import shutil -import sys from typing import Literal from uuid import uuid4 -from agentscope.tool import Glob, Grep, Read, Toolkit from pydantic import Field -import yaml -from ..auto_bash import AutoBash from ..code.repository import current_branch, require_clean, resolve_revision, validate_repository, validate_tree from ..code.snapshot import export_blob, tree from ..evaluation.artifacts import contained, read_json @@ -20,7 +14,7 @@ from ..evaluation.component import blocking from ..evaluation.repetition import previous_candidate_result from ..preparation.code import git from ..preparation.spec import SpecModel -from .workflow_common import WorkflowSandbox, save_record +from .workflow_common import review_report class CandidateGateReport(SpecModel): @@ -92,51 +86,18 @@ async def review_candidate(wrapper, home, context, budget, *, memory_limit_mib=N """Fail closed on sandbox setup; model/report failures allow validation as in dev_meta.""" async with review_export(home, "candidate-gate") as root: scope = await blocking(prepare_review_scope, home, context, root=root) - return await _review_candidate(wrapper, home, context, budget, scope, memory_limit_mib) - - -async def _review_candidate(wrapper, home, context, budget, scope, memory_limit_mib): - """Run inside the caller-owned export lifetime; persist only the report.""" - backend = WorkflowSandbox(scope, memory_limit_mib) - probe = await backend.exec_shell([sys.executable, "-I", "-B", "-c", "print('candidate gate sandbox')"], timeout=10) - if not probe.ok(): - raise RuntimeError("Candidate gate sandbox unavailable: " + probe.stderr.decode(errors="replace")) - tools = [ - Read(backend=backend), - Glob(backend=backend), - Grep(backend=backend), - AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=True), - ] - for tool in tools: - tool.is_concurrency_safe = False - prompt = yaml.safe_load(Path(__file__).with_suffix(".yaml").read_text(encoding="utf-8")) - report = {**context, "validation_budget": budget, "source_path": scope["scratch"]} - try: - reply = await wrapper.reply( + return await review_report( + wrapper, + home, + scope, f"Review admission from {context['base_commit']} to {context['candidate_commit']}.\n" f"Source exports and complete diff.patch: {scope['scratch']}", - system_prompt=prompt["system_prompt"], - toolkit=Toolkit(tools=tools), - builtin_tools=[], - job_tools=[], - skills=[], - resume=None, - session_id=None, - fork_session=False, - output_schema=None, - _structured_report_model=CandidateGateReport, - permission_mode="bypass", + {**context, "validation_budget": budget, "source_path": scope["scratch"]}, + role="candidate_gate", + sandbox_label="Candidate gate", + category="candidate-gates", + report_model=CandidateGateReport, + failure_decision="accept", + failure_reason="Candidate gate failed; validation is allowed", + memory_limit_mib=memory_limit_mib, ) - report.update({key: reply.get(key) for key in ("session_id", "usage", "result")}) - text = reply["result"].strip() - if text.startswith("```") and text.endswith("```"): - text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip() - report.update(CandidateGateReport.model_validate(json.loads(text)).model_dump()) - except Exception as exc: - report.update( - decision="accept", - reason=f"Candidate gate failed; validation is allowed: {type(exc).__name__}: {exc}", - review_failed=True, - ) - path = await blocking(save_record, home, "candidate-gates", report) - return {**report, "review_path": path} diff --git a/plugins/home/src/reme_home/agent/construction_equivalence.py b/plugins/home/src/reme_home/agent/construction_equivalence.py index a2b20610..58e9a5f3 100644 --- a/plugins/home/src/reme_home/agent/construction_equivalence.py +++ b/plugins/home/src/reme_home/agent/construction_equivalence.py @@ -1,23 +1,17 @@ """Construction equivalence review over isolated source and construction exports.""" -import json -from pathlib import Path import shutil -import sys from typing import Literal -from agentscope.tool import Glob, Grep, Read, Toolkit from pydantic import Field -import yaml -from ..auto_bash import AutoBash from ..evaluation.artifacts import contained, file_sha256, inspect_archive, read_json, write_json from ..evaluation.component import blocking from ..evaluation.inputs import reuse_inputs from ..preparation.code import tree_fingerprint from ..preparation.spec import SpecModel from .candidate_gate import prepare_review_scope, review_export -from .workflow_common import WorkflowSandbox, save_record +from .workflow_common import review_report class ConstructionEquivalenceReport(SpecModel): @@ -65,54 +59,18 @@ async def review_construction(wrapper, home, context, spec, cases, fingerprints, """Stop on sandbox setup errors; review errors conservatively require rebuilding.""" async with review_export(home, "construction-equivalence") as root: scope = await blocking(prepare_equivalence_scope, home, context, spec, cases, fingerprints, root=root) - return await _review_construction(wrapper, home, context, scope, memory_limit_mib) - - -async def _review_construction(wrapper, home, context, scope, memory_limit_mib): - """Run inside the caller-owned export lifetime; persist only the report.""" - backend = WorkflowSandbox(scope, memory_limit_mib) - probe = await backend.exec_shell( - [sys.executable, "-I", "-B", "-c", "print('construction review sandbox')"], - timeout=10, - ) - if not probe.ok(): - raise RuntimeError("Construction review sandbox unavailable: " + probe.stderr.decode(errors="replace")) - tools = [ - Read(backend=backend), - Glob(backend=backend), - Grep(backend=backend), - AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=True), - ] - for tool in tools: - tool.is_concurrency_safe = False - prompt = yaml.safe_load(Path(__file__).with_suffix(".yaml").read_text(encoding="utf-8")) - report = {**context, "source_path": scope["scratch"]} - try: - reply = await wrapper.reply( + return await review_report( + wrapper, + home, + scope, f"Compare construction from {context['base_commit']} to {context['candidate_commit']}.\n" f"Read-only source, diff and construction evidence: {scope['scratch']}", - system_prompt=prompt["system_prompt"], - toolkit=Toolkit(tools=tools), - builtin_tools=[], - job_tools=[], - skills=[], - resume=None, - session_id=None, - fork_session=False, - output_schema=None, - _structured_report_model=ConstructionEquivalenceReport, - permission_mode="bypass", + {**context, "source_path": scope["scratch"]}, + role="construction_equivalence", + sandbox_label="Construction review", + category="construction-equivalence", + report_model=ConstructionEquivalenceReport, + failure_decision="construction_changed", + failure_reason="Equivalence review failed; rebuild construction", + memory_limit_mib=memory_limit_mib, ) - report.update({key: reply.get(key) for key in ("session_id", "usage", "result")}) - text = reply["result"].strip() - if text.startswith("```") and text.endswith("```"): - text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip() - report.update(ConstructionEquivalenceReport.model_validate(json.loads(text)).model_dump()) - except Exception as exc: - report.update( - decision="construction_changed", - reason=f"Equivalence review failed; rebuild construction: {type(exc).__name__}: {exc}", - review_failed=True, - ) - path = await blocking(save_record, home, "construction-equivalence", report) - return {**report, "review_path": path} diff --git a/plugins/home/src/reme_home/agent/optimizer.py b/plugins/home/src/reme_home/agent/optimizer.py index aafc4d02..38189134 100644 --- a/plugins/home/src/reme_home/agent/optimizer.py +++ b/plugins/home/src/reme_home/agent/optimizer.py @@ -2,7 +2,6 @@ import asyncio import json -import math import time from pathlib import Path from typing import Callable @@ -23,7 +22,7 @@ from ..evaluation.budget import validate_max_evaluations, validation_budget_stat from ..preparation.spec import SpecModel from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory from .diagnostic import diagnostic_tool, evaluation_paths -from .bounded import TimeBudgetReminder, _format_minutes, _remove_hint_by_identity +from .bounded import RunTimeBudgetTimer, _format_minutes, _remove_hint_by_identity from .compression import RecoverableCompressionAgent from .workflow_common import required_job, save_record, workflow_reply from .wrapper import CaseStudyAgentWrapper @@ -37,55 +36,23 @@ MUTATION_JOBS = ( PENDING_CYCLE_TRANSITIONS = "home.optimizer.pending_cycle_transitions" -class ValidationCycleTimer: +class ValidationCycleTimer(RunTimeBudgetTimer): """Emit escalating reminders as a scored-validation cycle ages.""" + reminder_thresholds = (0.90, 0.75, 0.50) + def __init__(self, time_budget_minutes: float, *, clock: Callable[[], float] = time.monotonic) -> None: - if ( - not isinstance(time_budget_minutes, (int, float)) - or isinstance(time_budget_minutes, bool) - or not math.isfinite(time_budget_minutes) - or time_budget_minutes <= 0 - ): - raise ValueError("time_budget_minutes must be a positive number") - self.time_budget_minutes = float(time_budget_minutes) - self._clock = clock - self._started_at = self._clock() - self._last_reminded_ratio = 0.0 - self._next_reminder_is_persistent = False - self.context_compression_count = 0 + super().__init__(time_budget_minutes, agent_label="optimizer", clock=clock) + self.start() def reset(self) -> None: """Start a fresh cycle after a scored validation finishes.""" - self._started_at = self._clock() - self._last_reminded_ratio = 0.0 - self._next_reminder_is_persistent = False - self.context_compression_count = 0 + self.start() - def record_context_compression(self) -> None: - """Record one completed compression since the last scored validation.""" - - self.context_compression_count += 1 - - def next_reminder(self) -> TimeBudgetReminder | None: - """Return one reminder for the latest newly reached deadline.""" - - elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0) + def _reminder_content(self, elapsed_minutes: float, reached_ratio: float) -> str: + """Keep optimizer guidance tied to the last scored validation.""" elapsed_ratio = elapsed_minutes / self.time_budget_minutes - reached_ratio: float | None = None - if elapsed_ratio >= 1.0: - reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10 - else: - for threshold in (0.9, 0.75, 0.5): - if elapsed_ratio >= threshold: - reached_ratio = threshold - break - if reached_ratio is None or reached_ratio <= self._last_reminded_ratio: - return None - self._last_reminded_ratio = reached_ratio - persistent = self._next_reminder_is_persistent - self._next_reminder_is_persistent = not persistent elapsed = _format_minutes(elapsed_minutes) percentage = round(reached_ratio * 100) timing = ( @@ -97,22 +64,16 @@ class ValidationCycleTimer: "scored validation. " ) if elapsed_ratio >= 1.0: - return TimeBudgetReminder( - content=( - f"{timing}{compression}" - "Prioritize completing the code changes, committing them, passing the smoke test, and starting " - "validation as soon as possible." - ), - persistent=persistent, + return ( + f"{timing}{compression}" + "Prioritize completing the code changes, committing them, passing the smoke test, and starting " + "validation as soon as possible." ) remaining = _format_minutes(self.time_budget_minutes - elapsed_minutes) - return TimeBudgetReminder( - content=( - f"{timing}{compression}" - f"Complete the code changes, commit them, pass the smoke test, and start validation within {remaining}." - "" - ), - persistent=persistent, + return ( + f"{timing}{compression}" + f"Complete the code changes, commit them, pass the smoke test, and start validation within {remaining}." + "" ) diff --git a/plugins/home/src/reme_home/agent/workflow_common.py b/plugins/home/src/reme_home/agent/workflow_common.py index 5383e69f..ac0323cd 100644 --- a/plugins/home/src/reme_home/agent/workflow_common.py +++ b/plugins/home/src/reme_home/agent/workflow_common.py @@ -150,6 +150,72 @@ def save_record(home, category, payload): return str(path) +async def review_report( + wrapper, + home, + scope, + task, + report, + *, + role, + sandbox_label, + category, + report_model, + failure_decision, + failure_reason, + memory_limit_mib=None, +): + """Share read-only review mechanics while callers choose their failure policy. + + Sandbox and prompt setup and report persistence deliberately stay outside + the model/report recovery boundary. Cancellation propagates to the export owner. + """ + backend = WorkflowSandbox(scope, memory_limit_mib) + probe = await backend.exec_shell( + [sys.executable, "-I", "-B", "-c", f"print('{sandbox_label.lower()} sandbox')"], + timeout=10, + ) + if not probe.ok(): + raise RuntimeError(f"{sandbox_label} sandbox unavailable: " + probe.stderr.decode(errors="replace")) + tools = [ + Read(backend=backend), + Glob(backend=backend), + Grep(backend=backend), + AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=True), + ] + for tool in tools: + tool.is_concurrency_safe = False + prompt = yaml.safe_load(Path(__file__).with_name(f"{role}.yaml").read_text(encoding="utf-8")) + try: + reply = await wrapper.reply( + task, + system_prompt=prompt["system_prompt"], + toolkit=Toolkit(tools=tools), + builtin_tools=[], + job_tools=[], + skills=[], + resume=None, + session_id=None, + fork_session=False, + output_schema=None, + _structured_report_model=report_model, + permission_mode="bypass", + ) + report.update({key: reply.get(key) for key in ("session_id", "usage", "result")}) + text = reply["result"].strip() + if text.startswith("```") and text.endswith("```"): + text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip() + report.update(report_model.model_validate(json.loads(text)).model_dump()) + except Exception as exc: + report.update( + decision=failure_decision, + reason=f"{failure_reason}: {type(exc).__name__}: {exc}", + review_failed=True, + ) + path = await blocking(save_record, home, category, report) + return {**report, "review_path": path} + + _INSPECT = """\ import json, pathlib, sys path = pathlib.Path(sys.argv[2]) diff --git a/plugins/home/src/reme_home/evaluation/artifacts.py b/plugins/home/src/reme_home/evaluation/artifacts.py index 56c8ccfe..33911212 100644 --- a/plugins/home/src/reme_home/evaluation/artifacts.py +++ b/plugins/home/src/reme_home/evaluation/artifacts.py @@ -331,10 +331,12 @@ def rebuild_summary(run: Path) -> dict: INFRA_TYPES = { "APIConnectionError", "APITimeoutError", + "AuthenticationError", "ConnectError", "ConnectionError", "ConnectTimeout", "NetworkError", + "PermissionDeniedError", "PoolTimeout", "RateLimitError", "ReadError", @@ -350,7 +352,7 @@ INFRA_TYPES = { def failure_kind(exception, stage: str) -> str: - """Only candidate execution errors count against candidate scores.""" + """Only candidate execution errors count against scores; provider access errors are infrastructure failures.""" if stage not in {"construction", "answer"}: return "infra_error" exception = exception if isinstance(exception, dict) else {} @@ -360,7 +362,7 @@ def failure_kind(exception, stage: str) -> str: if isinstance(causes, list) and any(failure_kind(cause, stage) == "infra_error" for cause in causes): return "infra_error" status = exception.get("status_code") - if isinstance(status, int) and (status in {408, 429} or status >= 500): + if isinstance(status, int) and (status in {401, 403, 408, 429} or status >= 500): return "infra_error" name = str(exception.get("type", "")) module = str(exception.get("module", "")) diff --git a/plugins/home/src/reme_home/evaluation/component.py b/plugins/home/src/reme_home/evaluation/component.py index b7a563f8..99b9393e 100644 --- a/plugins/home/src/reme_home/evaluation/component.py +++ b/plugins/home/src/reme_home/evaluation/component.py @@ -235,8 +235,12 @@ class HomeEvaluation(BaseComponent): manifest["selection_plan"] = plan write_json(run / "manifest.json", manifest) return - self.history.record_started(run, policy if policy is not None else self.sampling_policy, plan) - manifest.update(read_json(run / "manifest.json")) + try: + self.history.record_started(run, policy if policy is not None else self.sampling_policy, plan) + finally: + # Cache publication can fail after the durable start was written. + # Preserve those fields when the caller later saves failure status. + manifest.update(read_json(run / "manifest.json")) def _finish_sampling(self, run): self.history.record(run) diff --git a/plugins/home/src/reme_home/evaluation/history.py b/plugins/home/src/reme_home/evaluation/history.py index 12b16703..81faff9d 100644 --- a/plugins/home/src/reme_home/evaluation/history.py +++ b/plugins/home/src/reme_home/evaluation/history.py @@ -114,35 +114,45 @@ def start_validation(home: Path, run: Path, policy: SamplingPolicy, plan: dict | return {} key = run.relative_to(home / "evaluations").as_posix() state = read_state(home) - if key in state["started_validations"]: - return read_json(contained(home, manifest["sampling"]["plan_path"])) - plan = ( - dict(plan) - if plan is not None - else { - "mode": "explicit", - "case_ids": manifest["case_ids"], - "query_ids": manifest["query_ids"], - "policy": policy.model_dump(), - } - ) - # Preserve the draw's state snapshot separately from the committed start round. - if "round" in plan: - plan["selection_round"] = plan["round"] - plan.update( - round=state["round"] + 1, - validation_id=manifest["validation_id"], - validation_key=key, - sandbox_started_at=utc_now().isoformat(), - ) relative = f"sampling/plans/{key}.json" - write_json(contained(home, relative), plan) + if key in state["started_validations"]: + plan = read_json(contained(home, relative)) + else: + plan = ( + dict(plan) + if plan is not None + else { + "mode": "explicit", + "case_ids": manifest["case_ids"], + "query_ids": manifest["query_ids"], + "policy": policy.model_dump(), + } + ) + # Preserve the draw's state snapshot separately from the committed start round. + if "round" in plan: + plan["selection_round"] = plan["round"] + plan.update( + round=state["round"] + 1, + validation_id=manifest["validation_id"], + validation_key=key, + sandbox_started_at=utc_now().isoformat(), + ) + # Invalidate before publishing evidence: interruption at either following + # write must rebuild from the started plan rather than trust an old cache. + cache = contained(home, "sampling/state.json") + cache.unlink(missing_ok=True) + if key not in state["started_validations"]: + write_json(contained(home, relative), plan) manifest["sampling"] = {"round": plan["round"], "mode": plan["mode"], "plan_path": relative} manifest["sandbox_started_at"] = plan["sandbox_started_at"] write_json(run / "manifest.json", manifest) - state["round"] = plan["round"] + state["round"] = max(state["round"], plan["round"]) state["started_validations"][key] = plan["round"] - write_json(contained(home, "sampling/state.json"), state) + try: + write_json(cache, state) + except BaseException: + cache.unlink(missing_ok=True) + raise return plan @@ -231,10 +241,17 @@ def rebuild_sampling_state(home: Path, *, persist: bool = True) -> dict: manifest.get("mode") == "debug" or manifest.get("dataset_kind") != "training-set" or manifest.get("options", {}).get("record_history") is False - or not (manifest.get("sandbox_started_at")) ): continue - plan = read_json(contained(home, manifest["sampling"]["plan_path"])) + key = path.parent.relative_to(home / "evaluations").as_posix() + plan_path = contained(home, manifest.get("sampling", {}).get("plan_path", f"sampling/plans/{key}.json")) + if not manifest.get("sandbox_started_at") and not plan_path.is_file(): + continue + plan = read_json(plan_path) + # A started plan is independent evidence even if the process stopped + # before publishing the manifest's start fields. + if not plan.get("sandbox_started_at"): + continue runs.append((plan["round"], path.parent, plan["validation_key"])) for round_id, run, key in sorted(runs): state["round"] = max(state["round"], round_id) diff --git a/plugins/home/src/reme_home/plugin.yaml b/plugins/home/src/reme_home/plugin.yaml index cee4cf75..03b570fe 100644 --- a/plugins/home/src/reme_home/plugin.yaml +++ b/plugins/home/src/reme_home/plugin.yaml @@ -86,12 +86,12 @@ application_defaults: thinking_enable: false agent_wrapper: home_construction_equivalence: - backend: home_workflow_agent + backend: home_case_study_agent as_llm: home_construction_equivalence react_config: max_iters: 100 home_candidate_gate: - backend: home_workflow_agent + backend: home_case_study_agent as_llm: home_candidate_gate react_config: max_iters: 60 @@ -106,7 +106,7 @@ application_defaults: react_config: max_iters: 3000 home_diagnostic: - backend: home_workflow_agent + backend: home_case_study_agent as_llm: home_diagnostic time_budget_minutes: 60 react_config: diff --git a/plugins/home/tests/test_construction_equivalence.py b/plugins/home/tests/test_construction_equivalence.py index a6052af0..db5216c4 100644 --- a/plugins/home/tests/test_construction_equivalence.py +++ b/plugins/home/tests/test_construction_equivalence.py @@ -434,12 +434,11 @@ async def test_partial_export_and_report_failures_clean_review_files( assert (await app.run_job("home_evaluate")).success commit_candidate(prepared) if stage == "report": - module = candidate_gate if kind == "candidate-gate" else construction_equivalence def fail_report(*args, **kwargs): raise OSError("report write failed") - monkeypatch.setattr(module, "save_record", fail_report) + monkeypatch.setattr(workflow_common, "save_record", fail_report) elif kind == "candidate-gate": original = candidate_gate.export_blob diff --git a/plugins/home/tests/test_evaluation.py b/plugins/home/tests/test_evaluation.py index 3dd2b220..98a5ff32 100644 --- a/plugins/home/tests/test_evaluation.py +++ b/plugins/home/tests/test_evaluation.py @@ -680,13 +680,31 @@ async def test_startup_rejects_disabled_or_replaced_judge_job(app, tmp_path, ove @pytest.mark.asyncio @pytest.mark.parametrize("phase", ["answer", "construction"]) -@pytest.mark.parametrize("kind", ["timeout", "wrapped_timeout", "http_429", "http_503", "code", "legacy"]) +@pytest.mark.parametrize( + "kind", + [ + "timeout", + "wrapped_timeout", + "http_401", + "http_403", + "sdk_auth", + "sdk_permission", + "http_429", + "http_503", + "code", + "legacy", + ], +) async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monkeypatch, phase, kind): from types import SimpleNamespace import httpx from deploy.benchmark_docker.worker import _run_job_on_app from reme.components.job.base_job import BaseJob + authentication_failure = kind in {"http_401", "http_403", "sdk_auth", "sdk_permission"} + method = "run_queries" if phase == "answer" else "run_build" + original = getattr(FakeWorker, method) + async def failing_step(_context): if kind == "timeout": raise TimeoutError("model timed out") @@ -695,6 +713,14 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk raise TimeoutError("model timed out") except TimeoutError as error: raise RuntimeError("model invocation failed") from error + if kind.startswith("sdk_"): + import openai + + request = httpx.Request("POST", "https://example.test") + status = 401 if kind == "sdk_auth" else 403 + response = httpx.Response(status, request=request) + error_type = openai.AuthenticationError if kind == "sdk_auth" else openai.PermissionDeniedError + raise error_type("provider access denied", response=response, body=None) if kind.startswith("http_"): request = httpx.Request("POST", "https://example.test") response = httpx.Response(int(kind.split("_")[1]), request=request) @@ -761,6 +787,74 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk if kind != "legacy": assert failure["exception"] == result["exception"] assert all(worker.closed for worker in FakeFactory.instances[-1].workers) + if authentication_failure: + history = app.context.components["home.evaluation_history"]["default"] + assert history.list_results() == [] + # Correcting the credentials must allow the same commit to be assessed. + monkeypatch.setattr(FakeWorker, method, original) + retry = await app.run_job("home_evaluate", case_ids=["1"]) + assert retry.success, retry.answer + assert retry.answer["status"] == "completed" + assert retry.answer["commit_sha"] == summary["commit_sha"] + assert Path(retry.answer["result_dir"]).name == "2" + assert retry.answer["scored_query_count"] == 1 + assert len(history.list_results()) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("phase", ["construction", "answer"]) +@pytest.mark.parametrize("kind", ["http_503", "wrapped_http_503", "wrapped_timeout", "code"]) +async def test_pool_exceptions_preserve_classification_and_retry_policy(app, prepared, monkeypatch, phase, kind): + import httpx + + async def fail(worker, inputs): + if kind == "code": + raise ValueError("candidate bug") + try: + if kind == "wrapped_timeout": + raise TimeoutError("transport timed out") + request = httpx.Request("POST", "https://example.test") + response = httpx.Response(503, request=request) + raise httpx.HTTPStatusError("service unavailable", request=request, response=response) + except (TimeoutError, httpx.HTTPStatusError) as error: + if kind.startswith("wrapped_"): + raise RuntimeError("worker invocation failed") from error + raise + + monkeypatch.setattr(FakeWorker, "run_build" if phase == "construction" else "run_queries", fail) + response = await app.run_job("home_evaluate", case_ids=["1"]) + infrastructure = kind != "code" + assert response.success is (not infrastructure) + summary = response.answer + failure_kind = "infra_error" if infrastructure else "candidate_failure" + assert summary["case_status_counts"][failure_kind] == 1 + assert summary["mean_query_score"] == (None if infrastructure else 0) + run = Path(summary["result_dir"]) + failure = json.loads((run / "cases/1/failure.json").read_text()) + assert failure["stage"] == phase + assert failure["failure_kind"] == failure_kind + if kind == "code": + expected = {"type": "ValueError", "module": "builtins"} + else: + cause = ( + {"type": "TimeoutError", "module": "builtins"} + if kind == "wrapped_timeout" + else {"type": "HTTPStatusError", "module": "httpx", "status_code": 503} + ) + expected = ( + {"type": "RuntimeError", "module": "builtins", "causes": [cause]} if kind.startswith("wrapped_") else cause + ) + assert failure["exception"] == expected + + retry = await app.run_job("home_evaluate", case_ids=["1"]) + if infrastructure: + assert retry.answer["status"] == "completed" + assert Path(retry.answer["result_dir"]).name == "2" + assert len(FakeFactory.instances) == 2 + else: + assert retry.answer["error_code"] == "UNCHANGED_CANDIDATE" + assert len(FakeFactory.instances) == 1 + assert all(worker.closed for factory in FakeFactory.instances for worker in factory.workers) @pytest.mark.asyncio diff --git a/plugins/home/tests/test_evaluation_artifacts.py b/plugins/home/tests/test_evaluation_artifacts.py index 42124637..59b415d9 100644 --- a/plugins/home/tests/test_evaluation_artifacts.py +++ b/plugins/home/tests/test_evaluation_artifacts.py @@ -8,11 +8,37 @@ import json import pytest -from reme_home.evaluation.artifacts import average_tokens, read_json, rebuild_summary, summarize, write_json +from reme_home.evaluation.artifacts import ( + average_tokens, + failure_kind, + read_json, + rebuild_summary, + summarize, + write_json, +) from reme_home.evaluation.compaction import compact_construction, compact_query from reme_home.preparation.data.models import CaseSpec, QuerySpec +@pytest.mark.parametrize("stage", ["construction", "answer"]) +@pytest.mark.parametrize("wrapped", [False, True]) +@pytest.mark.parametrize( + "exception, expected", + [ + ({"type": "HTTPStatusError", "module": "httpx", "status_code": 401}, "infra_error"), + ({"type": "HTTPStatusError", "module": "httpx", "status_code": 403}, "infra_error"), + ({"type": "AuthenticationError", "module": "openai"}, "infra_error"), + ({"type": "PermissionDeniedError", "module": "openai"}, "infra_error"), + ({"type": "HTTPStatusError", "module": "httpx", "status_code": 400}, "candidate_failure"), + ({"type": "ValueError", "module": "builtins"}, "candidate_failure"), + ], +) +def test_provider_access_errors_are_infrastructure_failures(stage, wrapped, exception, expected): + if wrapped: + exception = {"type": "RuntimeError", "module": "builtins", "causes": [exception]} + assert failure_kind(exception, stage) == expected + + def test_scores_are_rounded_without_changing_reference_text_or_token_metrics(tmp_path): raw = { "query_id": "q", diff --git a/plugins/home/tests/test_evaluation_history.py b/plugins/home/tests/test_evaluation_history.py index 7fb2aaa0..514bbe6d 100644 --- a/plugins/home/tests/test_evaluation_history.py +++ b/plugins/home/tests/test_evaluation_history.py @@ -7,7 +7,15 @@ from pathlib import Path import pytest from agentscope.message import ToolResultState -from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state +from reme_home.evaluation.history import ( + HomeEvaluationHistory, + EvaluationRecord, + read_state, + rebuild_sampling_state, + start_validation, +) +from reme_home.evaluation.budget import validation_budget_status +from reme_home.evaluation import history as history_module from reme_home.sampling import SamplingPolicy, create_sampling_plan from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import FakeFactory, @@ -31,6 +39,86 @@ def manifest(response): return json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text()) +@pytest.mark.asyncio +async def test_start_cache_failure_keeps_start_evidence_and_exhausts_budget(app, prepared, monkeypatch): + commit_candidate(prepared) + manager = history(app) + manager.rebuild() # Start with a valid but empty cache, not just a missing cache. + original = history_module.write_json + + def fail_cache(path, value, **kwargs): + if path == prepared / "sampling/state.json": + raise OSError("injected start cache publication failure") + return original(path, value, **kwargs) + + monkeypatch.setattr(history_module, "write_json", fail_cache) + response = await app.run_job("home_evaluate", max_evaluations=1, sampling=True) + assert not response.success + paths = list((prepared / "evaluations").glob("*/*/*/manifest.json")) + assert len(paths) == 1 + payload = json.loads(paths[0].read_text()) + assert payload["status"] == "failed" and payload["sandbox_started_at"] + plan = json.loads((prepared / payload["sampling"]["plan_path"]).read_text()) + assert payload["sandbox_started_at"] == plan["sandbox_started_at"] + assert payload["sampling"]["round"] == plan["round"] == 1 + assert not (prepared / "sampling/state.json").exists() + expected = {plan["validation_key"]: 1} + assert read_state(prepared)["started_validations"] == expected + assert rebuild_sampling_state(prepared, persist=False)["started_validations"] == expected + budget = validation_budget_status(prepared, 1) + assert budget["used_validations"] == 1 and budget["remaining_validations"] == 0 + rejected = await app.run_job("home_evaluate", max_evaluations=1) + assert rejected.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED" + assert len(FakeFactory.instances) == 1 + assert manager.list_results() == [] + + +@pytest.mark.parametrize("interrupted_write", ["manifest.json", "state.json"]) +def test_interrupted_start_rebuilds_from_plan_and_retry_is_idempotent(tmp_path, monkeypatch, interrupted_write): + class Interrupted(BaseException): + """Model process interruption outside ordinary exception handling.""" + + home = tmp_path / "home" + run = home / "evaluations/candidate/abcdef0/1" + run.mkdir(parents=True) + original = history_module.write_json + original( + run / "manifest.json", + { + "mode": "scored", + "dataset_kind": "training-set", + "status": "running", + "validation_id": "1", + "case_ids": ["1"], + "query_ids": {"1": ["q1"]}, + }, + ) + rebuild_sampling_state(home) # An old cache must not hide newly published evidence. + + def interrupt(path, value, **kwargs): + if path.name == interrupted_write: + raise Interrupted() + return original(path, value, **kwargs) + + monkeypatch.setattr(history_module, "write_json", interrupt) + with pytest.raises(Interrupted): + start_validation(home, run, SamplingPolicy()) + plan_path = home / "sampling/plans/candidate/abcdef0/1.json" + evidence = plan_path.read_bytes() + plan = json.loads(evidence) + assert not (home / "sampling/state.json").exists() + if interrupted_write == "manifest.json": + assert "sandbox_started_at" not in json.loads((run / "manifest.json").read_text()) + assert read_state(home)["started_validations"] == {"candidate/abcdef0/1": 1} + assert validation_budget_status(home, 1)["remaining_validations"] == 0 + monkeypatch.setattr(history_module, "write_json", original) + assert start_validation(home, run, SamplingPolicy()) == plan + assert plan_path.read_bytes() == evidence + assert json.loads((run / "manifest.json").read_text())["sandbox_started_at"] == plan["sandbox_started_at"] + assert read_state(home)["round"] == 1 + assert rebuild_sampling_state(home)["started_validations"] == {"candidate/abcdef0/1": 1} + + @pytest.mark.asyncio async def test_history_catalog_keeps_successful_low_scores_and_deduplicates(app, prepared): manager = history(app) diff --git a/reme/__init__.py b/reme/__init__.py index 9e0606d4..85bc11e1 100644 --- a/reme/__init__.py +++ b/reme/__init__.py @@ -1,6 +1,6 @@ """ReMe CLI package.""" -__version__ = "0.4.2.1" +__version__ = "0.4.2.2" from . import config from . import constants diff --git a/tests/unit/test_package_versions.py b/tests/unit/test_package_versions.py index bb9bb80f..0e3319ca 100644 --- a/tests/unit/test_package_versions.py +++ b/tests/unit/test_package_versions.py @@ -27,15 +27,16 @@ bump_version = _load_script("bump_version") package_studio = _load_script("package_studio") -def test_home_requires_first_sandbox_release() -> None: - """Home must reject published ReMe wheels that predate its sandbox imports.""" +def test_home_requires_workflow_interfaces_release() -> None: + """Home must reject ReMe wheels that predate its workflow interfaces.""" config = tomllib.loads((REPOSITORY / "plugins" / "home" / "pyproject.toml").read_text(encoding="utf-8")) requirement = next(Requirement(value) for value in config["project"]["dependencies"] if value.startswith("reme-ai")) assert requirement.extras == {"as"} assert Version("0.4.1.14") not in requirement.specifier assert Version("0.4.2.0") not in requirement.specifier - assert Version("0.4.2.1") in requirement.specifier + assert Version("0.4.2.1") not in requirement.specifier + assert Version("0.4.2.2") in requirement.specifier assert Version(bump_version.read_version()) in requirement.specifier