feat(home): run case studies with bounded concurrency and fix sandbox test

This commit is contained in:
酱牛肉 2026-10-30 18:53:35 +08:00
parent febc84586e
commit 4e1ceaee40
5 changed files with 173 additions and 28 deletions

View file

@ -736,14 +736,18 @@ or through the case-study batch Job below.
`home_case_study(evaluation_path, case_ids, focus=...)` investigates up to eight
unique case IDs in input order, sharing the evaluation and unchanged focus.
Cases run sequentially, each with fresh agent state, sandbox and timer. The
Cases run concurrently, each with fresh agent state, sandbox and timer. The
host Step setting `concurrency` defaults to 4, matching dev_meta, and must be a
positive integer; set it to 1 for sequential execution. Results preserve input
order regardless of completion order. Cancelling the batch cancels running and
queued cases and waits for their cleanup before returning. The
whole selection is validated before model calls; a case execution failure is
returned individually without cancelling later cases. It follows dev_meta's
case-study workflow: trace judging and answer failures backward through retrieval,
memory construction, source sessions and
the evaluated implementation. The text report cites evidence paths/lines or
JSON fields and distinguishes supported causes, hypotheses and missing evidence;
it does not prescribe fixes. There is no parallel case-study dispatcher.
it does not prescribe fixes.
The `home_case_study` component uses `home_case_study_agent`, a thin subclass of
the bounded wrapper, defaulting to the dedicated `as_llm: home_case_study`
@ -785,7 +789,7 @@ Linux requires Bubblewrap and, with the default 4096-MiB limit, a writable
delegated cgroup v2. macOS requires Seatbelt (`sandbox-exec`); memory limits are
not enforced there. The Job probes sandbox startup before calling the model and
never falls back to an unrestricted executor. The command whitelist parser is
not used by this agent. Sandbox policy, output-report token guidance and executor
not used by this agent. Batch concurrency, sandbox policy, output-report token guidance and executor
memory policy are host Step configuration, not model arguments:
```yaml
@ -794,6 +798,7 @@ jobs:
steps:
- backend: home_case_study_step
agent_wrapper: home_case_study
concurrency: 4
output_token_limit: 2500
# memory_limit_mib: 0 # Explicitly disable Linux cgroup memory limiting if needed.
```

View file

@ -1,5 +1,6 @@
"""Sequential case-local investigations using ReMe Jobs and sandboxed command tools."""
"""Concurrent case-local investigations using ReMe Jobs and sandboxed command tools."""
import asyncio
import json
import sys
from pathlib import Path
@ -276,7 +277,7 @@ class HomeCaseStudyStep(BaseStep):
agent_wrapper: BoundedAgentWrapper = Ref(BoundedAgentWrapper, "agent_wrapper")
async def execute(self):
"""Validate the whole batch, then isolate each sequential investigation and failure."""
"""Validate the whole batch, then run bounded concurrent, isolated investigations."""
if self.context.get("_allowed_paths") is not None:
raise ValueError("Home case study requires unrestricted access to its managed artifacts")
if not self.workspace.is_started or not self.agent_wrapper.is_started:
@ -297,6 +298,9 @@ class HomeCaseStudyStep(BaseStep):
output_limit = self.kwargs.get("output_token_limit", 2500)
if isinstance(output_limit, bool) or not isinstance(output_limit, int) or output_limit < 1:
raise ValueError("output_token_limit must be a positive integer")
concurrency = self.kwargs.get("concurrency", 4)
if isinstance(concurrency, bool) or not isinstance(concurrency, int) or concurrency < 1:
raise ValueError("concurrency must be a positive integer")
prompts = yaml.safe_load(PROMPT.read_text(encoding="utf-8"))
if not isinstance(prompts, dict) or any(
not isinstance(prompts.get(key), str) or not prompts[key].strip()
@ -315,22 +319,33 @@ class HomeCaseStudyStep(BaseStep):
response.metadata.update(validation_path=scopes[0]["validation_path"], case_id=spec.case_ids[0])
response.metadata.update({key: result[key] for key in ("session_id", "usage") if key in result})
return response
results = []
response.answer = {"results": results}
for scope in scopes:
try:
result = await self._investigate(scope, spec.focus, prompts, output_limit)
results.append(
{
semaphore = asyncio.Semaphore(concurrency)
async def investigate(scope):
async with semaphore:
try:
result = await self._investigate(scope, spec.focus, prompts, output_limit)
return {
"case_id": scope["case_id"],
"success": True,
"report": result["result"],
**{key: result[key] for key in ("session_id", "usage") if key in result},
},
)
except Exception as exc: # A failed investigator must not discard or cancel sibling reports.
self.logger.exception(f"Case-study investigation failed: case_id={scope['case_id']}")
results.append({"case_id": scope["case_id"], "success": False, "error": f"{type(exc).__name__}: {exc}"})
}
except Exception as exc: # One failed investigator must not cancel sibling reports.
self.logger.exception(f"Case-study investigation failed: case_id={scope['case_id']}")
return {"case_id": scope["case_id"], "success": False, "error": f"{type(exc).__name__}: {exc}"}
tasks = [asyncio.create_task(investigate(scope)) for scope in scopes]
try:
# Preserve input order even when investigators complete out of order.
results = await asyncio.gather(*tasks)
except asyncio.CancelledError:
# Include queued cases and wait for running agents/executors to finish cancellation cleanup.
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
raise
response.answer = {"results": results}
response.success = all(item["success"] for item in results)
# Failed or unreported calls have unknown usage; never present a partial sum as a full total.
response.metadata["usage"] = (

View file

@ -66,7 +66,7 @@ application_defaults:
jobs:
home_case_study:
backend: base
description: "Investigate up to eight ordered evaluation cases sequentially with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error"
description: "Investigate up to eight evaluation cases concurrently with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error in input order"
parameters:
type: object
additionalProperties: false
@ -92,6 +92,7 @@ application_defaults:
steps:
- backend: home_case_study_step
agent_wrapper: home_case_study
concurrency: 4
output_token_limit: 2500
home_trace_query_evidence:
backend: base

View file

@ -1,4 +1,4 @@
"""Single case investigations use normal ReMe lifecycle and sandboxed executors."""
"""Concurrent case investigations use normal ReMe lifecycle and sandboxed executors."""
# pylint: disable=missing-function-docstring,missing-class-docstring,redefined-outer-name
# pylint: disable=unused-argument,protected-access
@ -380,7 +380,8 @@ async def test_batch_shares_focus_but_isolates_agents_tools_and_sandboxes(app, p
assert response.metadata["usage"]["total_tokens"] == 20
assert len(model.calls) == 4
for i, case_id in enumerate(["case2", first]):
calls = model.calls[i * 2 : i * 2 + 2]
calls = [call for call in model.calls if f"Case: {case_id};" in str(call["messages"])]
assert len(calls) == 2
assert "Shared hypothesis" in str(calls[0]["messages"])
outputs = [b for m in calls[1]["messages"] for b in m.content if b.type == "tool_result"]
evidence = json.loads(outputs[0].output[0].text)
@ -420,16 +421,136 @@ async def test_entire_batch_is_validated_before_spending_model_tokens(app, batch
@pytest.mark.asyncio
async def test_batch_cancellation_propagates_without_starting_next_case(app, batch_run, sandbox, monkeypatch):
async def test_batch_child_cancellation_cleans_up_siblings(app, batch_run, sandbox, monkeypatch):
run, first, _query_id, _commit = batch_run
both_started = asyncio.Event()
cleaned = []
started = []
async def cancel(*args, **kwargs):
raise asyncio.CancelledError
async def cancel(self, scope, *args):
case_id = scope["case_id"]
started.append(case_id)
if len(started) == 2:
both_started.set()
try:
await both_started.wait()
if case_id == first:
raise asyncio.CancelledError
await asyncio.Event().wait()
finally:
await asyncio.sleep(0)
cleaned.append(case_id)
monkeypatch.setattr(app.context.components["agent_wrapper"]["home_case_study"], "reply", cancel)
monkeypatch.setattr(case_study.HomeCaseStudyStep, "_investigate", cancel)
with pytest.raises(asyncio.CancelledError):
await app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "case2"])
assert len(sandbox) == 1
await asyncio.wait_for(
app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "case2"]),
timeout=5,
)
assert set(cleaned) == {first, "case2"}
@pytest.fixture
def many_case_run(batch_run, case_run):
run, first, query_id, _commit = batch_run
source = case_run[-1]
manifest = json.loads((run / "manifest.json").read_text())
for case_id in ["case3", "case4", "case5", "case6"]:
case = json.loads(source.read_text())
case["case_id"] = case_id
write_json(source.parent / f"batch-{case_id}.json", case)
shutil.copytree(run / f"cases/{first}", run / f"cases/{case_id}")
manifest["case_ids"].append(case_id)
manifest["query_ids"][case_id] = [query_id]
write_json(run / "manifest.json", manifest)
return run, manifest["case_ids"]
@pytest.mark.parametrize("concurrency", [1, 2, 4])
@pytest.mark.asyncio
async def test_batch_bounds_concurrency_and_preserves_order(app, many_case_run, monkeypatch, concurrency):
run, case_ids = many_case_run
params = app.context.jobs["home_case_study"].step_specs[0][1]
assert params["concurrency"] == 4
params["concurrency"] = concurrency
started = asyncio.Queue()
gates = {case_id: asyncio.Event() for case_id in case_ids}
active = set()
completed = []
peak = 0
async def investigate(self, scope, focus, prompts, output_limit):
nonlocal peak
case_id = scope["case_id"]
active.add(case_id)
peak = max(peak, len(active))
started.put_nowait(case_id)
try:
await gates[case_id].wait()
completed.append(case_id)
if case_id == case_ids[1]:
raise RuntimeError("one investigator failed")
return {"result": f"report {case_id}"}
finally:
active.remove(case_id)
monkeypatch.setattr(case_study.HomeCaseStudyStep, "_investigate", investigate)
task = asyncio.create_task(app.run_job("home_case_study", evaluation_path=str(run), case_ids=case_ids))
try:
running = [await asyncio.wait_for(started.get(), timeout=5) for _ in range(concurrency)]
assert started.empty() and len(active) == concurrency
for index in range(len(case_ids)):
case_id = running.pop() # Finish newer cases first to force out-of-order completion.
gates[case_id].set()
if index < len(case_ids) - concurrency:
running.append(await asyncio.wait_for(started.get(), timeout=5))
response = await asyncio.wait_for(task, timeout=5)
finally:
task.cancel()
await asyncio.gather(task, return_exceptions=True)
assert peak == concurrency and not active
assert completed != case_ids if concurrency > 1 else completed == case_ids
assert not response.success and response.metadata["usage"] is None
results = response.answer["results"]
assert [item["case_id"] for item in results] == case_ids
assert "one investigator failed" in results[1]["error"]
assert all(item["success"] for index, item in enumerate(results) if index != 1)
@pytest.mark.asyncio
async def test_batch_cancellation_cleans_running_and_queued_cases(app, many_case_run, monkeypatch):
run, case_ids = many_case_run
started = asyncio.Queue()
cleaned = []
async def wait_for_cancel(self, scope, *args):
case_id = scope["case_id"]
started.put_nowait(case_id)
try:
await asyncio.Event().wait()
finally:
await asyncio.sleep(0)
cleaned.append(case_id)
monkeypatch.setattr(case_study.HomeCaseStudyStep, "_investigate", wait_for_cancel)
task = asyncio.create_task(app.run_job("home_case_study", evaluation_path=str(run), case_ids=case_ids))
try:
running = [await asyncio.wait_for(started.get(), timeout=5) for _ in range(4)]
finally:
task.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=5)
assert started.empty() and set(cleaned) == set(running)
@pytest.mark.parametrize("concurrency", [0, -1, True, 1.5, "4", None])
@pytest.mark.asyncio
async def test_invalid_concurrency_rejected_before_agents_start(app, batch_run, sandbox, concurrency):
run, first, _query_id, _commit = batch_run
app.context.jobs["home_case_study"].step_specs[0][1]["concurrency"] = concurrency
response = await app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "case2"])
assert not response.success and "concurrency must be a positive integer" in response.answer
assert not sandbox
@pytest.mark.parametrize(

View file

@ -64,8 +64,10 @@ def test_failed_macos_lookup_adds_no_grant(tmp_path, monkeypatch, result):
@pytest.mark.asyncio
async def test_linux_environment_is_inside_bubblewrap(tmp_path, monkeypatch):
@pytest.mark.parametrize("same_prefix", [True, False], ids=["same-prefix", "different-prefix"])
async def test_linux_environment_is_inside_bubblewrap(tmp_path, monkeypatch, same_prefix):
monkeypatch.setattr(sys, "platform", "linux")
monkeypatch.setattr(sys, "base_prefix", sys.prefix if same_prefix else str(tmp_path / "base-python"))
monkeypatch.setattr(sandbox, "create_file_sandbox_backend", PathBubbleBackend)
adapted = sandbox.create_case_study_sandbox(
cwd=tmp_path,
@ -84,9 +86,10 @@ async def test_linux_environment_is_inside_bubblewrap(tmp_path, monkeypatch):
assert "--clearenv" in argv and "--unshare-all" in argv
assert argv.index("--") < argv.index("/usr/bin/env")
assert "GIT_CONFIG_NOSYSTEM=1" in argv and "GIT_CONFIG_GLOBAL=/dev/null" in argv
assert adapted.environment["PATH"].split(":")[:2] == list(
python_paths = list(
dict.fromkeys([str(sandbox.Path(sys.prefix) / "bin"), str(sandbox.Path(sys.base_prefix) / "bin")]),
)
assert adapted.environment["PATH"].split(":")[: len(python_paths)] == python_paths
assert backend._writable_paths == []
assert await adapted.getcwd() == str(tmp_path)
backend.exec_shell.side_effect = asyncio.CancelledError