mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-16 09:21:50 +00:00
feat(home): run case studies with bounded concurrency and fix sandbox test
This commit is contained in:
parent
febc84586e
commit
4e1ceaee40
5 changed files with 173 additions and 28 deletions
|
|
@ -736,14 +736,18 @@ or through the case-study batch Job below.
|
|||
|
||||
`home_case_study(evaluation_path, case_ids, focus=...)` investigates up to eight
|
||||
unique case IDs in input order, sharing the evaluation and unchanged focus.
|
||||
Cases run sequentially, each with fresh agent state, sandbox and timer. The
|
||||
Cases run concurrently, each with fresh agent state, sandbox and timer. The
|
||||
host Step setting `concurrency` defaults to 4, matching dev_meta, and must be a
|
||||
positive integer; set it to 1 for sequential execution. Results preserve input
|
||||
order regardless of completion order. Cancelling the batch cancels running and
|
||||
queued cases and waits for their cleanup before returning. The
|
||||
whole selection is validated before model calls; a case execution failure is
|
||||
returned individually without cancelling later cases. It follows dev_meta's
|
||||
case-study workflow: trace judging and answer failures backward through retrieval,
|
||||
memory construction, source sessions and
|
||||
the evaluated implementation. The text report cites evidence paths/lines or
|
||||
JSON fields and distinguishes supported causes, hypotheses and missing evidence;
|
||||
it does not prescribe fixes. There is no parallel case-study dispatcher.
|
||||
it does not prescribe fixes.
|
||||
|
||||
The `home_case_study` component uses `home_case_study_agent`, a thin subclass of
|
||||
the bounded wrapper, defaulting to the dedicated `as_llm: home_case_study`
|
||||
|
|
@ -785,7 +789,7 @@ Linux requires Bubblewrap and, with the default 4096-MiB limit, a writable
|
|||
delegated cgroup v2. macOS requires Seatbelt (`sandbox-exec`); memory limits are
|
||||
not enforced there. The Job probes sandbox startup before calling the model and
|
||||
never falls back to an unrestricted executor. The command whitelist parser is
|
||||
not used by this agent. Sandbox policy, output-report token guidance and executor
|
||||
not used by this agent. Batch concurrency, sandbox policy, output-report token guidance and executor
|
||||
memory policy are host Step configuration, not model arguments:
|
||||
|
||||
```yaml
|
||||
|
|
@ -794,6 +798,7 @@ jobs:
|
|||
steps:
|
||||
- backend: home_case_study_step
|
||||
agent_wrapper: home_case_study
|
||||
concurrency: 4
|
||||
output_token_limit: 2500
|
||||
# memory_limit_mib: 0 # Explicitly disable Linux cgroup memory limiting if needed.
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
"""Sequential case-local investigations using ReMe Jobs and sandboxed command tools."""
|
||||
"""Concurrent case-local investigations using ReMe Jobs and sandboxed command tools."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
|
@ -276,7 +277,7 @@ class HomeCaseStudyStep(BaseStep):
|
|||
agent_wrapper: BoundedAgentWrapper = Ref(BoundedAgentWrapper, "agent_wrapper")
|
||||
|
||||
async def execute(self):
|
||||
"""Validate the whole batch, then isolate each sequential investigation and failure."""
|
||||
"""Validate the whole batch, then run bounded concurrent, isolated investigations."""
|
||||
if self.context.get("_allowed_paths") is not None:
|
||||
raise ValueError("Home case study requires unrestricted access to its managed artifacts")
|
||||
if not self.workspace.is_started or not self.agent_wrapper.is_started:
|
||||
|
|
@ -297,6 +298,9 @@ class HomeCaseStudyStep(BaseStep):
|
|||
output_limit = self.kwargs.get("output_token_limit", 2500)
|
||||
if isinstance(output_limit, bool) or not isinstance(output_limit, int) or output_limit < 1:
|
||||
raise ValueError("output_token_limit must be a positive integer")
|
||||
concurrency = self.kwargs.get("concurrency", 4)
|
||||
if isinstance(concurrency, bool) or not isinstance(concurrency, int) or concurrency < 1:
|
||||
raise ValueError("concurrency must be a positive integer")
|
||||
prompts = yaml.safe_load(PROMPT.read_text(encoding="utf-8"))
|
||||
if not isinstance(prompts, dict) or any(
|
||||
not isinstance(prompts.get(key), str) or not prompts[key].strip()
|
||||
|
|
@ -315,22 +319,33 @@ class HomeCaseStudyStep(BaseStep):
|
|||
response.metadata.update(validation_path=scopes[0]["validation_path"], case_id=spec.case_ids[0])
|
||||
response.metadata.update({key: result[key] for key in ("session_id", "usage") if key in result})
|
||||
return response
|
||||
results = []
|
||||
response.answer = {"results": results}
|
||||
for scope in scopes:
|
||||
try:
|
||||
result = await self._investigate(scope, spec.focus, prompts, output_limit)
|
||||
results.append(
|
||||
{
|
||||
semaphore = asyncio.Semaphore(concurrency)
|
||||
|
||||
async def investigate(scope):
|
||||
async with semaphore:
|
||||
try:
|
||||
result = await self._investigate(scope, spec.focus, prompts, output_limit)
|
||||
return {
|
||||
"case_id": scope["case_id"],
|
||||
"success": True,
|
||||
"report": result["result"],
|
||||
**{key: result[key] for key in ("session_id", "usage") if key in result},
|
||||
},
|
||||
)
|
||||
except Exception as exc: # A failed investigator must not discard or cancel sibling reports.
|
||||
self.logger.exception(f"Case-study investigation failed: case_id={scope['case_id']}")
|
||||
results.append({"case_id": scope["case_id"], "success": False, "error": f"{type(exc).__name__}: {exc}"})
|
||||
}
|
||||
except Exception as exc: # One failed investigator must not cancel sibling reports.
|
||||
self.logger.exception(f"Case-study investigation failed: case_id={scope['case_id']}")
|
||||
return {"case_id": scope["case_id"], "success": False, "error": f"{type(exc).__name__}: {exc}"}
|
||||
|
||||
tasks = [asyncio.create_task(investigate(scope)) for scope in scopes]
|
||||
try:
|
||||
# Preserve input order even when investigators complete out of order.
|
||||
results = await asyncio.gather(*tasks)
|
||||
except asyncio.CancelledError:
|
||||
# Include queued cases and wait for running agents/executors to finish cancellation cleanup.
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
raise
|
||||
response.answer = {"results": results}
|
||||
response.success = all(item["success"] for item in results)
|
||||
# Failed or unreported calls have unknown usage; never present a partial sum as a full total.
|
||||
response.metadata["usage"] = (
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ application_defaults:
|
|||
jobs:
|
||||
home_case_study:
|
||||
backend: base
|
||||
description: "Investigate up to eight ordered evaluation cases sequentially with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error"
|
||||
description: "Investigate up to eight evaluation cases concurrently with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error in input order"
|
||||
parameters:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
|
|
@ -92,6 +92,7 @@ application_defaults:
|
|||
steps:
|
||||
- backend: home_case_study_step
|
||||
agent_wrapper: home_case_study
|
||||
concurrency: 4
|
||||
output_token_limit: 2500
|
||||
home_trace_query_evidence:
|
||||
backend: base
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Single case investigations use normal ReMe lifecycle and sandboxed executors."""
|
||||
"""Concurrent case investigations use normal ReMe lifecycle and sandboxed executors."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,missing-class-docstring,redefined-outer-name
|
||||
# pylint: disable=unused-argument,protected-access
|
||||
|
|
@ -380,7 +380,8 @@ async def test_batch_shares_focus_but_isolates_agents_tools_and_sandboxes(app, p
|
|||
assert response.metadata["usage"]["total_tokens"] == 20
|
||||
assert len(model.calls) == 4
|
||||
for i, case_id in enumerate(["case2", first]):
|
||||
calls = model.calls[i * 2 : i * 2 + 2]
|
||||
calls = [call for call in model.calls if f"Case: {case_id};" in str(call["messages"])]
|
||||
assert len(calls) == 2
|
||||
assert "Shared hypothesis" in str(calls[0]["messages"])
|
||||
outputs = [b for m in calls[1]["messages"] for b in m.content if b.type == "tool_result"]
|
||||
evidence = json.loads(outputs[0].output[0].text)
|
||||
|
|
@ -420,16 +421,136 @@ async def test_entire_batch_is_validated_before_spending_model_tokens(app, batch
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_cancellation_propagates_without_starting_next_case(app, batch_run, sandbox, monkeypatch):
|
||||
async def test_batch_child_cancellation_cleans_up_siblings(app, batch_run, sandbox, monkeypatch):
|
||||
run, first, _query_id, _commit = batch_run
|
||||
both_started = asyncio.Event()
|
||||
cleaned = []
|
||||
started = []
|
||||
|
||||
async def cancel(*args, **kwargs):
|
||||
raise asyncio.CancelledError
|
||||
async def cancel(self, scope, *args):
|
||||
case_id = scope["case_id"]
|
||||
started.append(case_id)
|
||||
if len(started) == 2:
|
||||
both_started.set()
|
||||
try:
|
||||
await both_started.wait()
|
||||
if case_id == first:
|
||||
raise asyncio.CancelledError
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
await asyncio.sleep(0)
|
||||
cleaned.append(case_id)
|
||||
|
||||
monkeypatch.setattr(app.context.components["agent_wrapper"]["home_case_study"], "reply", cancel)
|
||||
monkeypatch.setattr(case_study.HomeCaseStudyStep, "_investigate", cancel)
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "case2"])
|
||||
assert len(sandbox) == 1
|
||||
await asyncio.wait_for(
|
||||
app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "case2"]),
|
||||
timeout=5,
|
||||
)
|
||||
assert set(cleaned) == {first, "case2"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def many_case_run(batch_run, case_run):
|
||||
run, first, query_id, _commit = batch_run
|
||||
source = case_run[-1]
|
||||
manifest = json.loads((run / "manifest.json").read_text())
|
||||
for case_id in ["case3", "case4", "case5", "case6"]:
|
||||
case = json.loads(source.read_text())
|
||||
case["case_id"] = case_id
|
||||
write_json(source.parent / f"batch-{case_id}.json", case)
|
||||
shutil.copytree(run / f"cases/{first}", run / f"cases/{case_id}")
|
||||
manifest["case_ids"].append(case_id)
|
||||
manifest["query_ids"][case_id] = [query_id]
|
||||
write_json(run / "manifest.json", manifest)
|
||||
return run, manifest["case_ids"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("concurrency", [1, 2, 4])
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_bounds_concurrency_and_preserves_order(app, many_case_run, monkeypatch, concurrency):
|
||||
run, case_ids = many_case_run
|
||||
params = app.context.jobs["home_case_study"].step_specs[0][1]
|
||||
assert params["concurrency"] == 4
|
||||
params["concurrency"] = concurrency
|
||||
started = asyncio.Queue()
|
||||
gates = {case_id: asyncio.Event() for case_id in case_ids}
|
||||
active = set()
|
||||
completed = []
|
||||
peak = 0
|
||||
|
||||
async def investigate(self, scope, focus, prompts, output_limit):
|
||||
nonlocal peak
|
||||
case_id = scope["case_id"]
|
||||
active.add(case_id)
|
||||
peak = max(peak, len(active))
|
||||
started.put_nowait(case_id)
|
||||
try:
|
||||
await gates[case_id].wait()
|
||||
completed.append(case_id)
|
||||
if case_id == case_ids[1]:
|
||||
raise RuntimeError("one investigator failed")
|
||||
return {"result": f"report {case_id}"}
|
||||
finally:
|
||||
active.remove(case_id)
|
||||
|
||||
monkeypatch.setattr(case_study.HomeCaseStudyStep, "_investigate", investigate)
|
||||
task = asyncio.create_task(app.run_job("home_case_study", evaluation_path=str(run), case_ids=case_ids))
|
||||
try:
|
||||
running = [await asyncio.wait_for(started.get(), timeout=5) for _ in range(concurrency)]
|
||||
assert started.empty() and len(active) == concurrency
|
||||
for index in range(len(case_ids)):
|
||||
case_id = running.pop() # Finish newer cases first to force out-of-order completion.
|
||||
gates[case_id].set()
|
||||
if index < len(case_ids) - concurrency:
|
||||
running.append(await asyncio.wait_for(started.get(), timeout=5))
|
||||
response = await asyncio.wait_for(task, timeout=5)
|
||||
finally:
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
assert peak == concurrency and not active
|
||||
assert completed != case_ids if concurrency > 1 else completed == case_ids
|
||||
assert not response.success and response.metadata["usage"] is None
|
||||
results = response.answer["results"]
|
||||
assert [item["case_id"] for item in results] == case_ids
|
||||
assert "one investigator failed" in results[1]["error"]
|
||||
assert all(item["success"] for index, item in enumerate(results) if index != 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_cancellation_cleans_running_and_queued_cases(app, many_case_run, monkeypatch):
|
||||
run, case_ids = many_case_run
|
||||
started = asyncio.Queue()
|
||||
cleaned = []
|
||||
|
||||
async def wait_for_cancel(self, scope, *args):
|
||||
case_id = scope["case_id"]
|
||||
started.put_nowait(case_id)
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
await asyncio.sleep(0)
|
||||
cleaned.append(case_id)
|
||||
|
||||
monkeypatch.setattr(case_study.HomeCaseStudyStep, "_investigate", wait_for_cancel)
|
||||
task = asyncio.create_task(app.run_job("home_case_study", evaluation_path=str(run), case_ids=case_ids))
|
||||
try:
|
||||
running = [await asyncio.wait_for(started.get(), timeout=5) for _ in range(4)]
|
||||
finally:
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await asyncio.wait_for(task, timeout=5)
|
||||
assert started.empty() and set(cleaned) == set(running)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("concurrency", [0, -1, True, 1.5, "4", None])
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_concurrency_rejected_before_agents_start(app, batch_run, sandbox, concurrency):
|
||||
run, first, _query_id, _commit = batch_run
|
||||
app.context.jobs["home_case_study"].step_specs[0][1]["concurrency"] = concurrency
|
||||
response = await app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "case2"])
|
||||
assert not response.success and "concurrency must be a positive integer" in response.answer
|
||||
assert not sandbox
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
|
|
@ -64,8 +64,10 @@ def test_failed_macos_lookup_adds_no_grant(tmp_path, monkeypatch, result):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_linux_environment_is_inside_bubblewrap(tmp_path, monkeypatch):
|
||||
@pytest.mark.parametrize("same_prefix", [True, False], ids=["same-prefix", "different-prefix"])
|
||||
async def test_linux_environment_is_inside_bubblewrap(tmp_path, monkeypatch, same_prefix):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
monkeypatch.setattr(sys, "base_prefix", sys.prefix if same_prefix else str(tmp_path / "base-python"))
|
||||
monkeypatch.setattr(sandbox, "create_file_sandbox_backend", PathBubbleBackend)
|
||||
adapted = sandbox.create_case_study_sandbox(
|
||||
cwd=tmp_path,
|
||||
|
|
@ -84,9 +86,10 @@ async def test_linux_environment_is_inside_bubblewrap(tmp_path, monkeypatch):
|
|||
assert "--clearenv" in argv and "--unshare-all" in argv
|
||||
assert argv.index("--") < argv.index("/usr/bin/env")
|
||||
assert "GIT_CONFIG_NOSYSTEM=1" in argv and "GIT_CONFIG_GLOBAL=/dev/null" in argv
|
||||
assert adapted.environment["PATH"].split(":")[:2] == list(
|
||||
python_paths = list(
|
||||
dict.fromkeys([str(sandbox.Path(sys.prefix) / "bin"), str(sandbox.Path(sys.base_prefix) / "bin")]),
|
||||
)
|
||||
assert adapted.environment["PATH"].split(":")[: len(python_paths)] == python_paths
|
||||
assert backend._writable_paths == []
|
||||
assert await adapted.getcwd() == str(tmp_path)
|
||||
backend.exec_shell.side_effect = asyncio.CancelledError
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue