diff --git a/plugins/home/README.md b/plugins/home/README.md index fc8edcc1..25b0767a 100644 --- a/plugins/home/README.md +++ b/plugins/home/README.md @@ -736,14 +736,18 @@ or through the case-study batch Job below. `home_case_study(evaluation_path, case_ids, focus=...)` investigates up to eight unique case IDs in input order, sharing the evaluation and unchanged focus. -Cases run sequentially, each with fresh agent state, sandbox and timer. The +Cases run concurrently, each with fresh agent state, sandbox and timer. The +host Step setting `concurrency` defaults to 4, matching dev_meta, and must be a +positive integer; set it to 1 for sequential execution. Results preserve input +order regardless of completion order. Cancelling the batch cancels running and +queued cases and waits for their cleanup before returning. The whole selection is validated before model calls; a case execution failure is returned individually without cancelling later cases. It follows dev_meta's case-study workflow: trace judging and answer failures backward through retrieval, memory construction, source sessions and the evaluated implementation. The text report cites evidence paths/lines or JSON fields and distinguishes supported causes, hypotheses and missing evidence; -it does not prescribe fixes. There is no parallel case-study dispatcher. +it does not prescribe fixes. The `home_case_study` component uses `home_case_study_agent`, a thin subclass of the bounded wrapper, defaulting to the dedicated `as_llm: home_case_study` @@ -785,7 +789,7 @@ Linux requires Bubblewrap and, with the default 4096-MiB limit, a writable delegated cgroup v2. macOS requires Seatbelt (`sandbox-exec`); memory limits are not enforced there. The Job probes sandbox startup before calling the model and never falls back to an unrestricted executor. The command whitelist parser is -not used by this agent. Sandbox policy, output-report token guidance and executor +not used by this agent. Batch concurrency, sandbox policy, output-report token guidance and executor memory policy are host Step configuration, not model arguments: ```yaml @@ -794,6 +798,7 @@ jobs: steps: - backend: home_case_study_step agent_wrapper: home_case_study + concurrency: 4 output_token_limit: 2500 # memory_limit_mib: 0 # Explicitly disable Linux cgroup memory limiting if needed. ``` diff --git a/plugins/home/src/reme_home/agent/case_study.py b/plugins/home/src/reme_home/agent/case_study.py index a8256d8b..e8b1c652 100644 --- a/plugins/home/src/reme_home/agent/case_study.py +++ b/plugins/home/src/reme_home/agent/case_study.py @@ -1,5 +1,6 @@ -"""Sequential case-local investigations using ReMe Jobs and sandboxed command tools.""" +"""Concurrent case-local investigations using ReMe Jobs and sandboxed command tools.""" +import asyncio import json import sys from pathlib import Path @@ -276,7 +277,7 @@ class HomeCaseStudyStep(BaseStep): agent_wrapper: BoundedAgentWrapper = Ref(BoundedAgentWrapper, "agent_wrapper") async def execute(self): - """Validate the whole batch, then isolate each sequential investigation and failure.""" + """Validate the whole batch, then run bounded concurrent, isolated investigations.""" if self.context.get("_allowed_paths") is not None: raise ValueError("Home case study requires unrestricted access to its managed artifacts") if not self.workspace.is_started or not self.agent_wrapper.is_started: @@ -297,6 +298,9 @@ class HomeCaseStudyStep(BaseStep): output_limit = self.kwargs.get("output_token_limit", 2500) if isinstance(output_limit, bool) or not isinstance(output_limit, int) or output_limit < 1: raise ValueError("output_token_limit must be a positive integer") + concurrency = self.kwargs.get("concurrency", 4) + if isinstance(concurrency, bool) or not isinstance(concurrency, int) or concurrency < 1: + raise ValueError("concurrency must be a positive integer") prompts = yaml.safe_load(PROMPT.read_text(encoding="utf-8")) if not isinstance(prompts, dict) or any( not isinstance(prompts.get(key), str) or not prompts[key].strip() @@ -315,22 +319,33 @@ class HomeCaseStudyStep(BaseStep): response.metadata.update(validation_path=scopes[0]["validation_path"], case_id=spec.case_ids[0]) response.metadata.update({key: result[key] for key in ("session_id", "usage") if key in result}) return response - results = [] - response.answer = {"results": results} - for scope in scopes: - try: - result = await self._investigate(scope, spec.focus, prompts, output_limit) - results.append( - { + semaphore = asyncio.Semaphore(concurrency) + + async def investigate(scope): + async with semaphore: + try: + result = await self._investigate(scope, spec.focus, prompts, output_limit) + return { "case_id": scope["case_id"], "success": True, "report": result["result"], **{key: result[key] for key in ("session_id", "usage") if key in result}, - }, - ) - except Exception as exc: # A failed investigator must not discard or cancel sibling reports. - self.logger.exception(f"Case-study investigation failed: case_id={scope['case_id']}") - results.append({"case_id": scope["case_id"], "success": False, "error": f"{type(exc).__name__}: {exc}"}) + } + except Exception as exc: # One failed investigator must not cancel sibling reports. + self.logger.exception(f"Case-study investigation failed: case_id={scope['case_id']}") + return {"case_id": scope["case_id"], "success": False, "error": f"{type(exc).__name__}: {exc}"} + + tasks = [asyncio.create_task(investigate(scope)) for scope in scopes] + try: + # Preserve input order even when investigators complete out of order. + results = await asyncio.gather(*tasks) + except asyncio.CancelledError: + # Include queued cases and wait for running agents/executors to finish cancellation cleanup. + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + raise + response.answer = {"results": results} response.success = all(item["success"] for item in results) # Failed or unreported calls have unknown usage; never present a partial sum as a full total. response.metadata["usage"] = ( diff --git a/plugins/home/src/reme_home/plugin.yaml b/plugins/home/src/reme_home/plugin.yaml index 30ea4f95..0d2c3b74 100644 --- a/plugins/home/src/reme_home/plugin.yaml +++ b/plugins/home/src/reme_home/plugin.yaml @@ -66,7 +66,7 @@ application_defaults: jobs: home_case_study: backend: base - description: "Investigate up to eight ordered evaluation cases sequentially with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error" + description: "Investigate up to eight evaluation cases concurrently with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error in input order" parameters: type: object additionalProperties: false @@ -92,6 +92,7 @@ application_defaults: steps: - backend: home_case_study_step agent_wrapper: home_case_study + concurrency: 4 output_token_limit: 2500 home_trace_query_evidence: backend: base diff --git a/plugins/home/tests/test_case_study.py b/plugins/home/tests/test_case_study.py index 466e63a7..36effc0e 100644 --- a/plugins/home/tests/test_case_study.py +++ b/plugins/home/tests/test_case_study.py @@ -1,4 +1,4 @@ -"""Single case investigations use normal ReMe lifecycle and sandboxed executors.""" +"""Concurrent case investigations use normal ReMe lifecycle and sandboxed executors.""" # pylint: disable=missing-function-docstring,missing-class-docstring,redefined-outer-name # pylint: disable=unused-argument,protected-access @@ -380,7 +380,8 @@ async def test_batch_shares_focus_but_isolates_agents_tools_and_sandboxes(app, p assert response.metadata["usage"]["total_tokens"] == 20 assert len(model.calls) == 4 for i, case_id in enumerate(["case2", first]): - calls = model.calls[i * 2 : i * 2 + 2] + calls = [call for call in model.calls if f"Case: {case_id};" in str(call["messages"])] + assert len(calls) == 2 assert "Shared hypothesis" in str(calls[0]["messages"]) outputs = [b for m in calls[1]["messages"] for b in m.content if b.type == "tool_result"] evidence = json.loads(outputs[0].output[0].text) @@ -420,16 +421,136 @@ async def test_entire_batch_is_validated_before_spending_model_tokens(app, batch @pytest.mark.asyncio -async def test_batch_cancellation_propagates_without_starting_next_case(app, batch_run, sandbox, monkeypatch): +async def test_batch_child_cancellation_cleans_up_siblings(app, batch_run, sandbox, monkeypatch): run, first, _query_id, _commit = batch_run + both_started = asyncio.Event() + cleaned = [] + started = [] - async def cancel(*args, **kwargs): - raise asyncio.CancelledError + async def cancel(self, scope, *args): + case_id = scope["case_id"] + started.append(case_id) + if len(started) == 2: + both_started.set() + try: + await both_started.wait() + if case_id == first: + raise asyncio.CancelledError + await asyncio.Event().wait() + finally: + await asyncio.sleep(0) + cleaned.append(case_id) - monkeypatch.setattr(app.context.components["agent_wrapper"]["home_case_study"], "reply", cancel) + monkeypatch.setattr(case_study.HomeCaseStudyStep, "_investigate", cancel) with pytest.raises(asyncio.CancelledError): - await app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "case2"]) - assert len(sandbox) == 1 + await asyncio.wait_for( + app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "case2"]), + timeout=5, + ) + assert set(cleaned) == {first, "case2"} + + +@pytest.fixture +def many_case_run(batch_run, case_run): + run, first, query_id, _commit = batch_run + source = case_run[-1] + manifest = json.loads((run / "manifest.json").read_text()) + for case_id in ["case3", "case4", "case5", "case6"]: + case = json.loads(source.read_text()) + case["case_id"] = case_id + write_json(source.parent / f"batch-{case_id}.json", case) + shutil.copytree(run / f"cases/{first}", run / f"cases/{case_id}") + manifest["case_ids"].append(case_id) + manifest["query_ids"][case_id] = [query_id] + write_json(run / "manifest.json", manifest) + return run, manifest["case_ids"] + + +@pytest.mark.parametrize("concurrency", [1, 2, 4]) +@pytest.mark.asyncio +async def test_batch_bounds_concurrency_and_preserves_order(app, many_case_run, monkeypatch, concurrency): + run, case_ids = many_case_run + params = app.context.jobs["home_case_study"].step_specs[0][1] + assert params["concurrency"] == 4 + params["concurrency"] = concurrency + started = asyncio.Queue() + gates = {case_id: asyncio.Event() for case_id in case_ids} + active = set() + completed = [] + peak = 0 + + async def investigate(self, scope, focus, prompts, output_limit): + nonlocal peak + case_id = scope["case_id"] + active.add(case_id) + peak = max(peak, len(active)) + started.put_nowait(case_id) + try: + await gates[case_id].wait() + completed.append(case_id) + if case_id == case_ids[1]: + raise RuntimeError("one investigator failed") + return {"result": f"report {case_id}"} + finally: + active.remove(case_id) + + monkeypatch.setattr(case_study.HomeCaseStudyStep, "_investigate", investigate) + task = asyncio.create_task(app.run_job("home_case_study", evaluation_path=str(run), case_ids=case_ids)) + try: + running = [await asyncio.wait_for(started.get(), timeout=5) for _ in range(concurrency)] + assert started.empty() and len(active) == concurrency + for index in range(len(case_ids)): + case_id = running.pop() # Finish newer cases first to force out-of-order completion. + gates[case_id].set() + if index < len(case_ids) - concurrency: + running.append(await asyncio.wait_for(started.get(), timeout=5)) + response = await asyncio.wait_for(task, timeout=5) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert peak == concurrency and not active + assert completed != case_ids if concurrency > 1 else completed == case_ids + assert not response.success and response.metadata["usage"] is None + results = response.answer["results"] + assert [item["case_id"] for item in results] == case_ids + assert "one investigator failed" in results[1]["error"] + assert all(item["success"] for index, item in enumerate(results) if index != 1) + + +@pytest.mark.asyncio +async def test_batch_cancellation_cleans_running_and_queued_cases(app, many_case_run, monkeypatch): + run, case_ids = many_case_run + started = asyncio.Queue() + cleaned = [] + + async def wait_for_cancel(self, scope, *args): + case_id = scope["case_id"] + started.put_nowait(case_id) + try: + await asyncio.Event().wait() + finally: + await asyncio.sleep(0) + cleaned.append(case_id) + + monkeypatch.setattr(case_study.HomeCaseStudyStep, "_investigate", wait_for_cancel) + task = asyncio.create_task(app.run_job("home_case_study", evaluation_path=str(run), case_ids=case_ids)) + try: + running = [await asyncio.wait_for(started.get(), timeout=5) for _ in range(4)] + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=5) + assert started.empty() and set(cleaned) == set(running) + + +@pytest.mark.parametrize("concurrency", [0, -1, True, 1.5, "4", None]) +@pytest.mark.asyncio +async def test_invalid_concurrency_rejected_before_agents_start(app, batch_run, sandbox, concurrency): + run, first, _query_id, _commit = batch_run + app.context.jobs["home_case_study"].step_specs[0][1]["concurrency"] = concurrency + response = await app.run_job("home_case_study", evaluation_path=str(run), case_ids=[first, "case2"]) + assert not response.success and "concurrency must be a positive integer" in response.answer + assert not sandbox @pytest.mark.parametrize( diff --git a/plugins/home/tests/test_case_study_sandbox.py b/plugins/home/tests/test_case_study_sandbox.py index a1fe57ad..b6e3f014 100644 --- a/plugins/home/tests/test_case_study_sandbox.py +++ b/plugins/home/tests/test_case_study_sandbox.py @@ -64,8 +64,10 @@ def test_failed_macos_lookup_adds_no_grant(tmp_path, monkeypatch, result): @pytest.mark.asyncio -async def test_linux_environment_is_inside_bubblewrap(tmp_path, monkeypatch): +@pytest.mark.parametrize("same_prefix", [True, False], ids=["same-prefix", "different-prefix"]) +async def test_linux_environment_is_inside_bubblewrap(tmp_path, monkeypatch, same_prefix): monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(sys, "base_prefix", sys.prefix if same_prefix else str(tmp_path / "base-python")) monkeypatch.setattr(sandbox, "create_file_sandbox_backend", PathBubbleBackend) adapted = sandbox.create_case_study_sandbox( cwd=tmp_path, @@ -84,9 +86,10 @@ async def test_linux_environment_is_inside_bubblewrap(tmp_path, monkeypatch): assert "--clearenv" in argv and "--unshare-all" in argv assert argv.index("--") < argv.index("/usr/bin/env") assert "GIT_CONFIG_NOSYSTEM=1" in argv and "GIT_CONFIG_GLOBAL=/dev/null" in argv - assert adapted.environment["PATH"].split(":")[:2] == list( + python_paths = list( dict.fromkeys([str(sandbox.Path(sys.prefix) / "bin"), str(sandbox.Path(sys.base_prefix) / "bin")]), ) + assert adapted.environment["PATH"].split(":")[: len(python_paths)] == python_paths assert backend._writable_paths == [] assert await adapted.getcwd() == str(tmp_path) backend.exec_shell.side_effect = asyncio.CancelledError