fix(eval): repair native containment checks

eval/pyproject.toml

@@ -9,6 +9,7 @@ dependencies = [
     "litellm[proxy]!=1.82.7,!=1.82.8,>=1.99.0",
     "cryptography>=50.0.0",
     "python-multipart>=0.0.30",
+    "restrictedpython>=8.3",
     "datasets>=3.0.0",
     "typer>=0.12.0",
     "rich>=13.0.0",

eval/tests/test_model_gateway.py

@@ -36,6 +36,26 @@ from workflow_bench.model_gateway import (
 )
 
 
+def test_supervisor_reports_proxy_failure_without_aborting_on_its_stdin_reader():
+    supervisor = Path(__file__).resolve().parents[1] / "workflow_bench" / "gateway_supervisor.py"
+    process = subprocess.Popen(
+        [sys.executable, str(supervisor), sys.executable, "-c", "raise RuntimeError('proxy failed')"],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    try:
+        # Keep the owner pipe open: the proxy exits independently of its owner.
+        process.wait(timeout=10)
+        assert process.returncode == 1
+        assert b"proxy failed" in process.stderr.read()
+    finally:
+        process.stdin.close()
+        if process.poll() is None:
+            process.kill()
+        process.wait(timeout=5)
+
+
 def test_locked_litellm_translates_messages_to_offline_responses(monkeypatch, tmp_path):
     from workflow_bench import model_gateway
 
@@ -171,7 +191,9 @@ time.sleep(60)
         deadline = time.monotonic() + 12
         while (not ready.exists() or not proxy_pid.exists()) and parent.poll() is None and time.monotonic() < deadline:
             time.sleep(0.02)
-        assert ready.exists(), parent.communicate(timeout=1)
+        if not ready.exists():
+            _, stderr = parent.communicate(timeout=1)
+            pytest.fail(stderr)
         port = json.loads(ready.read_text())["port"]
         getattr(parent, termination)()
         parent.wait(timeout=5)
@@ -226,7 +248,9 @@ def test_openai_litellm_config_routes_each_id_to_openai_and_env_key(tmp_path: Pa
     assert config["litellm_settings"]["request_timeout"] == GATEWAY_REQUEST_TIMEOUT_S
     path = write_openai_litellm_config(tmp_path / "litellm.yaml", ["gpt-4.1"])
     assert yaml.safe_load(path.read_text())["model_list"][0]["model_name"] == "gpt-4.1"
-    assert path.stat().st_mode & 0o777 == 0o600
+    if os.name != "nt":
+        # Windows chmod exposes a read-only flag, not POSIX access bits.
+        assert path.stat().st_mode & 0o777 == 0o600
 
 
 def test_resolve_model_access_starts_proxy_only_for_openai_ids() -> None:
@@ -311,7 +335,8 @@ def test_openai_gateway_never_leaves_proxy_output_on_an_undrained_pipe(tmp_path:
     assert captured["stderr"] is subprocess.STDOUT
     assert captured["stdout"] is not subprocess.PIPE
     assert getattr(captured["stdout"], "name", "") == str(gateway.log_path)
-    assert gateway.log_path.stat().st_mode & 0o777 == 0o600
+    if os.name != "nt":
+        assert gateway.log_path.stat().st_mode & 0o777 == 0o600
 
 
 def test_gateway_startup_budget_outlives_a_cold_litellm_import(monkeypatch, tmp_path: Path) -> None:

eval/tests/test_proposer_sandbox.py

@@ -94,6 +94,8 @@ def test_review_preparation_preserves_existing_runtime_files_and_tracks_only_cre
         created = json.loads((sandbox.private_root / "review-created-paths.json").read_text())
         assert "bunfig.toml" not in created
         assert ".npmrc" in created
+        assert ".mcp.json" in created
+        assert json.loads((clone / ".mcp.json").read_text()) == {}
         assert (clone / "bunfig.toml").read_text() == "existing configuration\n"
         assert (clone / ".git/commondir").read_text() == ".\n"
 

eval/uv.lock

@@ -978,6 +978,7 @@ dependencies = [
     { name = "python-dotenv" },
     { name = "python-multipart" },
     { name = "pyyaml" },
+    { name = "restrictedpython" },
     { name = "rich" },
     { name = "tabulate" },
     { name = "typer" },
@@ -1004,6 +1005,7 @@ requires-dist = [
     { name = "python-dotenv", specifier = ">=1.2.2" },
     { name = "python-multipart", specifier = ">=0.0.30" },
     { name = "pyyaml", specifier = ">=6.0" },
+    { name = "restrictedpython", specifier = ">=8.3" },
     { name = "rich", specifier = ">=13.0.0" },
     { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5.0" },
     { name = "tabulate", specifier = ">=0.9.0" },
@@ -2851,11 +2853,11 @@ wheels = [
 
 [[package]]
 name = "restrictedpython"
-version = "8.1"
+version = "8.5"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/5f/1c/aec08bcb4ab14a1521579fbe21ceff2a634bb1f737f11cf7f9c8bb96e680/restrictedpython-8.1.tar.gz", hash = "sha256:4a69304aceacf6bee74bdf153c728221d4e3109b39acbfe00b3494927080d898", size = 838331, upload-time = "2025-10-19T14:11:32.531Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/7c/3b/8e41f7cfabbb30b1013ebc7484303d6c87da2906ec432d69dea11d2f7d75/restrictedpython-8.5.tar.gz", hash = "sha256:4ed1269dbe3caa88db650d1af325198a952aeb1451eca05df0cfa65db4466215", size = 455879, upload-time = "2026-08-19T07:02:10.934Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/1a/c0/3848f4006f7e164ee20833ca984067e4b3fc99fe7f1dfa88b4927e681299/restrictedpython-8.1-py3-none-any.whl", hash = "sha256:4769449c6cdb10f2071649ba386902befff0eff2a8fd6217989fa7b16aeae926", size = 27651, upload-time = "2025-10-19T14:11:30.201Z" },
+    { url = "https://files.pythonhosted.org/packages/58/57/16ce3c721f5a33317e4110575d5c9976c0c45f7fd96ca2e0adeab06e6026/restrictedpython-8.5-py3-none-any.whl", hash = "sha256:6c70e0a3af13e830d37225788cdc8ab5804a8df4b500c135086eaef34b5c01e0", size = 30962, upload-time = "2026-08-19T07:02:09.553Z" },
 ]
 
 [[package]]

eval/workflow_bench/gateway_supervisor.py

@@ -6,6 +6,7 @@ PYTHONPATH. The gateway receives DEVNULL, never the owner-liveness descriptor.
 
 from __future__ import annotations
 
+import os
 import signal
 import sys
 import threading
@@ -18,7 +19,9 @@ def main() -> int:
 
     def watch_owner() -> None:
         try:
-            sys.stdin.buffer.read(1)
+            # A buffered stdin lock held by a daemon aborts CPython shutdown
+            # when the proxy exits while the owner is still alive.
+            os.read(sys.stdin.fileno(), 1)
         finally:
             cancelled.set()
 

eval/workflow_bench/model_gateway.py

@@ -299,6 +299,10 @@ class OpenAIGateway(AbstractContextManager["OpenAIGateway"]):
             "OPENAI_API_KEY": self.openai_api_key,
             "LITELLM_MASTER_KEY": self.auth_token,
         }
+        if os.name == "nt":
+            # Windows subprocess DLL/socket initialization needs SystemRoot.
+            # Keep the rest of the gateway's credential boundary explicit.
+            env["SystemRoot"] = os.environ["SystemRoot"]
         # Never hand the proxy a pipe: nothing drains it after startup, so the
         # proxy would block forever once its request logs fill the 64 KiB pipe
         # buffer, and every later session request would hang without a status.

eval/workflow_bench/proposer_sandbox.py

@@ -73,6 +73,7 @@ class SandboxError(RuntimeError):
 # paths even when absent. Mount targets must exist before sealing the clone.
 REVIEW_RUNTIME_FILES = (
     "bunfig.toml",
+    ".mcp.json",
     "package.json",
     ".npmrc",
     ".yarnrc",
@@ -157,6 +158,8 @@ def prepare_review_workspace(sandbox: SandboxSession, artifact_name: str) -> Pat
             missing = not os.path.lexists(clone / name)
             _prepare_clone_target(clone, PurePosixPath(name), directory=directory, label="review runtime")
             if missing:
+                if name == ".mcp.json":
+                    (clone / name).write_text("{}\n")
                 created.append(name)
     common_dir = clone / ".git/commondir"
     missing = not os.path.lexists(common_dir)