From fc61507da5b1a06716727384a32a14dabb43b628 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 10:48:39 +0000 Subject: [PATCH] fix(eval): repair native containment checks --- eval/pyproject.toml | 1 + eval/tests/test_model_gateway.py | 31 ++++++++++++++++++++--- eval/tests/test_proposer_sandbox.py | 2 ++ eval/uv.lock | 8 +++--- eval/workflow_bench/gateway_supervisor.py | 5 +++- eval/workflow_bench/model_gateway.py | 4 +++ eval/workflow_bench/proposer_sandbox.py | 3 +++ 7 files changed, 47 insertions(+), 7 deletions(-) diff --git a/eval/pyproject.toml b/eval/pyproject.toml index 36e00876c..5c5deaf61 100644 --- a/eval/pyproject.toml +++ b/eval/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "litellm[proxy]!=1.82.7,!=1.82.8,>=1.99.0", "cryptography>=50.0.0", "python-multipart>=0.0.30", + "restrictedpython>=8.3", "datasets>=3.0.0", "typer>=0.12.0", "rich>=13.0.0", diff --git a/eval/tests/test_model_gateway.py b/eval/tests/test_model_gateway.py index e15891e7e..3553b1bcf 100644 --- a/eval/tests/test_model_gateway.py +++ b/eval/tests/test_model_gateway.py @@ -36,6 +36,26 @@ from workflow_bench.model_gateway import ( ) +def test_supervisor_reports_proxy_failure_without_aborting_on_its_stdin_reader(): + supervisor = Path(__file__).resolve().parents[1] / "workflow_bench" / "gateway_supervisor.py" + process = subprocess.Popen( + [sys.executable, str(supervisor), sys.executable, "-c", "raise RuntimeError('proxy failed')"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + # Keep the owner pipe open: the proxy exits independently of its owner. + process.wait(timeout=10) + assert process.returncode == 1 + assert b"proxy failed" in process.stderr.read() + finally: + process.stdin.close() + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + def test_locked_litellm_translates_messages_to_offline_responses(monkeypatch, tmp_path): from workflow_bench import model_gateway @@ -171,7 +191,9 @@ time.sleep(60) deadline = time.monotonic() + 12 while (not ready.exists() or not proxy_pid.exists()) and parent.poll() is None and time.monotonic() < deadline: time.sleep(0.02) - assert ready.exists(), parent.communicate(timeout=1) + if not ready.exists(): + _, stderr = parent.communicate(timeout=1) + pytest.fail(stderr) port = json.loads(ready.read_text())["port"] getattr(parent, termination)() parent.wait(timeout=5) @@ -226,7 +248,9 @@ def test_openai_litellm_config_routes_each_id_to_openai_and_env_key(tmp_path: Pa assert config["litellm_settings"]["request_timeout"] == GATEWAY_REQUEST_TIMEOUT_S path = write_openai_litellm_config(tmp_path / "litellm.yaml", ["gpt-4.1"]) assert yaml.safe_load(path.read_text())["model_list"][0]["model_name"] == "gpt-4.1" - assert path.stat().st_mode & 0o777 == 0o600 + if os.name != "nt": + # Windows chmod exposes a read-only flag, not POSIX access bits. + assert path.stat().st_mode & 0o777 == 0o600 def test_resolve_model_access_starts_proxy_only_for_openai_ids() -> None: @@ -311,7 +335,8 @@ def test_openai_gateway_never_leaves_proxy_output_on_an_undrained_pipe(tmp_path: assert captured["stderr"] is subprocess.STDOUT assert captured["stdout"] is not subprocess.PIPE assert getattr(captured["stdout"], "name", "") == str(gateway.log_path) - assert gateway.log_path.stat().st_mode & 0o777 == 0o600 + if os.name != "nt": + assert gateway.log_path.stat().st_mode & 0o777 == 0o600 def test_gateway_startup_budget_outlives_a_cold_litellm_import(monkeypatch, tmp_path: Path) -> None: diff --git a/eval/tests/test_proposer_sandbox.py b/eval/tests/test_proposer_sandbox.py index c2d11c05a..2e30cf6a2 100644 --- a/eval/tests/test_proposer_sandbox.py +++ b/eval/tests/test_proposer_sandbox.py @@ -94,6 +94,8 @@ def test_review_preparation_preserves_existing_runtime_files_and_tracks_only_cre created = json.loads((sandbox.private_root / "review-created-paths.json").read_text()) assert "bunfig.toml" not in created assert ".npmrc" in created + assert ".mcp.json" in created + assert json.loads((clone / ".mcp.json").read_text()) == {} assert (clone / "bunfig.toml").read_text() == "existing configuration\n" assert (clone / ".git/commondir").read_text() == ".\n" diff --git a/eval/uv.lock b/eval/uv.lock index 0c794861b..45c13a3ef 100644 --- a/eval/uv.lock +++ b/eval/uv.lock @@ -978,6 +978,7 @@ dependencies = [ { name = "python-dotenv" }, { name = "python-multipart" }, { name = "pyyaml" }, + { name = "restrictedpython" }, { name = "rich" }, { name = "tabulate" }, { name = "typer" }, @@ -1004,6 +1005,7 @@ requires-dist = [ { name = "python-dotenv", specifier = ">=1.2.2" }, { name = "python-multipart", specifier = ">=0.0.30" }, { name = "pyyaml", specifier = ">=6.0" }, + { name = "restrictedpython", specifier = ">=8.3" }, { name = "rich", specifier = ">=13.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5.0" }, { name = "tabulate", specifier = ">=0.9.0" }, @@ -2851,11 +2853,11 @@ wheels = [ [[package]] name = "restrictedpython" -version = "8.1" +version = "8.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/1c/aec08bcb4ab14a1521579fbe21ceff2a634bb1f737f11cf7f9c8bb96e680/restrictedpython-8.1.tar.gz", hash = "sha256:4a69304aceacf6bee74bdf153c728221d4e3109b39acbfe00b3494927080d898", size = 838331, upload-time = "2025-10-19T14:11:32.531Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/3b/8e41f7cfabbb30b1013ebc7484303d6c87da2906ec432d69dea11d2f7d75/restrictedpython-8.5.tar.gz", hash = "sha256:4ed1269dbe3caa88db650d1af325198a952aeb1451eca05df0cfa65db4466215", size = 455879, upload-time = "2026-08-19T07:02:10.934Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/c0/3848f4006f7e164ee20833ca984067e4b3fc99fe7f1dfa88b4927e681299/restrictedpython-8.1-py3-none-any.whl", hash = "sha256:4769449c6cdb10f2071649ba386902befff0eff2a8fd6217989fa7b16aeae926", size = 27651, upload-time = "2025-10-19T14:11:30.201Z" }, + { url = "https://files.pythonhosted.org/packages/58/57/16ce3c721f5a33317e4110575d5c9976c0c45f7fd96ca2e0adeab06e6026/restrictedpython-8.5-py3-none-any.whl", hash = "sha256:6c70e0a3af13e830d37225788cdc8ab5804a8df4b500c135086eaef34b5c01e0", size = 30962, upload-time = "2026-08-19T07:02:09.553Z" }, ] [[package]] diff --git a/eval/workflow_bench/gateway_supervisor.py b/eval/workflow_bench/gateway_supervisor.py index 6a3d844dc..422a38bb5 100644 --- a/eval/workflow_bench/gateway_supervisor.py +++ b/eval/workflow_bench/gateway_supervisor.py @@ -6,6 +6,7 @@ PYTHONPATH. The gateway receives DEVNULL, never the owner-liveness descriptor. from __future__ import annotations +import os import signal import sys import threading @@ -18,7 +19,9 @@ def main() -> int: def watch_owner() -> None: try: - sys.stdin.buffer.read(1) + # A buffered stdin lock held by a daemon aborts CPython shutdown + # when the proxy exits while the owner is still alive. + os.read(sys.stdin.fileno(), 1) finally: cancelled.set() diff --git a/eval/workflow_bench/model_gateway.py b/eval/workflow_bench/model_gateway.py index 01826d6f2..203aed377 100644 --- a/eval/workflow_bench/model_gateway.py +++ b/eval/workflow_bench/model_gateway.py @@ -299,6 +299,10 @@ class OpenAIGateway(AbstractContextManager["OpenAIGateway"]): "OPENAI_API_KEY": self.openai_api_key, "LITELLM_MASTER_KEY": self.auth_token, } + if os.name == "nt": + # Windows subprocess DLL/socket initialization needs SystemRoot. + # Keep the rest of the gateway's credential boundary explicit. + env["SystemRoot"] = os.environ["SystemRoot"] # Never hand the proxy a pipe: nothing drains it after startup, so the # proxy would block forever once its request logs fill the 64 KiB pipe # buffer, and every later session request would hang without a status. diff --git a/eval/workflow_bench/proposer_sandbox.py b/eval/workflow_bench/proposer_sandbox.py index dd329e9bd..a2dfaa54a 100644 --- a/eval/workflow_bench/proposer_sandbox.py +++ b/eval/workflow_bench/proposer_sandbox.py @@ -73,6 +73,7 @@ class SandboxError(RuntimeError): # paths even when absent. Mount targets must exist before sealing the clone. REVIEW_RUNTIME_FILES = ( "bunfig.toml", + ".mcp.json", "package.json", ".npmrc", ".yarnrc", @@ -157,6 +158,8 @@ def prepare_review_workspace(sandbox: SandboxSession, artifact_name: str) -> Pat missing = not os.path.lexists(clone / name) _prepare_clone_target(clone, PurePosixPath(name), directory=directory, label="review runtime") if missing: + if name == ".mcp.json": + (clone / name).write_text("{}\n") created.append(name) common_dir = clone / ".git/commondir" missing = not os.path.lexists(common_dir)