diff --git a/litellm/proxy/mcp_server/README.md b/litellm/proxy/mcp_server/README.md index e57231eee74..cec4e3b8893 100644 --- a/litellm/proxy/mcp_server/README.md +++ b/litellm/proxy/mcp_server/README.md @@ -1,6 +1,12 @@ # MCP code mode -Code mode lets an MCP client run JavaScript that combines tool calls and returns only the output it needs. Enable it in the gateway configuration and restart the proxy. The QuickJS-NG runtime is included in the proxy installation on platforms with supported wheels; no separate sandbox service or client integration is required. +Code mode lets an MCP client run JavaScript that combines tool calls and returns only the output it needs. It is off by default. Install the optional runtime only in deployments that will enable it; regular proxy installations do not include it. + +```sh +pip install "litellm[proxy,mcp-code-mode]" +``` + +Keep your existing LiteLLM version pin when adding the extra to a deployed proxy. For containers, include the extra in the image build. Then enable it in the gateway configuration and restart. No separate sandbox service or client integration is required, and the gateway never installs packages at runtime. ```yaml litellm_settings: @@ -9,7 +15,7 @@ litellm_settings: Reconnect the MCP client or refresh its tool list. It should discover `litellm_execute_code` alongside its existing tools. The setting is off by default. Turning it off also rejects calls from clients that cached the code tool. -Intel macOS and Windows ARM installations need a manual source installation of `quickjs-ng` before enabling code mode. A missing runtime produces a startup error. The original `quickjs` package conflicts with `quickjs-ng`; use an environment without that package. These requirements do not affect the gateway when code mode is disabled. +The extra installs QuickJS-NG. Platforms without a compatible wheel need a source build and its compiler toolchain; these builds are not part of the verified platform matrix. A missing runtime produces an actionable startup error. The original `quickjs` package shares files with `quickjs-ng`; use a separate environment without that package before installing the extra. Regular proxy installations and disabled gateways do not require either runtime. The code tool accepts a `code` string. Discover a tool and its argument schema normally, then copy the code-mode `tool_id` from its description or `search_tools` into `tools.call_tool`. Copy that reference unchanged: clients can rename the displayed tool. Exact gateway catalog names also work. Calls return the full MCP result, including `content`, `structuredContent` when provided, and `isError`. Use `text(value)` to return a JSON value to the model. For example, after discovering a tool named `example-add` that accepts `a` and `b`: diff --git a/litellm/proxy/mcp_server/code_runtime.py b/litellm/proxy/mcp_server/code_runtime.py index 74da4eb63c4..6ebeef3d784 100644 --- a/litellm/proxy/mcp_server/code_runtime.py +++ b/litellm/proxy/mcp_server/code_runtime.py @@ -51,7 +51,9 @@ def check_runtime() -> None: try: metadata.distribution("quickjs-ng") except metadata.PackageNotFoundError as exc: - raise ValueError("MCP code mode requires quickjs-ng; install it on a supported platform") from exc + raise ValueError( + "MCP code mode requires quickjs-ng. Install 'litellm[proxy,mcp-code-mode]' on a supported platform." + ) from exc try: metadata.distribution("quickjs") except metadata.PackageNotFoundError: @@ -59,7 +61,9 @@ def check_runtime() -> None: else: raise ValueError("MCP code mode requires removing the conflicting quickjs package and reinstalling quickjs-ng") if importlib.util.find_spec("_quickjs") is None: - raise ValueError("MCP code mode requires quickjs-ng; install it on a supported platform") + raise ValueError( + "MCP code mode requires quickjs-ng. Install 'litellm[proxy,mcp-code-mode]' on a supported platform." + ) async def _build_reply(call: Call, callback: ToolCallback, maximum: int) -> bytes: diff --git a/pyproject.toml b/pyproject.toml index bb767e53c8c..b201bb9bd80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ Documentation = "https://docs.litellm.ai" # sub-packages and the opentelemetry trio move in lockstep, and grpcio is # supply-chain-pinned to a vetted, aged release. [project.optional-dependencies] +mcp-code-mode = ["quickjs-ng>=0.17.0.2,<0.18"] proxy = [ "gunicorn>=23.0.0,<24.0", "uvicorn>=0.33.0,<1.0", @@ -82,7 +83,6 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=2.3.0,<3", - "quickjs-ng>=0.17.0.2,<0.18; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64' or platform_machine == 'i686')) or (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'win32' and (platform_machine == 'AMD64' or platform_machine == 'x86'))", "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3", "litellm-proxy-extras==0.4.112", diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index cd951c6cd28..8ce5aae481d 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -66,6 +66,7 @@ unauthorized_licenses: gpl v3 [Authorized Packages] +quickjs-ng: ==0.17.0.2 # MIT, verified at https://github.com/genotrance/quickjs-ng/blob/d63694c90845cba04437a23c4a320c2ea857f0b4/LICENSE litellm-admin-mcp: ==0.1.0 # MIT, verified at https://github.com/BerriAI/liteadmin-mcp/blob/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a/LICENSE # Apache-2.0 https://github.com/chroma-core/hnswlib#Apache-2.0-1-ov-file chroma-hnswlib: >=0.7.3 diff --git a/tests/integration/mcp/test_mcp_code_dispatch.py b/tests/integration/mcp/test_mcp_code_dispatch.py index 332a2211e39..ec8cc8382c9 100644 --- a/tests/integration/mcp/test_mcp_code_dispatch.py +++ b/tests/integration/mcp/test_mcp_code_dispatch.py @@ -68,8 +68,9 @@ def test_code_calls_chain_with_caller_permissions(code_gateway: Gateway) -> None def test_code_timeout_returns_error_and_releases_caller_slot(code_gateway: Gateway) -> None: - with code_gateway.scenario() as scenario: - key: Final = scenario.key(max_parallel_requests=1) + with mcp_peer() as peer, code_gateway.scenario() as scenario: + identity: Final = register_mcp(scenario, peer, "timeout" + uuid.uuid4().hex[:8]) + key: Final = scenario.key(object_permission={"mcp_servers": [identity]}, max_parallel_requests=1) _, result = official_client_outcomes( code_gateway, key, "/mcp", "litellm_execute_code", {"code": "while (true) {}"} ) diff --git a/tests/unit/proxy/mcp_server/test_code_runtime.py b/tests/unit/proxy/mcp_server/test_code_runtime.py index ab199a800e2..c6d42040ef9 100644 --- a/tests/unit/proxy/mcp_server/test_code_runtime.py +++ b/tests/unit/proxy/mcp_server/test_code_runtime.py @@ -22,7 +22,7 @@ def test_code_mode_reports_missing_runtime_dependency(monkeypatch: pytest.Monkey raise code_runtime.metadata.PackageNotFoundError(name) monkeypatch.setattr(code_runtime.metadata, "distribution", missing) - with pytest.raises(ValueError, match="requires quickjs-ng"): + with pytest.raises(ValueError, match=r"requires quickjs-ng\. Install 'litellm\[proxy,mcp-code-mode\]'"): code_runtime.check_runtime() diff --git a/uv.lock b/uv.lock index 374af6f4bf6..3427428d63b 100644 --- a/uv.lock +++ b/uv.lock @@ -5018,6 +5018,9 @@ mcp = [ { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, { name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] +mcp-code-mode = [ + { name = "quickjs-ng" }, +] mlflow = [ { name = "mlflow" }, ] @@ -5050,7 +5053,6 @@ proxy = [ { name = "pyroscope-io", marker = "sys_platform != 'win32'" }, { name = "python-multipart" }, { name = "pyyaml" }, - { name = "quickjs-ng", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'i686' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'x86' and sys_platform == 'win32')" }, { name = "restrictedpython" }, { name = "rich" }, { name = "rq" }, @@ -5310,7 +5312,7 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0.3,<7.0" }, { name = "pyyaml", marker = "extra == 'cli'", specifier = ">=6.0.3,<7.0" }, { name = "pyyaml", marker = "extra == 'proxy'", specifier = ">=6.0.3,<7.0" }, - { name = "quickjs-ng", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin' and extra == 'proxy') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'proxy') or (platform_machine == 'i686' and sys_platform == 'linux' and extra == 'proxy') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'proxy') or (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'proxy') or (platform_machine == 'x86' and sys_platform == 'win32' and extra == 'proxy')", specifier = ">=0.17.0.2,<0.18" }, + { name = "quickjs-ng", marker = "extra == 'mcp-code-mode'", specifier = ">=0.17.0.2,<0.18" }, { name = "redisvl", marker = "extra == 'extra-proxy'", specifier = ">=0.4.1,<1.0" }, { name = "requests", marker = "extra == 'cli'", specifier = ">=2.32.0,<3.0" }, { name = "resend", marker = "extra == 'extra-proxy'", specifier = ">=2.23.0,<3.0" }, @@ -5334,7 +5336,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.22.1,<1.0" }, { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" }, ] -provides-extras = ["proxy", "cli", "extra-proxy", "utils", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] +provides-extras = ["mcp-code-mode", "proxy", "cli", "extra-proxy", "utils", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] [package.metadata.requires-dev] admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz" }]