scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-03 07:23:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 166

fix(mcp): keep code runtime installation opt-in

Browse source
This commit is contained in:
moe-berri 2026-10-30 02:22:01 -07:00
parent aa7f6e91b3
commit 88ab326d8f
7 changed files with 25 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/proxy/mcp_server/README.md
View file

@ -1,6 +1,12 @@
# MCP code mode
Code mode lets an MCP client run JavaScript that combines tool calls and returns only the output it needs. Enable it in the gateway configuration and restart the proxy. The QuickJS-NG runtime is included in the proxy installation on platforms with supported wheels; no separate sandbox service or client integration is required.
Code mode lets an MCP client run JavaScript that combines tool calls and returns only the output it needs. It is off by default. Install the optional runtime only in deployments that will enable it; regular proxy installations do not include it.
```sh
pip install "litellm[proxy,mcp-code-mode]"
```
Keep your existing LiteLLM version pin when adding the extra to a deployed proxy. For containers, include the extra in the image build. Then enable it in the gateway configuration and restart. No separate sandbox service or client integration is required, and the gateway never installs packages at runtime.
```yaml
litellm_settings:
@ -9,7 +15,7 @@ litellm_settings:
Reconnect the MCP client or refresh its tool list. It should discover `litellm_execute_code` alongside its existing tools. The setting is off by default. Turning it off also rejects calls from clients that cached the code tool.
Intel macOS and Windows ARM installations need a manual source installation of `quickjs-ng` before enabling code mode. A missing runtime produces a startup error. The original `quickjs` package conflicts with `quickjs-ng`; use an environment without that package. These requirements do not affect the gateway when code mode is disabled.
The extra installs QuickJS-NG. Platforms without a compatible wheel need a source build and its compiler toolchain; these builds are not part of the verified platform matrix. A missing runtime produces an actionable startup error. The original `quickjs` package shares files with `quickjs-ng`; use a separate environment without that package before installing the extra. Regular proxy installations and disabled gateways do not require either runtime.
The code tool accepts a `code` string. Discover a tool and its argument schema normally, then copy the code-mode `tool_id` from its description or `search_tools` into `tools.call_tool`. Copy that reference unchanged: clients can rename the displayed tool. Exact gateway catalog names also work. Calls return the full MCP result, including `content`, `structuredContent` when provided, and `isError`. Use `text(value)` to return a JSON value to the model. For example, after discovering a tool named `example-add` that accepts `a` and `b`:

8
litellm/proxy/mcp_server/code_runtime.py
View file

@ -51,7 +51,9 @@ def check_runtime() -> None:
try:
metadata.distribution("quickjs-ng")
except metadata.PackageNotFoundError as exc:
raise ValueError("MCP code mode requires quickjs-ng; install it on a supported platform") from exc
raise ValueError(
"MCP code mode requires quickjs-ng. Install 'litellm[proxy,mcp-code-mode]' on a supported platform."
) from exc
try:
metadata.distribution("quickjs")
except metadata.PackageNotFoundError:
@ -59,7 +61,9 @@ def check_runtime() -> None:
else:
raise ValueError("MCP code mode requires removing the conflicting quickjs package and reinstalling quickjs-ng")
if importlib.util.find_spec("_quickjs") is None:
raise ValueError("MCP code mode requires quickjs-ng; install it on a supported platform")
raise ValueError(
"MCP code mode requires quickjs-ng. Install 'litellm[proxy,mcp-code-mode]' on a supported platform."
)
async def _build_reply(call: Call, callback: ToolCallback, maximum: int) -> bytes:

2
pyproject.toml
View file

@ -54,6 +54,7 @@ Documentation = "https://docs.litellm.ai"
# sub-packages and the opentelemetry trio move in lockstep, and grpcio is
# supply-chain-pinned to a vetted, aged release.
[project.optional-dependencies]
mcp-code-mode = ["quickjs-ng>=0.17.0.2,<0.18"]
proxy = [
"gunicorn>=23.0.0,<24.0",
"uvicorn>=0.33.0,<1.0",
@ -82,7 +83,6 @@ proxy = [
"azure-identity>=1.25.2,<2.0",
"azure-storage-blob>=12.28.0,<13.0",
"mcp>=2.3.0,<3",
"quickjs-ng>=0.17.0.2,<0.18; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64' or platform_machine == 'i686')) or (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'win32' and (platform_machine == 'AMD64' or platform_machine == 'x86'))",
"httpx2>=2.10.0,<3",
"pydantic>=2.12.0,<3",
"litellm-proxy-extras==0.4.112",

1
tests/code_coverage_tests/liccheck.ini
View file

@ -66,6 +66,7 @@ unauthorized_licenses:
gpl v3
[Authorized Packages]
quickjs-ng: ==0.17.0.2 # MIT, verified at https://github.com/genotrance/quickjs-ng/blob/d63694c90845cba04437a23c4a320c2ea857f0b4/LICENSE
litellm-admin-mcp: ==0.1.0 # MIT, verified at https://github.com/BerriAI/liteadmin-mcp/blob/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a/LICENSE
# Apache-2.0 https://github.com/chroma-core/hnswlib#Apache-2.0-1-ov-file
chroma-hnswlib: >=0.7.3

5
tests/integration/mcp/test_mcp_code_dispatch.py
View file

@ -68,8 +68,9 @@ def test_code_calls_chain_with_caller_permissions(code_gateway: Gateway) -> None
def test_code_timeout_returns_error_and_releases_caller_slot(code_gateway: Gateway) -> None:
with code_gateway.scenario() as scenario:
key: Final = scenario.key(max_parallel_requests=1)
with mcp_peer() as peer, code_gateway.scenario() as scenario:
identity: Final = register_mcp(scenario, peer, "timeout" + uuid.uuid4().hex[:8])
key: Final = scenario.key(object_permission={"mcp_servers": [identity]}, max_parallel_requests=1)
_, result = official_client_outcomes(
code_gateway, key, "/mcp", "litellm_execute_code", {"code": "while (true) {}"}
)

2
tests/unit/proxy/mcp_server/test_code_runtime.py
View file

@ -22,7 +22,7 @@ def test_code_mode_reports_missing_runtime_dependency(monkeypatch: pytest.Monkey
raise code_runtime.metadata.PackageNotFoundError(name)
monkeypatch.setattr(code_runtime.metadata, "distribution", missing)
with pytest.raises(ValueError, match="requires quickjs-ng"):
with pytest.raises(ValueError, match=r"requires quickjs-ng\. Install 'litellm\[proxy,mcp-code-mode\]'"):
code_runtime.check_runtime()

8
uv.lock generated
View file

@ -5018,6 +5018,9 @@ mcp = [
{ name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" },
{ name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" },
]
mcp-code-mode = [
{ name = "quickjs-ng" },
]
mlflow = [
{ name = "mlflow" },
]
@ -5050,7 +5053,6 @@ proxy = [
{ name = "pyroscope-io", marker = "sys_platform != 'win32'" },
{ name = "python-multipart" },
{ name = "pyyaml" },
{ name = "quickjs-ng", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'i686' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'x86' and sys_platform == 'win32')" },
{ name = "restrictedpython" },
{ name = "rich" },
{ name = "rq" },
@ -5310,7 +5312,7 @@ requires-dist = [
{ name = "pyyaml", specifier = ">=6.0.3,<7.0" },
{ name = "pyyaml", marker = "extra == 'cli'", specifier = ">=6.0.3,<7.0" },
{ name = "pyyaml", marker = "extra == 'proxy'", specifier = ">=6.0.3,<7.0" },
{ name = "quickjs-ng", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin' and extra == 'proxy') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'proxy') or (platform_machine == 'i686' and sys_platform == 'linux' and extra == 'proxy') or (platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'proxy') or (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'proxy') or (platform_machine == 'x86' and sys_platform == 'win32' and extra == 'proxy')", specifier = ">=0.17.0.2,<0.18" },
{ name = "quickjs-ng", marker = "extra == 'mcp-code-mode'", specifier = ">=0.17.0.2,<0.18" },
{ name = "redisvl", marker = "extra == 'extra-proxy'", specifier = ">=0.4.1,<1.0" },
{ name = "requests", marker = "extra == 'cli'", specifier = ">=2.32.0,<3.0" },
{ name = "resend", marker = "extra == 'extra-proxy'", specifier = ">=2.23.0,<3.0" },
@ -5334,7 +5336,7 @@ requires-dist = [
{ name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.22.1,<1.0" },
{ name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" },
]
provides-extras = ["proxy", "cli", "extra-proxy", "utils", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"]
provides-extras = ["mcp-code-mode", "proxy", "cli", "extra-proxy", "utils", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"]
[package.metadata.requires-dev]
admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz" }]