From b04aa3d02d15af1f6612e5c1830baa1f2e903484 Mon Sep 17 00:00:00 2001 From: narmstrong-berry Date: Fri, 6 Nov 2026 04:26:09 +0000 Subject: [PATCH 1/2] fix(messages): let proxy /v1/messages requests run on the rust path The proxy attaches its shared aiohttp session to every request and a default stream_timeout to every stream, and the messages detectors treated both as Python-only features, so no proxy request reached Rust. A shared session is only a connection-reuse hint and no native client reads it, so the common caller_client detector now fires only for an explicit client. The native messages call already applies stream_timeout as the stream's timeout, so only request_timeout or httpx.Timeout keeps a messages call on Python --- litellm/messages/dispatch.py | 1 + litellm/rust_bridge/catalog.py | 2 +- litellm/rust_bridge/detectors.py | 4 +-- litellm/rust_bridge/messages/detectors.py | 6 ++++ .../messages/test_dispatch.py | 30 +++++++++++++++++++ .../responses/test_dispatch.py | 1 - 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 4c94f26c6c1..ee3435d306c 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -123,6 +123,7 @@ def messages_target(model: str, custom_llm_provider: str | None, model_info: obj _FEATURES: Final[detectors.Detectors] = MappingProxyType( { **detectors.COMMON, + Feature.TIMEOUT_VARIANTS: route_detectors.timeout_variants, Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity, Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier, Feature.RESPONSE_CACHE: detectors.response_cache, diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 9adfcb60d74..9cd9bd854e6 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -39,7 +39,7 @@ class Feature(str, Enum): PROXY_MCP_TOOLS = "proxy_mcp_tools" """litellm_proxy MCP tools that Python expands and may auto-execute.""" CALLER_CLIENT = "caller_client" - """A caller-supplied HTTP client or shared session that Python sends through.""" + """A caller-supplied HTTP client that Python sends through.""" TIMEOUT_VARIANTS = "timeout_variants" """request_timeout, stream_timeout or httpx.Timeout phases beyond a plain timeout.""" DROP_PARAMS = "drop_params" diff --git a/litellm/rust_bridge/detectors.py b/litellm/rust_bridge/detectors.py index f2591bdbcd1..c38c3ea7135 100644 --- a/litellm/rust_bridge/detectors.py +++ b/litellm/rust_bridge/detectors.py @@ -44,8 +44,8 @@ def latest_alias(request: NativeCall) -> bool: def caller_client(request: NativeCall) -> bool: - """Python sends through the caller's HTTP client or shared session.""" - return present(request, "client") or present(request, "shared_session") + """Python sends through the caller's HTTP client; a shared session only reuses connections, which native pools.""" + return present(request, "client") def extra_body(request: NativeCall) -> bool: diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 6a0c867979b..e3f4a180c36 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -4,6 +4,8 @@ from __future__ import annotations from typing import Final +import httpx + import litellm from litellm.anthropic_beta_headers_manager import is_foundry_api_base from litellm.litellm_core_utils.core_helpers import drops_unsupported_params @@ -12,6 +14,10 @@ from litellm.rust_bridge.detectors import api_base, overridden, present, streami from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str +def timeout_variants(request: NativeCall) -> bool: + return present(request, "request_timeout") or isinstance(request.resolved.get("timeout"), httpx.Timeout) + + def provider_affinity(request: NativeCall) -> bool: return is_opencode_api_base(api_base(request)) diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 6d65f289524..9d98122de6f 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -528,6 +528,36 @@ def test_each_detected_feature_reaches_its_rule( assert served(kwargs) is python_result +@pytest.mark.parametrize( + "kwargs", + ( + pytest.param({"shared_session": object()}, id="proxy-shared-session"), + pytest.param({"stream": True, "timeout": 600.0, "stream_timeout": 600.0}, id="proxy-stream-timeout"), + ), +) +def test_proxy_request_shape_stays_native(kwargs: Mapping[str, object]) -> None: + rules: Final[Rules] = ( + RouteRule( + Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.CALLER_CLIENT, Feature.TIMEOUT_VARIANTS}) + ), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + ) + native_result: Final = response() + + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: rejected fallback + pytest.fail("A proxy-shaped request must not fall back to Python") + + served: Final = _DISPATCH.run( + (16, MESSAGES, "anthropic/claude-sonnet-4-5"), + kwargs, + python=python, + binding=messages_binding(lambda request: native_result), + native=lambda hook, request, call_args, native_kwargs: hook(request), + rules=rules, + ) + assert served is native_result + + @pytest.mark.asyncio @pytest.mark.parametrize( "kwargs", diff --git a/tests/test_litellm_rust/responses/test_dispatch.py b/tests/test_litellm_rust/responses/test_dispatch.py index 4498dbdf39d..19255379dcf 100644 --- a/tests/test_litellm_rust/responses/test_dispatch.py +++ b/tests/test_litellm_rust/responses/test_dispatch.py @@ -260,7 +260,6 @@ ENCODED_ID: Final = ResponsesAPIRequestUtils._build_encrypted_item_id("deploymen (Feature.MOCK_RESPONSE, {"mock_response": "hi"}, {}), (Feature.PROXY_MCP_TOOLS, {"tools": [{"type": "mcp", "server_label": "g", "server_url": "litellm_proxy"}]}, {}), (Feature.CALLER_CLIENT, {"client": object()}, {}), - pytest.param(Feature.CALLER_CLIENT, {"shared_session": object()}, {}, id="shared-session"), pytest.param(Feature.EXTRA_BODY, {"extra_body": {"stream": True}}, {}, id="extra-body-stream"), pytest.param( Feature.PROVIDER_AFFINITY, From 7d93569fbf0654a3e04cf05453b3fd2fe3d673c5 Mon Sep 17 00:00:00 2001 From: narmstrong-berri Date: Sat, 7 Nov 2026 02:57:51 +0000 Subject: [PATCH 2/2] test(messages): expect a shared session alone to keep the native path --- tests/unit/rust_bridge/test_detectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index 3001d23e57e..7e4f1781b54 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -109,7 +109,7 @@ def _wrap_nested_level(value: object, depth: int) -> object: (detectors.prompt_management, {"model": "litellm_agent/gpt-5"}, True), (detectors.prompt_management, {"model": "gpt-5"}, False), (detectors.caller_client, {"client": object()}, True), - (detectors.caller_client, {"shared_session": object()}, True), + (detectors.caller_client, {"shared_session": object()}, False), (detectors.caller_client, {"client": None, "shared_session": None}, False), (detectors.caller_client, {}, False), (detectors.extra_body, {"extra_body": {"stream": True}}, True),