From 9ca4db794c20a9e99a86a39b1ff62ab7f694205b Mon Sep 17 00:00:00 2001 From: Sunny-Soni00 Date: Mon, 7 Sep 2026 22:21:29 +0530 Subject: [PATCH] fix(proxy): resolve pass-through endpoint type from the route, not just the host get_endpoint_type only matched Anthropic and OpenAI by hostname, so a pass-through endpoint pointed at a provider that speaks either API on any other host resolved to GENERIC. GENERIC matches no branch in the streaming logging dispatch, so the stream was recorded as "cannot parse chunks to standard response object" with no usage and no cost. Fall back to the request path once the existing host checks miss: a route ending in /v1/messages is Anthropic, one ending in /v1/chat/completions is OpenAI. The fallback can only fire where the function returns GENERIC today, so nothing that already classifies can reclassify. Suffix matching keeps routes like /v1/messages/count_tokens on the generic path. Fixes #40117 --- .../pass_through_endpoints.py | 5 + .../test_pass_through_endpoints.py | 92 +++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 3cb9acc6110..709caf8af72 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -369,6 +369,11 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): or (parsed_url.hostname and "openai.com" in parsed_url.hostname) ): return EndpointType.OPENAI + route: Final = parsed_url.path.rstrip("/") + if route.endswith("/v1/messages"): + return EndpointType.ANTHROPIC + elif route.endswith("/v1/chat/completions"): + return EndpointType.OPENAI return EndpointType.GENERIC @staticmethod diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index fb4ed3db4d2..e459b84d1c9 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -5837,3 +5837,95 @@ def test_passthrough_client_cannot_forge_session_id_omission(client_metadata_key ) == "per-call-random-trace-id" ) + + +ANTHROPIC_SSE_STREAM = ( + "event: message_start", + 'data: {"type":"message_start","message":{"id":"msg_01","type":"message","role":"assistant",' + '"model":"claude-sonnet-4-5-20250929","content":[],"stop_reason":null,"stop_sequence":null,' + '"usage":{"input_tokens":17,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,' + '"output_tokens":5}}}', + "event: content_block_start", + 'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + "event: content_block_delta", + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello there."}}', + "event: content_block_stop", + 'data: {"type":"content_block_stop","index":0}', + "event: message_delta", + 'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},' + '"usage":{"output_tokens":40}}', + "event: message_stop", + 'data: {"type":"message_stop"}', +) + + +def _log_passthrough_stream(url: str): + from datetime import datetime + + from litellm.proxy.pass_through_endpoints.streaming_handler import ( + PassThroughStreamingHandler, + ) + + logging_obj = MagicMock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {} + logging_obj.optional_params = {} + logging_obj.litellm_call_id = "test-call-id" + + return PassThroughStreamingHandler._build_passthrough_logging_result( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=PassThroughEndpointLogging(), + url_route=url, + request_body={"model": "claude-sonnet-4-5-20250929"}, + endpoint_type=HttpPassThroughEndpointHelpers.get_endpoint_type(url), + start_time=datetime.now(), + raw_bytes=[(line + "\n").encode("utf-8") for line in ANTHROPIC_SSE_STREAM], + end_time=datetime.now(), + model="claude-sonnet-4-5-20250929", + ) + + +def test_anthropic_compatible_passthrough_stream_is_billed_off_api_anthropic_com(): + """ + Regression for #40117: the endpoint type was resolved from the hostname alone, so a + stream from a provider that speaks the Anthropic Messages API on any other host fell + through to GENERIC, matched no branch in the streaming logging dispatch, and was + recorded as "cannot parse chunks to standard response object" with no usage and no + cost. The same bytes must produce the same accounting whatever host served them. + """ + anthropic_result, anthropic_kwargs = _log_passthrough_stream("https://api.anthropic.com/v1/messages") + custom_result, custom_kwargs = _log_passthrough_stream("https://my-provider.example.com/v1/messages") + + assert custom_kwargs["response_cost"] == anthropic_kwargs["response_cost"] + assert custom_kwargs["response_cost"] > 0 + assert custom_result.usage.prompt_tokens == anthropic_result.usage.prompt_tokens == 17 + assert custom_result.usage.completion_tokens == anthropic_result.usage.completion_tokens == 40 + + +def test_openai_compatible_passthrough_route_resolves_off_api_openai_com(): + from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType + + assert ( + HttpPassThroughEndpointHelpers.get_endpoint_type("https://my-provider.example.com/v1/chat/completions") + == EndpointType.OPENAI + ) + assert ( + HttpPassThroughEndpointHelpers.get_endpoint_type("https://my-provider.example.com/openai/v1/chat/completions/") + == EndpointType.OPENAI + ) + + +def test_passthrough_routes_outside_the_known_shapes_stay_generic(): + """ + The path fallback must only claim the two canonical routes. Sending a body the + Anthropic or OpenAI handler cannot parse into that handler drops the log row + entirely, which is the failure mode LIT-4527 fixed for Vertex. + """ + from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType + + for url in ( + "https://my-provider.example.com/v1/messages/count_tokens", + "https://my-provider.example.com/v1/messages/batches", + "https://my-provider.example.com/v1/embeddings", + "https://upstream.example.com/ml/api/v1/time-series-forecast/predict", + ): + assert HttpPassThroughEndpointHelpers.get_endpoint_type(url) == EndpointType.GENERIC