From 9ca4db794c20a9e99a86a39b1ff62ab7f694205b Mon Sep 17 00:00:00 2001 From: Sunny-Soni00 Date: Mon, 7 Sep 2026 22:21:29 +0530 Subject: [PATCH 1/3] fix(proxy): resolve pass-through endpoint type from the route, not just the host get_endpoint_type only matched Anthropic and OpenAI by hostname, so a pass-through endpoint pointed at a provider that speaks either API on any other host resolved to GENERIC. GENERIC matches no branch in the streaming logging dispatch, so the stream was recorded as "cannot parse chunks to standard response object" with no usage and no cost. Fall back to the request path once the existing host checks miss: a route ending in /v1/messages is Anthropic, one ending in /v1/chat/completions is OpenAI. The fallback can only fire where the function returns GENERIC today, so nothing that already classifies can reclassify. Suffix matching keeps routes like /v1/messages/count_tokens on the generic path. Fixes #40117 --- .../pass_through_endpoints.py | 5 + .../test_pass_through_endpoints.py | 92 +++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 3cb9acc6110..709caf8af72 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -369,6 +369,11 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): or (parsed_url.hostname and "openai.com" in parsed_url.hostname) ): return EndpointType.OPENAI + route: Final = parsed_url.path.rstrip("/") + if route.endswith("/v1/messages"): + return EndpointType.ANTHROPIC + elif route.endswith("/v1/chat/completions"): + return EndpointType.OPENAI return EndpointType.GENERIC @staticmethod diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index fb4ed3db4d2..e459b84d1c9 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -5837,3 +5837,95 @@ def test_passthrough_client_cannot_forge_session_id_omission(client_metadata_key ) == "per-call-random-trace-id" ) + + +ANTHROPIC_SSE_STREAM = ( + "event: message_start", + 'data: {"type":"message_start","message":{"id":"msg_01","type":"message","role":"assistant",' + '"model":"claude-sonnet-4-5-20250929","content":[],"stop_reason":null,"stop_sequence":null,' + '"usage":{"input_tokens":17,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,' + '"output_tokens":5}}}', + "event: content_block_start", + 'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + "event: content_block_delta", + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello there."}}', + "event: content_block_stop", + 'data: {"type":"content_block_stop","index":0}', + "event: message_delta", + 'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},' + '"usage":{"output_tokens":40}}', + "event: message_stop", + 'data: {"type":"message_stop"}', +) + + +def _log_passthrough_stream(url: str): + from datetime import datetime + + from litellm.proxy.pass_through_endpoints.streaming_handler import ( + PassThroughStreamingHandler, + ) + + logging_obj = MagicMock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {} + logging_obj.optional_params = {} + logging_obj.litellm_call_id = "test-call-id" + + return PassThroughStreamingHandler._build_passthrough_logging_result( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=PassThroughEndpointLogging(), + url_route=url, + request_body={"model": "claude-sonnet-4-5-20250929"}, + endpoint_type=HttpPassThroughEndpointHelpers.get_endpoint_type(url), + start_time=datetime.now(), + raw_bytes=[(line + "\n").encode("utf-8") for line in ANTHROPIC_SSE_STREAM], + end_time=datetime.now(), + model="claude-sonnet-4-5-20250929", + ) + + +def test_anthropic_compatible_passthrough_stream_is_billed_off_api_anthropic_com(): + """ + Regression for #40117: the endpoint type was resolved from the hostname alone, so a + stream from a provider that speaks the Anthropic Messages API on any other host fell + through to GENERIC, matched no branch in the streaming logging dispatch, and was + recorded as "cannot parse chunks to standard response object" with no usage and no + cost. The same bytes must produce the same accounting whatever host served them. + """ + anthropic_result, anthropic_kwargs = _log_passthrough_stream("https://api.anthropic.com/v1/messages") + custom_result, custom_kwargs = _log_passthrough_stream("https://my-provider.example.com/v1/messages") + + assert custom_kwargs["response_cost"] == anthropic_kwargs["response_cost"] + assert custom_kwargs["response_cost"] > 0 + assert custom_result.usage.prompt_tokens == anthropic_result.usage.prompt_tokens == 17 + assert custom_result.usage.completion_tokens == anthropic_result.usage.completion_tokens == 40 + + +def test_openai_compatible_passthrough_route_resolves_off_api_openai_com(): + from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType + + assert ( + HttpPassThroughEndpointHelpers.get_endpoint_type("https://my-provider.example.com/v1/chat/completions") + == EndpointType.OPENAI + ) + assert ( + HttpPassThroughEndpointHelpers.get_endpoint_type("https://my-provider.example.com/openai/v1/chat/completions/") + == EndpointType.OPENAI + ) + + +def test_passthrough_routes_outside_the_known_shapes_stay_generic(): + """ + The path fallback must only claim the two canonical routes. Sending a body the + Anthropic or OpenAI handler cannot parse into that handler drops the log row + entirely, which is the failure mode LIT-4527 fixed for Vertex. + """ + from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType + + for url in ( + "https://my-provider.example.com/v1/messages/count_tokens", + "https://my-provider.example.com/v1/messages/batches", + "https://my-provider.example.com/v1/embeddings", + "https://upstream.example.com/ml/api/v1/time-series-forecast/predict", + ): + assert HttpPassThroughEndpointHelpers.get_endpoint_type(url) == EndpointType.GENERIC From 6282438c60e2dd800ff429f3f3c1a8c14f69a884 Mon Sep 17 00:00:00 2001 From: Sunny-Soni00 Date: Tue, 8 Sep 2026 09:33:02 +0530 Subject: [PATCH 2/3] test(proxy): type the pass-through logging helper and trim its docstrings Addresses review on #40137: the new helper carried no return annotation, and the two regression docstrings restated what the assertions already show. Keeps only the part that is not obvious from the test, which is the failure each one pins. --- .../test_pass_through_endpoints.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index e459b84d1c9..1b12cacd372 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -6,7 +6,7 @@ from collections.abc import Callable from contextlib import ExitStack, contextmanager from io import BytesIO from types import SimpleNamespace -from typing import Optional +from typing import Final, Optional from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -31,7 +31,11 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( ) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy._types import ( + PassThroughEndpointLoggingResultValues, + ProxyException, + UserAPIKeyAuth, +) from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, ) @@ -5859,14 +5863,14 @@ ANTHROPIC_SSE_STREAM = ( ) -def _log_passthrough_stream(url: str): +def _log_passthrough_stream(url: str) -> tuple[PassThroughEndpointLoggingResultValues, dict]: from datetime import datetime from litellm.proxy.pass_through_endpoints.streaming_handler import ( PassThroughStreamingHandler, ) - logging_obj = MagicMock(spec=LiteLLMLoggingObj) + logging_obj: Final = MagicMock(spec=LiteLLMLoggingObj) logging_obj.model_call_details = {} logging_obj.optional_params = {} logging_obj.litellm_call_id = "test-call-id" @@ -5886,11 +5890,8 @@ def _log_passthrough_stream(url: str): def test_anthropic_compatible_passthrough_stream_is_billed_off_api_anthropic_com(): """ - Regression for #40117: the endpoint type was resolved from the hostname alone, so a - stream from a provider that speaks the Anthropic Messages API on any other host fell - through to GENERIC, matched no branch in the streaming logging dispatch, and was - recorded as "cannot parse chunks to standard response object" with no usage and no - cost. The same bytes must produce the same accounting whatever host served them. + Regression for #40117: classified off the hostname alone, this stream reached the + generic branch and logged "cannot parse chunks to standard response object" at zero cost. """ anthropic_result, anthropic_kwargs = _log_passthrough_stream("https://api.anthropic.com/v1/messages") custom_result, custom_kwargs = _log_passthrough_stream("https://my-provider.example.com/v1/messages") @@ -5916,8 +5917,7 @@ def test_openai_compatible_passthrough_route_resolves_off_api_openai_com(): def test_passthrough_routes_outside_the_known_shapes_stay_generic(): """ - The path fallback must only claim the two canonical routes. Sending a body the - Anthropic or OpenAI handler cannot parse into that handler drops the log row + Over-claiming a route feeds a body the handler cannot parse and drops the log row entirely, which is the failure mode LIT-4527 fixed for Vertex. """ from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType From 93640dddb8d2c4422cdc726b8944b51f3753f093 Mon Sep 17 00:00:00 2001 From: Sunny-Soni00 Date: Tue, 8 Sep 2026 09:49:05 +0530 Subject: [PATCH 3/3] test(proxy): return the resolved cost instead of a bare kwargs dict Addresses review on #40137: the helper's second element was an unparameterized dict. The tests only ever read response_cost from it, so hand back that value and drop the coarse type. Absent cost still reads as None, so the regression still fails on the unfixed classifier. --- .../test_pass_through_endpoints.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 1b12cacd372..5524fe57761 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -5863,7 +5863,7 @@ ANTHROPIC_SSE_STREAM = ( ) -def _log_passthrough_stream(url: str) -> tuple[PassThroughEndpointLoggingResultValues, dict]: +def _log_passthrough_stream(url: str) -> tuple[PassThroughEndpointLoggingResultValues, float | None]: from datetime import datetime from litellm.proxy.pass_through_endpoints.streaming_handler import ( @@ -5875,7 +5875,7 @@ def _log_passthrough_stream(url: str) -> tuple[PassThroughEndpointLoggingResultV logging_obj.optional_params = {} logging_obj.litellm_call_id = "test-call-id" - return PassThroughStreamingHandler._build_passthrough_logging_result( + result, kwargs = PassThroughStreamingHandler._build_passthrough_logging_result( litellm_logging_obj=logging_obj, passthrough_success_handler_obj=PassThroughEndpointLogging(), url_route=url, @@ -5886,6 +5886,7 @@ def _log_passthrough_stream(url: str) -> tuple[PassThroughEndpointLoggingResultV end_time=datetime.now(), model="claude-sonnet-4-5-20250929", ) + return result, kwargs.get("response_cost") def test_anthropic_compatible_passthrough_stream_is_billed_off_api_anthropic_com(): @@ -5893,11 +5894,11 @@ def test_anthropic_compatible_passthrough_stream_is_billed_off_api_anthropic_com Regression for #40117: classified off the hostname alone, this stream reached the generic branch and logged "cannot parse chunks to standard response object" at zero cost. """ - anthropic_result, anthropic_kwargs = _log_passthrough_stream("https://api.anthropic.com/v1/messages") - custom_result, custom_kwargs = _log_passthrough_stream("https://my-provider.example.com/v1/messages") + anthropic_result, anthropic_cost = _log_passthrough_stream("https://api.anthropic.com/v1/messages") + custom_result, custom_cost = _log_passthrough_stream("https://my-provider.example.com/v1/messages") - assert custom_kwargs["response_cost"] == anthropic_kwargs["response_cost"] - assert custom_kwargs["response_cost"] > 0 + assert custom_cost == anthropic_cost + assert custom_cost is not None and custom_cost > 0 assert custom_result.usage.prompt_tokens == anthropic_result.usage.prompt_tokens == 17 assert custom_result.usage.completion_tokens == anthropic_result.usage.completion_tokens == 40