diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 3cb9acc6110..709caf8af72 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -369,6 +369,11 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): or (parsed_url.hostname and "openai.com" in parsed_url.hostname) ): return EndpointType.OPENAI + route: Final = parsed_url.path.rstrip("/") + if route.endswith("/v1/messages"): + return EndpointType.ANTHROPIC + elif route.endswith("/v1/chat/completions"): + return EndpointType.OPENAI return EndpointType.GENERIC @staticmethod diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index fb4ed3db4d2..e459b84d1c9 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -5837,3 +5837,95 @@ def test_passthrough_client_cannot_forge_session_id_omission(client_metadata_key ) == "per-call-random-trace-id" ) + + +ANTHROPIC_SSE_STREAM = ( + "event: message_start", + 'data: {"type":"message_start","message":{"id":"msg_01","type":"message","role":"assistant",' + '"model":"claude-sonnet-4-5-20250929","content":[],"stop_reason":null,"stop_sequence":null,' + '"usage":{"input_tokens":17,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,' + '"output_tokens":5}}}', + "event: content_block_start", + 'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + "event: content_block_delta", + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello there."}}', + "event: content_block_stop", + 'data: {"type":"content_block_stop","index":0}', + "event: message_delta", + 'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},' + '"usage":{"output_tokens":40}}', + "event: message_stop", + 'data: {"type":"message_stop"}', +) + + +def _log_passthrough_stream(url: str): + from datetime import datetime + + from litellm.proxy.pass_through_endpoints.streaming_handler import ( + PassThroughStreamingHandler, + ) + + logging_obj = MagicMock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {} + logging_obj.optional_params = {} + logging_obj.litellm_call_id = "test-call-id" + + return PassThroughStreamingHandler._build_passthrough_logging_result( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=PassThroughEndpointLogging(), + url_route=url, + request_body={"model": "claude-sonnet-4-5-20250929"}, + endpoint_type=HttpPassThroughEndpointHelpers.get_endpoint_type(url), + start_time=datetime.now(), + raw_bytes=[(line + "\n").encode("utf-8") for line in ANTHROPIC_SSE_STREAM], + end_time=datetime.now(), + model="claude-sonnet-4-5-20250929", + ) + + +def test_anthropic_compatible_passthrough_stream_is_billed_off_api_anthropic_com(): + """ + Regression for #40117: the endpoint type was resolved from the hostname alone, so a + stream from a provider that speaks the Anthropic Messages API on any other host fell + through to GENERIC, matched no branch in the streaming logging dispatch, and was + recorded as "cannot parse chunks to standard response object" with no usage and no + cost. The same bytes must produce the same accounting whatever host served them. + """ + anthropic_result, anthropic_kwargs = _log_passthrough_stream("https://api.anthropic.com/v1/messages") + custom_result, custom_kwargs = _log_passthrough_stream("https://my-provider.example.com/v1/messages") + + assert custom_kwargs["response_cost"] == anthropic_kwargs["response_cost"] + assert custom_kwargs["response_cost"] > 0 + assert custom_result.usage.prompt_tokens == anthropic_result.usage.prompt_tokens == 17 + assert custom_result.usage.completion_tokens == anthropic_result.usage.completion_tokens == 40 + + +def test_openai_compatible_passthrough_route_resolves_off_api_openai_com(): + from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType + + assert ( + HttpPassThroughEndpointHelpers.get_endpoint_type("https://my-provider.example.com/v1/chat/completions") + == EndpointType.OPENAI + ) + assert ( + HttpPassThroughEndpointHelpers.get_endpoint_type("https://my-provider.example.com/openai/v1/chat/completions/") + == EndpointType.OPENAI + ) + + +def test_passthrough_routes_outside_the_known_shapes_stay_generic(): + """ + The path fallback must only claim the two canonical routes. Sending a body the + Anthropic or OpenAI handler cannot parse into that handler drops the log row + entirely, which is the failure mode LIT-4527 fixed for Vertex. + """ + from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType + + for url in ( + "https://my-provider.example.com/v1/messages/count_tokens", + "https://my-provider.example.com/v1/messages/batches", + "https://my-provider.example.com/v1/embeddings", + "https://upstream.example.com/ml/api/v1/time-series-forecast/predict", + ): + assert HttpPassThroughEndpointHelpers.get_endpoint_type(url) == EndpointType.GENERIC