diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 6766d246894..a31b0f082e3 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2493,8 +2493,9 @@ class PrometheusLogger(CustomLogger): A request can fail before a deployment is resolved, so the provider is not always known. Prefer the resolved ``custom_llm_provider`` on ``litellm_params``, then any provider recovered onto a partial - ``standard_logging_object`` (e.g. a stream that broke mid-flight), and - finally infer it from the requested model name (e.g. ``gpt-4o-mini`` -> + ``standard_logging_object`` (e.g. a stream that broke mid-flight), then + the provider the request itself declares (e.g. a Gemini pass-through + route), and finally infer it from the requested model name (e.g. ``gpt-4o-mini`` -> ``openai``) since the proxy's failure ``request_data`` usually carries only the client-supplied model. Return ``None`` when it cannot be determined so the label emits empty rather than a guess. @@ -2504,7 +2505,7 @@ class PrometheusLogger(CustomLogger): if provider: return provider standard_logging_object: Final = request_data.get("standard_logging_object") or {} - provider = standard_logging_object.get("custom_llm_provider") + provider = standard_logging_object.get("custom_llm_provider") or request_data.get("custom_llm_provider") if provider: return provider model: Final = litellm_params.get("model") or request_data.get("model") diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 3cb9acc6110..023d8eeb961 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -58,6 +58,7 @@ from litellm.llms.base_llm.managed_resources.utils import ( resolve_passthrough_managed_id_provider, ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.llms.vertex_ai.common_utils import get_vertex_model_id_from_url from litellm.passthrough import BasePassthroughUtils from litellm.proxy._types import ( ConfigFieldInfo, @@ -730,12 +731,23 @@ def _carry_guardrail_logging_info(request_data: dict, guardrail_data: dict | Non metadata.setdefault("standard_logging_guardrail_information", list(entries)) +def _passthrough_model_from_url(url: httpx.URL | None, custom_llm_provider: str | None) -> str | None: + if url is None: + return None + url_route: Final = str(url) + tracked_google_route: Final = pass_through_endpoint_logging.is_gemini_route( + url_route, custom_llm_provider + ) or pass_through_endpoint_logging.is_vertex_route(url_route) + return get_vertex_model_id_from_url(url_route) if tracked_google_route else None + + def _build_passthrough_failure_request_payload( parsed_body: dict | None, kwargs: dict | None, logging_obj: LiteLLMLoggingObj | None, custom_llm_provider: str | None, upstream_usage: UpstreamReportedUsage | None = None, + url: httpx.URL | None = None, ) -> dict: """Build the ``request_data`` dict passed to ``post_call_failure_hook``. @@ -752,8 +764,8 @@ def _build_passthrough_failure_request_payload( request_payload.update(kwargs) if logging_obj is not None: request_payload["litellm_logging_obj"] = logging_obj - if "model" not in request_payload and parsed_body and isinstance(parsed_body, dict): - request_payload["model"] = parsed_body.get("model", "") + if "model" not in request_payload: + request_payload["model"] = _passthrough_model_from_url(url, custom_llm_provider) or "" if "custom_llm_provider" not in request_payload and custom_llm_provider: request_payload["custom_llm_provider"] = custom_llm_provider if upstream_usage is not None: @@ -1302,6 +1314,7 @@ async def pass_through_request( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, + url=url, ), ) @@ -1393,6 +1406,7 @@ async def pass_through_request( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, + url=url, ), ) @@ -1492,6 +1506,7 @@ async def pass_through_request( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, + url=url, ) failure_request_payload["response_body"] = response_body await _log_passthrough_upstream_failure( @@ -1697,19 +1712,13 @@ async def pass_through_request( # Monitoring: Trigger post_call_failure_hook # for pass through endpoint failure ######################################################### - request_payload: Final[dict] = _parsed_body or {} - # add user_api_key_dict, litellm_call_id, passthrough_logging_payloa for logging - if kwargs: - for key, value in kwargs.items(): - request_payload[key] = value - if logging_obj is not None: - request_payload["litellm_logging_obj"] = logging_obj - - if "model" not in request_payload and _parsed_body and isinstance(_parsed_body, dict): - request_payload["model"] = _parsed_body.get("model", "") - if "custom_llm_provider" not in request_payload and custom_llm_provider: - request_payload["custom_llm_provider"] = custom_llm_provider - + request_payload: Final = _build_passthrough_failure_request_payload( + parsed_body=_parsed_body, + kwargs=kwargs, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + url=url, + ) _carry_guardrail_logging_info(request_payload, post_call_guardrail_data) await proxy_logging_obj.post_call_failure_hook( diff --git a/tests/test_litellm/integrations/test_prometheus_labels.py b/tests/test_litellm/integrations/test_prometheus_labels.py index 859cdd30c11..93191e7d888 100644 --- a/tests/test_litellm/integrations/test_prometheus_labels.py +++ b/tests/test_litellm/integrations/test_prometheus_labels.py @@ -235,6 +235,24 @@ def test_extract_api_provider_from_request_data_failure_path(): assert extract({"model": "some-unknown-model-xyz"}) is None +def test_extract_api_provider_prefers_declared_provider_over_model_name_inference(): + from litellm.integrations.prometheus import PrometheusLogger + + extract = PrometheusLogger._extract_api_provider_from_request_data + + assert extract({"model": "gemini-3.8-flash"}) == "vertex_ai" + assert extract({"model": "gemini-3.8-flash", "custom_llm_provider": "gemini"}) == "gemini" + assert ( + extract( + { + "custom_llm_provider": "gemini", + "standard_logging_object": {"custom_llm_provider": "vertex_ai"}, + } + ) + == "vertex_ai" + ) + + def test_extract_api_provider_swallows_unknown_model_but_logs_unexpected_errors(): """ get_llm_provider raises BadRequestError for a model that maps to no diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index fb4ed3db4d2..d43e66fb956 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx from fastapi import Request, UploadFile from starlette.datastructures import FormData, Headers, QueryParams from starlette.datastructures import UploadFile as StarletteUploadFile @@ -21,6 +22,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( HttpPassThroughEndpointHelpers, InitPassThroughEndpointHelpers, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, + _build_passthrough_failure_request_payload, _registered_pass_through_routes, create_pass_through_route, initialize_pass_through_endpoints, @@ -344,53 +346,18 @@ async def test_pass_through_request_failure_handler(): Critical Test: When a users pass through endpoint request fails, we must log the failure code, exception in litellm spend logs. """ - with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: - with patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_get_client: - with patch( - "litellm.proxy.pass_through_endpoints.pass_through_endpoints.ProxyBaseLLMRequestProcessing" - ) as mock_processing: - # Setup mock for post_call_failure_hook and pre_call_hook - mock_proxy_logging.post_call_failure_hook = AsyncMock() - mock_proxy_logging.pre_call_hook = AsyncMock() + request_failure = httpx.HTTPError("Request failed") + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test") - # Setup mock for httpx client - mock_client = MagicMock() - mock_client.client = MagicMock() - mock_client.client.request = AsyncMock(side_effect=httpx.HTTPError("Request failed")) - mock_get_client.return_value = mock_client + failure = await _run_passthrough_failure( + "http://test.com", None, request_failure, user_api_key_dict=user_api_key_dict + ) - # Mock headers for custom headers - mock_processing.get_custom_headers.return_value = {} - - # Create mock request - mock_request = MagicMock(spec=Request) - mock_request.method = "POST" - mock_request.body = AsyncMock(return_value=b'{"test": "data"}') - mock_request.headers = Headers({}) - - # Create a simple empty QueryParams - mock_request.query_params = QueryParams({}) - - # Create mock user API key dict - mock_user_api_key_dict = MagicMock() - - # Call the function with a target that will trigger an HTTPError - with pytest.raises(ProxyException): - await pass_through_request( - request=mock_request, - target="http://test.com", - custom_headers={}, - user_api_key_dict=mock_user_api_key_dict, - ) - - # Assert post_call_failure_hook was called - mock_proxy_logging.post_call_failure_hook.assert_called_once() - - # Verify the arguments to post_call_failure_hook - call_args = mock_proxy_logging.post_call_failure_hook.call_args[1] - assert call_args["user_api_key_dict"] == mock_user_api_key_dict - assert isinstance(call_args["original_exception"], TypeError) # Now expecting TypeError - assert "traceback_str" in call_args + assert failure["user_api_key_dict"] is user_api_key_dict + assert failure["original_exception"] is request_failure + assert "traceback_str" in failure + assert failure["request_data"]["contents"] == "not a list" + assert failure["request_data"]["model"] == "" @pytest.mark.asyncio @@ -5837,3 +5804,110 @@ def test_passthrough_client_cannot_forge_session_id_omission(client_metadata_key ) == "per-call-random-trace-id" ) + + +_GEMINI_FLASH_TARGET = "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:generateContent" +_VERTEX_FLASH_TARGET = ( + "https://us-east5-aiplatform.googleapis.com/v1/projects/p/locations/us-east5" + "/publishers/google/models/gemini-3.8-flash:generateContent" +) + + +async def _run_passthrough_failure( + target: str, + custom_llm_provider: str | None, + upstream: httpx.Response | Exception, + stream: bool = False, + user_api_key_dict: UserAPIKeyAuth | None = None, +) -> dict: + with ( + pytest.MonkeyPatch.context() as transport_patch, + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging, + respx.mock(assert_all_called=False) as upstream_mock, + ): + transport_patch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream_route = upstream_mock.route(method="POST") + if isinstance(upstream, Exception): + upstream_route.mock(side_effect=upstream) + else: + upstream_route.mock(return_value=upstream) + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=None) + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = target.replace("https://", "http://test-proxy.com/") + mock_request.body = AsyncMock(return_value=b'{"contents": "not a list"}') + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + try: + await pass_through_request( + request=mock_request, + target=target, + custom_headers={}, + user_api_key_dict=user_api_key_dict or UserAPIKeyAuth(api_key="sk-test"), + custom_llm_provider=custom_llm_provider, + stream=stream, + ) + except ProxyException: + pass + litellm.in_memory_llm_clients_cache.flush_cache() + mock_proxy_logging.post_call_failure_hook.assert_called_once() + return mock_proxy_logging.post_call_failure_hook.call_args.kwargs + + +def _upstream_400(target: str) -> httpx.Response: + return httpx.Response( + status_code=400, + headers={"content-type": "application/json"}, + content=json.dumps({"error": {"code": 400, "status": "INVALID_ARGUMENT"}}).encode(), + request=httpx.Request("POST", target), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "target, custom_llm_provider, stream", + [ + (_GEMINI_FLASH_TARGET, "gemini", False), + (_GEMINI_FLASH_TARGET, "gemini", True), + (_VERTEX_FLASH_TARGET, None, False), + ], +) +async def test_passthrough_upstream_error_logs_model_from_url( + target: str, custom_llm_provider: str | None, stream: bool +): + failure = await _run_passthrough_failure(target, custom_llm_provider, _upstream_400(target), stream=stream) + assert failure["request_data"]["model"] == "gemini-3.8-flash" + + +@pytest.mark.asyncio +async def test_passthrough_internal_failure_logs_model_from_url(): + failure = await _run_passthrough_failure( + _GEMINI_FLASH_TARGET, "gemini", httpx.ReadTimeout("upstream timed out") + ) + assert failure["request_data"]["model"] == "gemini-3.8-flash" + + +@pytest.mark.parametrize( + "parsed_body, url, custom_llm_provider, expected_model", + [ + ({"model": "from-body"}, _GEMINI_FLASH_TARGET, "gemini", "from-body"), + ({"contents": []}, _GEMINI_FLASH_TARGET, "gemini", "gemini-3.8-flash"), + ({"contents": []}, _VERTEX_FLASH_TARGET, None, "gemini-3.8-flash"), + ({"contents": []}, "https://api.example.com/v1/models/gpt-x:thing", None, ""), + (None, None, None, ""), + ], +) +def test_build_passthrough_failure_request_payload_model_precedence( + parsed_body: dict | None, url: str | None, custom_llm_provider: str | None, expected_model: str +): + request_payload = _build_passthrough_failure_request_payload( + parsed_body=parsed_body, + kwargs=None, + logging_obj=None, + custom_llm_provider=custom_llm_provider, + url=httpx.URL(url) if url else None, + ) + assert request_payload["model"] == expected_model