scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-30 06:43:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 250

fix(passthrough): log the URL model and declared provider on Gemini and Vertex failures

Browse source 
Pass-through failures on Gemini and Vertex routes carry the model only in the
URL, so the failure spend row and the Prometheus failure counter logged an
empty model. The failure payload now falls back to the model parsed from the
URL on those routes, and the Prometheus provider label honors the provider the
request declares before inferring one from the model name
This commit is contained in:
mateo-berri 2026-09-04 23:45:26 -07:00
parent aea5358c48
commit 3561ce241e
4 changed files with 165 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
litellm/integrations/prometheus.py
View file

@ -2493,8 +2493,9 @@ class PrometheusLogger(CustomLogger):
A request can fail before a deployment is resolved, so the provider is
not always known. Prefer the resolved ``custom_llm_provider`` on
``litellm_params``, then any provider recovered onto a partial
``standard_logging_object`` (e.g. a stream that broke mid-flight), and
finally infer it from the requested model name (e.g. ``gpt-4o-mini`` ->
``standard_logging_object`` (e.g. a stream that broke mid-flight), then
the provider the request itself declares (e.g. a Gemini pass-through
route), and finally infer it from the requested model name (e.g. ``gpt-4o-mini`` ->
``openai``) since the proxy's failure ``request_data`` usually carries
only the client-supplied model. Return ``None`` when it cannot be
determined so the label emits empty rather than a guess.
@ -2504,7 +2505,7 @@ class PrometheusLogger(CustomLogger):
if provider:
return provider
standard_logging_object: Final = request_data.get("standard_logging_object") or {}
provider = standard_logging_object.get("custom_llm_provider")
provider = standard_logging_object.get("custom_llm_provider") or request_data.get("custom_llm_provider")
if provider:
return provider
model: Final = litellm_params.get("model") or request_data.get("model")

39
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -58,6 +58,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
resolve_passthrough_managed_id_provider,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.vertex_ai.common_utils import get_vertex_model_id_from_url
from litellm.passthrough import BasePassthroughUtils
from litellm.proxy._types import (
ConfigFieldInfo,
@ -730,12 +731,23 @@ def _carry_guardrail_logging_info(request_data: dict, guardrail_data: dict | Non
metadata.setdefault("standard_logging_guardrail_information", list(entries))
def _passthrough_model_from_url(url: httpx.URL | None, custom_llm_provider: str | None) -> str | None:
if url is None:
return None
url_route: Final = str(url)
tracked_google_route: Final = pass_through_endpoint_logging.is_gemini_route(
url_route, custom_llm_provider
) or pass_through_endpoint_logging.is_vertex_route(url_route)
return get_vertex_model_id_from_url(url_route) if tracked_google_route else None
def _build_passthrough_failure_request_payload(
parsed_body: dict | None,
kwargs: dict | None,
logging_obj: LiteLLMLoggingObj | None,
custom_llm_provider: str | None,
upstream_usage: UpstreamReportedUsage | None = None,
url: httpx.URL | None = None,
) -> dict:
"""Build the ``request_data`` dict passed to ``post_call_failure_hook``.
@ -752,8 +764,8 @@ def _build_passthrough_failure_request_payload(
request_payload.update(kwargs)
if logging_obj is not None:
request_payload["litellm_logging_obj"] = logging_obj
if "model" not in request_payload and parsed_body and isinstance(parsed_body, dict):
request_payload["model"] = parsed_body.get("model", "")
if "model" not in request_payload:
request_payload["model"] = _passthrough_model_from_url(url, custom_llm_provider) or ""
if "custom_llm_provider" not in request_payload and custom_llm_provider:
request_payload["custom_llm_provider"] = custom_llm_provider
if upstream_usage is not None:
@ -1302,6 +1314,7 @@ async def pass_through_request(
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
url=url,
),
)
@ -1393,6 +1406,7 @@ async def pass_through_request(
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
url=url,
),
)
@ -1492,6 +1506,7 @@ async def pass_through_request(
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
url=url,
)
failure_request_payload["response_body"] = response_body
await _log_passthrough_upstream_failure(
@ -1697,19 +1712,13 @@ async def pass_through_request(
# Monitoring: Trigger post_call_failure_hook
# for pass through endpoint failure
#########################################################
request_payload: Final[dict] = _parsed_body or {}
# add user_api_key_dict, litellm_call_id, passthrough_logging_payloa for logging
if kwargs:
for key, value in kwargs.items():
request_payload[key] = value
if logging_obj is not None:
request_payload["litellm_logging_obj"] = logging_obj
if "model" not in request_payload and _parsed_body and isinstance(_parsed_body, dict):
request_payload["model"] = _parsed_body.get("model", "")
if "custom_llm_provider" not in request_payload and custom_llm_provider:
request_payload["custom_llm_provider"] = custom_llm_provider
request_payload: Final = _build_passthrough_failure_request_payload(
parsed_body=_parsed_body,
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
url=url,
)
_carry_guardrail_logging_info(request_payload, post_call_guardrail_data)
await proxy_logging_obj.post_call_failure_hook(

18
tests/test_litellm/integrations/test_prometheus_labels.py
View file

@ -235,6 +235,24 @@ def test_extract_api_provider_from_request_data_failure_path():
assert extract({"model": "some-unknown-model-xyz"}) is None
def test_extract_api_provider_prefers_declared_provider_over_model_name_inference():
from litellm.integrations.prometheus import PrometheusLogger
extract = PrometheusLogger._extract_api_provider_from_request_data
assert extract({"model": "gemini-3.8-flash"}) == "vertex_ai"
assert extract({"model": "gemini-3.8-flash", "custom_llm_provider": "gemini"}) == "gemini"
assert (
extract(
{
"custom_llm_provider": "gemini",
"standard_logging_object": {"custom_llm_provider": "vertex_ai"},
}
)
== "vertex_ai"
)
def test_extract_api_provider_swallows_unknown_model_but_logs_unexpected_errors():
"""
get_llm_provider raises BadRequestError for a model that maps to no

164
tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from fastapi import Request, UploadFile
from starlette.datastructures import FormData, Headers, QueryParams
from starlette.datastructures import UploadFile as StarletteUploadFile
@ -21,6 +22,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
HttpPassThroughEndpointHelpers,
InitPassThroughEndpointHelpers,
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
_build_passthrough_failure_request_payload,
_registered_pass_through_routes,
create_pass_through_route,
initialize_pass_through_endpoints,
@ -344,53 +346,18 @@ async def test_pass_through_request_failure_handler():
Critical Test: When a users pass through endpoint request fails, we must log the failure code, exception in litellm spend logs.
"""
with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging:
with patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_get_client:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.ProxyBaseLLMRequestProcessing"
) as mock_processing:
# Setup mock for post_call_failure_hook and pre_call_hook
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_proxy_logging.pre_call_hook = AsyncMock()
request_failure = httpx.HTTPError("Request failed")
user_api_key_dict = UserAPIKeyAuth(api_key="sk-test")
# Setup mock for httpx client
mock_client = MagicMock()
mock_client.client = MagicMock()
mock_client.client.request = AsyncMock(side_effect=httpx.HTTPError("Request failed"))
mock_get_client.return_value = mock_client
failure = await _run_passthrough_failure(
"http://test.com", None, request_failure, user_api_key_dict=user_api_key_dict
)
# Mock headers for custom headers
mock_processing.get_custom_headers.return_value = {}
# Create mock request
mock_request = MagicMock(spec=Request)
mock_request.method = "POST"
mock_request.body = AsyncMock(return_value=b'{"test": "data"}')
mock_request.headers = Headers({})
# Create a simple empty QueryParams
mock_request.query_params = QueryParams({})
# Create mock user API key dict
mock_user_api_key_dict = MagicMock()
# Call the function with a target that will trigger an HTTPError
with pytest.raises(ProxyException):
await pass_through_request(
request=mock_request,
target="http://test.com",
custom_headers={},
user_api_key_dict=mock_user_api_key_dict,
)
# Assert post_call_failure_hook was called
mock_proxy_logging.post_call_failure_hook.assert_called_once()
# Verify the arguments to post_call_failure_hook
call_args = mock_proxy_logging.post_call_failure_hook.call_args[1]
assert call_args["user_api_key_dict"] == mock_user_api_key_dict
assert isinstance(call_args["original_exception"], TypeError) # Now expecting TypeError
assert "traceback_str" in call_args
assert failure["user_api_key_dict"] is user_api_key_dict
assert failure["original_exception"] is request_failure
assert "traceback_str" in failure
assert failure["request_data"]["contents"] == "not a list"
assert failure["request_data"]["model"] == ""
@pytest.mark.asyncio
@ -5837,3 +5804,110 @@ def test_passthrough_client_cannot_forge_session_id_omission(client_metadata_key
)
== "per-call-random-trace-id"
)
_GEMINI_FLASH_TARGET = "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:generateContent"
_VERTEX_FLASH_TARGET = (
"https://us-east5-aiplatform.googleapis.com/v1/projects/p/locations/us-east5"
"/publishers/google/models/gemini-3.8-flash:generateContent"
)
async def _run_passthrough_failure(
target: str,
custom_llm_provider: str | None,
upstream: httpx.Response | Exception,
stream: bool = False,
user_api_key_dict: UserAPIKeyAuth | None = None,
) -> dict:
with (
pytest.MonkeyPatch.context() as transport_patch,
patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging,
respx.mock(assert_all_called=False) as upstream_mock,
):
transport_patch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
upstream_route = upstream_mock.route(method="POST")
if isinstance(upstream, Exception):
upstream_route.mock(side_effect=upstream)
else:
upstream_route.mock(return_value=upstream)
mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=None)
mock_request = MagicMock(spec=Request)
mock_request.method = "POST"
mock_request.url = target.replace("https://", "http://test-proxy.com/")
mock_request.body = AsyncMock(return_value=b'{"contents": "not a list"}')
mock_request.headers = Headers({"content-type": "application/json"})
mock_request.query_params = QueryParams({})
try:
await pass_through_request(
request=mock_request,
target=target,
custom_headers={},
user_api_key_dict=user_api_key_dict or UserAPIKeyAuth(api_key="sk-test"),
custom_llm_provider=custom_llm_provider,
stream=stream,
)
except ProxyException:
pass
litellm.in_memory_llm_clients_cache.flush_cache()
mock_proxy_logging.post_call_failure_hook.assert_called_once()
return mock_proxy_logging.post_call_failure_hook.call_args.kwargs
def _upstream_400(target: str) -> httpx.Response:
return httpx.Response(
status_code=400,
headers={"content-type": "application/json"},
content=json.dumps({"error": {"code": 400, "status": "INVALID_ARGUMENT"}}).encode(),
request=httpx.Request("POST", target),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"target, custom_llm_provider, stream",
[
(_GEMINI_FLASH_TARGET, "gemini", False),
(_GEMINI_FLASH_TARGET, "gemini", True),
(_VERTEX_FLASH_TARGET, None, False),
],
)
async def test_passthrough_upstream_error_logs_model_from_url(
target: str, custom_llm_provider: str | None, stream: bool
):
failure = await _run_passthrough_failure(target, custom_llm_provider, _upstream_400(target), stream=stream)
assert failure["request_data"]["model"] == "gemini-3.8-flash"
@pytest.mark.asyncio
async def test_passthrough_internal_failure_logs_model_from_url():
failure = await _run_passthrough_failure(
_GEMINI_FLASH_TARGET, "gemini", httpx.ReadTimeout("upstream timed out")
)
assert failure["request_data"]["model"] == "gemini-3.8-flash"
@pytest.mark.parametrize(
"parsed_body, url, custom_llm_provider, expected_model",
[
({"model": "from-body"}, _GEMINI_FLASH_TARGET, "gemini", "from-body"),
({"contents": []}, _GEMINI_FLASH_TARGET, "gemini", "gemini-3.8-flash"),
({"contents": []}, _VERTEX_FLASH_TARGET, None, "gemini-3.8-flash"),
({"contents": []}, "https://api.example.com/v1/models/gpt-x:thing", None, ""),
(None, None, None, ""),
],
)
def test_build_passthrough_failure_request_payload_model_precedence(
parsed_body: dict | None, url: str | None, custom_llm_provider: str | None, expected_model: str
):
request_payload = _build_passthrough_failure_request_payload(
parsed_body=parsed_body,
kwargs=None,
logging_obj=None,
custom_llm_provider=custom_llm_provider,
url=httpx.URL(url) if url else None,
)
assert request_payload["model"] == expected_model