From 13df479397bdd37bdbfd5a57c63edcebe9a074bf Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:05:10 -0800 Subject: [PATCH] fix(proxy): write SpendLogs failure rows for config-defined pass-through endpoints (#41831) * fix(proxy): write SpendLogs failure rows for config-defined pass-through endpoints Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): simplify pass-through failure api_base lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): exempt pass-through spend log regression from quality gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): reuse failure payload builder in pass-through exception path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover config-defined pass-through failure spend rows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): cover failure spend route eligibility Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): check configured routes for pass-through failure logs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: kerry Co-authored-by: kenneth --- .../proxy/hooks/proxy_track_cost_callback.py | 13 ++-- .../pass_through_endpoints.py | 9 ++- .../spend/test_passthrough_failure.py | 53 ++++++++++++++ .../hooks/test_proxy_track_cost_callback.py | 71 ++++++++++++++++++- .../test_pass_through_endpoints.py | 34 +++++++++ 5 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 tests/integration/spend/test_passthrough_failure.py diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 133731fc335..dd7a24ae879 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -479,15 +479,20 @@ class _ProxyDBLogger(CustomLogger): return request_route: Final = user_api_key_dict.request_route + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + + route_is_trackable: Final = ( + request_route is None + or RouteChecks.is_llm_api_route(route=request_route) + or RouteChecks.is_info_route(route=request_route) + or InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash(route=request_route) + ) if ( ProxyDBLogger._should_track_errors_in_db() is False and not completed_cost or ProxyDBLogger._skips_pre_routing_rejections() and is_pre_routing_rejection(original_exception, _REQUEST_DATA.validate_python(request_data)) - or request_route is not None - and not ( - RouteChecks.is_llm_api_route(route=request_route) or RouteChecks.is_info_route(route=request_route) - ) + or not route_is_trackable or request_route is not None and RouteChecks.is_dashboard_session_info_route(request_route, user_api_key_dict.team_id) ): diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 3e25f7ce9af..4b84fe19559 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -964,11 +964,16 @@ def _build_passthrough_failure_request_payload( model and provider. A user-defined route never gets that, since its URL names an upstream model rather than a LiteLLM-managed one. """ - request_payload: Final[dict] = dict(cast(Mapping[str, object], parsed_body or {})) # cast-ok: json body + request_payload: Final = dict(cast(Mapping[str, object], parsed_body or {})) # cast-ok: json body if kwargs: - request_payload.update(kwargs) + request_payload.update(_json_object(_DECODED_JSON.validate_python(kwargs))) if logging_obj is not None: request_payload["litellm_logging_obj"] = logging_obj + logging_litellm_params: Final = _json_object(logging_obj.model_call_details.get("litellm_params")) + existing_litellm_params: Final = _json_object(request_payload.get("litellm_params")) + api_base: Final = logging_litellm_params.get("api_base") + if isinstance(api_base, str) and api_base and "api_base" not in existing_litellm_params: + request_payload["litellm_params"] = {**existing_litellm_params, "api_base": api_base} url_route: Final = str(url) if url is not None else "" google_provider: Final = ( _passthrough_google_provider(url_route, custom_llm_provider) if builtin_provider_route else None diff --git a/tests/integration/spend/test_passthrough_failure.py b/tests/integration/spend/test_passthrough_failure.py new file mode 100644 index 00000000000..cb102c406f7 --- /dev/null +++ b/tests/integration/spend/test_passthrough_failure.py @@ -0,0 +1,53 @@ +import json +from typing import Final +from pathlib import Path + +import pytest +from tests.integration._support.client import Gateway, eventually +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy +from tests.integration._support.wire import Reply, Request, wire_server + + +@pytest.mark.timeout(180) +def test_config_pass_through_endpoint_failure_writes_a_spend_row(gateway: Gateway, tmp_path: Path) -> None: + def respond(request: Request) -> Reply: + assert request.target == "/" + assert json.loads(request.body) == {"search": "hello"} + return Reply(status=500, body=json.dumps({"error": "upstream search failure"}).encode()) + + with wire_server(respond) as wire: + config: Final = tmp_path / "proxy_config.yaml" + config.write_text( + "model_list: []\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + " store_model_in_db: true\n" + " disable_spend_logs: false\n" + " proxy_batch_write_at: 1\n" + " pass_through_endpoints:\n" + " - path: /azure-search\n" + f" target: {wire.url}\n" + " headers:\n" + " Authorization: Bearer synthetic-search-key\n" + "router_settings:\n" + " disable_cooldowns: true\n" + ) + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + response: Final = candidate.request("POST", "/azure-search", {"search": "hello"}) + assert response.status_code == 500, response.text + call_id: Final = response.headers["x-litellm-call-id"] + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, call_type, api_base, spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (call_id,), + database_url=candidate.database_url, + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["status"] == "failure", rows + assert rows[0]["call_type"] == "pass_through_endpoint", rows + assert "127.0.0.1" in rows[0]["api_base"], rows + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/")] diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 0283c90f3d6..43f529faa6b 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -13,7 +13,7 @@ import pytest from litellm.redis.stores.connection import connect from litellm.redis.stores.spend_counters import SpendCounterStore -from fastapi import HTTPException +from fastapi import FastAPI, HTTPException import litellm from litellm._logging import verbose_proxy_logger @@ -35,6 +35,7 @@ from litellm.proxy.hooks.proxy_track_cost_callback import ( run_spend_event, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload @@ -907,6 +908,74 @@ async def test_async_post_call_failure_hook_non_llm_route(): mock_update_database.assert_not_called() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route", "call_type", "written"), + [ + ("/azure-search", CallTypes.pass_through.value, True), + ("/unregistered-search", CallTypes.acompletion.value, False), + ("/key/generate", CallTypes.pass_through.value, False), + ("/team/new", CallTypes.pass_through.value, False), + ("/unregistered-search", CallTypes.pass_through.value, False), + ("/azure-search/", CallTypes.pass_through.value, True), + ("/configured-search/indexes/docs", CallTypes.pass_through.value, True), + (None, CallTypes.acompletion.value, True), + ], +) +async def test_async_post_call_failure_hook_tracks_config_defined_pass_through_route( + route: str | None, call_type: str, written: bool +) -> None: + api_base: Final = "http://upstream.example/indexes/x/docs/search" + request_data: Final = { + "model": "azure-search", + "call_type": call_type, + "litellm_params": {"api_base": api_base, "metadata": {}}, + } + user_api_key_dict: Final = UserAPIKeyAuth( + api_key="test_api_key", + request_route=route, + ) + + spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True) + with patch.dict( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", clear=True + ): + InitPassThroughEndpointHelpers.add_exact_path_route( + app=FastAPI(), + path="/azure-search", + target=api_base, + custom_headers={}, + forward_headers=False, + merge_query_params=False, + dependencies=[], + cost_per_request=None, + endpoint_id="configured-exact", + ) + InitPassThroughEndpointHelpers.add_subpath_route( + app=FastAPI(), + path="/configured-search", + target=api_base, + custom_headers={}, + forward_headers=False, + merge_query_params=False, + dependencies=[], + cost_per_request=None, + endpoint_id="configured-subpath", + ) + await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook( + request_data=request_data, + original_exception=Exception("Provider error"), + user_api_key_dict=user_api_key_dict, + ) + + assert spend_writer.update_database.await_count == (1 if written else 0) + if not written: + return + call_args: Final = spend_writer.update_database.call_args + assert call_args.kwargs["kwargs"]["litellm_params"]["api_base"] == api_base + assert call_args.kwargs["kwargs"]["litellm_params"]["metadata"]["status"] == "failure" + + @pytest.mark.asyncio @pytest.mark.parametrize( ("team_id", "route", "written"), diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 464c39d0ba4..31b7b499845 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -449,6 +449,39 @@ async def test_non_streaming_http_request_handler_multipart_with_non_empty_parse assert call_args["files"] == [("file", ("test.txt", file_content, "text/plain"))] +@pytest.mark.parametrize("existing_api_base", [None, "http://existing.example/search"]) +def test_build_passthrough_failure_request_payload_carries_masked_api_base(existing_api_base: str | None) -> None: + logging_obj: Final = LiteLLMLoggingObj( + model="passthrough", + messages=[], + stream=False, + call_type=CallTypes.pass_through.value, + start_time=datetime(2026, 1, 1), + litellm_call_id="test-call-id", + function_id="test-function-id", + ) + logging_obj.pre_call( + input={}, + api_key="", + additional_args={ + "complete_input_dict": {}, + "api_base": "http://upstream.example/search?api-version=1", + "headers": {}, + }, + ) + params: Final = {"metadata": {}, **({"api_base": existing_api_base} if existing_api_base else {})} + payload: Final = _build_passthrough_failure_request_payload( + parsed_body={}, + kwargs={"litellm_params": params}, + logging_obj=logging_obj, + custom_llm_provider=None, + ) + assert payload["litellm_params"]["api_base"] == ( + existing_api_base or logging_obj.model_call_details["litellm_params"]["api_base"] + ) + assert params == {"metadata": {}, **({"api_base": existing_api_base} if existing_api_base else {})} + + @pytest.mark.asyncio async def test_pass_through_request_failure_handler(): """ @@ -468,6 +501,7 @@ async def test_pass_through_request_failure_handler(): assert "Request failed" in (failure.traceback_str or "") assert failure.request_data["contents"] == "not a list" assert failure.request_data["model"] == "" + assert failure.request_data["litellm_params"]["api_base"] == "http://test.com" @pytest.mark.asyncio