scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

fix(proxy): write SpendLogs failure rows for config-defined pass-through endpoints (#41831)

Browse source 
* fix(proxy): write SpendLogs failure rows for config-defined pass-through endpoints

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(proxy): simplify pass-through failure api_base lookup

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): exempt pass-through spend log regression from quality gate

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(proxy): reuse failure payload builder in pass-through exception path

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(integration): cover config-defined pass-through failure spend rows

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): cover failure spend route eligibility

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(proxy): check configured routes for pass-through failure logs

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: kerry <kerry@berri.ai>
Co-authored-by: kenneth <kenneth@berri.ai>
This commit is contained in:
devin-ai-integration[bot] 2026-11-09 17:05:10 -08:00 • committed by GitHub
parent a053bf2a5c
commit 13df479397
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 173 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -479,15 +479,20 @@ class _ProxyDBLogger(CustomLogger):
return
request_route: Final = user_api_key_dict.request_route
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers
route_is_trackable: Final = (
request_route is None
or RouteChecks.is_llm_api_route(route=request_route)
or RouteChecks.is_info_route(route=request_route)
or InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash(route=request_route)
)
if (
ProxyDBLogger._should_track_errors_in_db() is False
and not completed_cost
or ProxyDBLogger._skips_pre_routing_rejections()
and is_pre_routing_rejection(original_exception, _REQUEST_DATA.validate_python(request_data))
or request_route is not None
and not (
RouteChecks.is_llm_api_route(route=request_route) or RouteChecks.is_info_route(route=request_route)
)
or not route_is_trackable
or request_route is not None
and RouteChecks.is_dashboard_session_info_route(request_route, user_api_key_dict.team_id)
):

9
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -964,11 +964,16 @@ def _build_passthrough_failure_request_payload(
model and provider. A user-defined route never gets that, since its URL
names an upstream model rather than a LiteLLM-managed one.
"""
request_payload: Final[dict] = dict(cast(Mapping[str, object], parsed_body or {})) # cast-ok: json body
request_payload: Final = dict(cast(Mapping[str, object], parsed_body or {})) # cast-ok: json body
if kwargs:
request_payload.update(kwargs)
request_payload.update(_json_object(_DECODED_JSON.validate_python(kwargs)))
if logging_obj is not None:
request_payload["litellm_logging_obj"] = logging_obj
logging_litellm_params: Final = _json_object(logging_obj.model_call_details.get("litellm_params"))
existing_litellm_params: Final = _json_object(request_payload.get("litellm_params"))
api_base: Final = logging_litellm_params.get("api_base")
if isinstance(api_base, str) and api_base and "api_base" not in existing_litellm_params:
request_payload["litellm_params"] = {**existing_litellm_params, "api_base": api_base}
url_route: Final = str(url) if url is not None else ""
google_provider: Final = (
_passthrough_google_provider(url_route, custom_llm_provider) if builtin_provider_route else None

53
tests/integration/spend/test_passthrough_failure.py Normal file
View file

@ -0,0 +1,53 @@
import json
from typing import Final
from pathlib import Path
import pytest
from tests.integration._support.client import Gateway, eventually
from tests.integration._support.database import read_rows
from tests.integration._support.process import owned_proxy
from tests.integration._support.wire import Reply, Request, wire_server
@pytest.mark.timeout(180)
def test_config_pass_through_endpoint_failure_writes_a_spend_row(gateway: Gateway, tmp_path: Path) -> None:
def respond(request: Request) -> Reply:
assert request.target == "/"
assert json.loads(request.body) == {"search": "hello"}
return Reply(status=500, body=json.dumps({"error": "upstream search failure"}).encode())
with wire_server(respond) as wire:
config: Final = tmp_path / "proxy_config.yaml"
config.write_text(
"model_list: []\n"
"general_settings:\n"
" master_key: os.environ/LITELLM_MASTER_KEY\n"
" database_url: os.environ/DATABASE_URL\n"
" store_model_in_db: true\n"
" disable_spend_logs: false\n"
" proxy_batch_write_at: 1\n"
" pass_through_endpoints:\n"
" - path: /azure-search\n"
f" target: {wire.url}\n"
" headers:\n"
" Authorization: Bearer synthetic-search-key\n"
"router_settings:\n"
" disable_cooldowns: true\n"
)
with owned_proxy(gateway, tmp_path, {}, config=config) as candidate:
response: Final = candidate.request("POST", "/azure-search", {"search": "hello"})
assert response.status_code == 500, response.text
call_id: Final = response.headers["x-litellm-call-id"]
rows: Final = eventually(
lambda: read_rows(
'SELECT status, call_type, api_base, spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
(call_id,),
database_url=candidate.database_url,
),
lambda values: len(values) == 1,
seconds=70,
)
assert rows[0]["status"] == "failure", rows
assert rows[0]["call_type"] == "pass_through_endpoint", rows
assert "127.0.0.1" in rows[0]["api_base"], rows
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/")]

71
tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
View file

@ -13,7 +13,7 @@ import pytest
from litellm.redis.stores.connection import connect
from litellm.redis.stores.spend_counters import SpendCounterStore
from fastapi import HTTPException
from fastapi import FastAPI, HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
@ -35,6 +35,7 @@ from litellm.proxy.hooks.proxy_track_cost_callback import (
run_spend_event,
)
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers
from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event
from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
@ -907,6 +908,74 @@ async def test_async_post_call_failure_hook_non_llm_route():
mock_update_database.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("route", "call_type", "written"),
[
("/azure-search", CallTypes.pass_through.value, True),
("/unregistered-search", CallTypes.acompletion.value, False),
("/key/generate", CallTypes.pass_through.value, False),
("/team/new", CallTypes.pass_through.value, False),
("/unregistered-search", CallTypes.pass_through.value, False),
("/azure-search/", CallTypes.pass_through.value, True),
("/configured-search/indexes/docs", CallTypes.pass_through.value, True),
(None, CallTypes.acompletion.value, True),
],
)
async def test_async_post_call_failure_hook_tracks_config_defined_pass_through_route(
route: str | None, call_type: str, written: bool
) -> None:
api_base: Final = "http://upstream.example/indexes/x/docs/search"
request_data: Final = {
"model": "azure-search",
"call_type": call_type,
"litellm_params": {"api_base": api_base, "metadata": {}},
}
user_api_key_dict: Final = UserAPIKeyAuth(
api_key="test_api_key",
request_route=route,
)
spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True)
with patch.dict(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", clear=True
):
InitPassThroughEndpointHelpers.add_exact_path_route(
app=FastAPI(),
path="/azure-search",
target=api_base,
custom_headers={},
forward_headers=False,
merge_query_params=False,
dependencies=[],
cost_per_request=None,
endpoint_id="configured-exact",
)
InitPassThroughEndpointHelpers.add_subpath_route(
app=FastAPI(),
path="/configured-search",
target=api_base,
custom_headers={},
forward_headers=False,
merge_query_params=False,
dependencies=[],
cost_per_request=None,
endpoint_id="configured-subpath",
)
await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook(
request_data=request_data,
original_exception=Exception("Provider error"),
user_api_key_dict=user_api_key_dict,
)
assert spend_writer.update_database.await_count == (1 if written else 0)
if not written:
return
call_args: Final = spend_writer.update_database.call_args
assert call_args.kwargs["kwargs"]["litellm_params"]["api_base"] == api_base
assert call_args.kwargs["kwargs"]["litellm_params"]["metadata"]["status"] == "failure"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("team_id", "route", "written"),

34
tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -449,6 +449,39 @@ async def test_non_streaming_http_request_handler_multipart_with_non_empty_parse
assert call_args["files"] == [("file", ("test.txt", file_content, "text/plain"))]
@pytest.mark.parametrize("existing_api_base", [None, "http://existing.example/search"])
def test_build_passthrough_failure_request_payload_carries_masked_api_base(existing_api_base: str | None) -> None:
logging_obj: Final = LiteLLMLoggingObj(
model="passthrough",
messages=[],
stream=False,
call_type=CallTypes.pass_through.value,
start_time=datetime(2026, 1, 1),
litellm_call_id="test-call-id",
function_id="test-function-id",
)
logging_obj.pre_call(
input={},
api_key="",
additional_args={
"complete_input_dict": {},
"api_base": "http://upstream.example/search?api-version=1",
"headers": {},
},
)
params: Final = {"metadata": {}, **({"api_base": existing_api_base} if existing_api_base else {})}
payload: Final = _build_passthrough_failure_request_payload(
parsed_body={},
kwargs={"litellm_params": params},
logging_obj=logging_obj,
custom_llm_provider=None,
)
assert payload["litellm_params"]["api_base"] == (
existing_api_base or logging_obj.model_call_details["litellm_params"]["api_base"]
)
assert params == {"metadata": {}, **({"api_base": existing_api_base} if existing_api_base else {})}
@pytest.mark.asyncio
async def test_pass_through_request_failure_handler():
"""
@ -468,6 +501,7 @@ async def test_pass_through_request_failure_handler():
assert "Request failed" in (failure.traceback_str or "")
assert failure.request_data["contents"] == "not a list"
assert failure.request_data["model"] == ""
assert failure.request_data["litellm_params"]["api_base"] == "http://test.com"
@pytest.mark.asyncio