mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): write SpendLogs failure rows for config-defined pass-through endpoints
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
327515a3ba
commit
33f595bccc
4 changed files with 78 additions and 7 deletions
|
|
@ -152,13 +152,13 @@ class _ProxyDBLogger(CustomLogger):
|
|||
)
|
||||
|
||||
request_route: Final = user_api_key_dict.request_route
|
||||
if (
|
||||
_ProxyDBLogger._should_track_errors_in_db() is False
|
||||
or request_route is not None
|
||||
and not (
|
||||
RouteChecks.is_llm_api_route(route=request_route) or RouteChecks.is_info_route(route=request_route)
|
||||
)
|
||||
):
|
||||
route_is_trackable: Final = (
|
||||
request_route is None
|
||||
or RouteChecks.is_llm_api_route(route=request_route)
|
||||
or RouteChecks.is_info_route(route=request_route)
|
||||
or request_data.get("call_type") == CallTypes.pass_through.value
|
||||
)
|
||||
if _ProxyDBLogger._should_track_errors_in_db() is False or not route_is_trackable:
|
||||
return
|
||||
|
||||
_metadata = dict(
|
||||
|
|
|
|||
|
|
@ -778,6 +778,12 @@ def _build_passthrough_failure_request_payload(
|
|||
request_payload.update(kwargs)
|
||||
if logging_obj is not None:
|
||||
request_payload["litellm_logging_obj"] = logging_obj
|
||||
logging_litellm_params: Final = logging_obj.model_call_details.get("litellm_params", {})
|
||||
existing_litellm_params: Final = request_payload.get("litellm_params", {})
|
||||
if isinstance(logging_litellm_params, dict) and isinstance(existing_litellm_params, dict):
|
||||
api_base: Final = logging_litellm_params.get("api_base")
|
||||
if isinstance(api_base, str) and api_base and "api_base" not in existing_litellm_params:
|
||||
request_payload["litellm_params"] = {**existing_litellm_params, "api_base": api_base}
|
||||
if "model" not in request_payload and parsed_body and isinstance(parsed_body, dict):
|
||||
request_payload["model"] = parsed_body.get("model", "")
|
||||
if "custom_llm_provider" not in request_payload and custom_llm_provider:
|
||||
|
|
|
|||
|
|
@ -267,6 +267,36 @@ async def test_async_post_call_failure_hook_non_llm_route():
|
|||
mock_update_database.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_post_call_failure_hook_tracks_config_defined_pass_through_route():
|
||||
logger = _ProxyDBLogger()
|
||||
api_base = "http://upstream.example/indexes/x/docs/search"
|
||||
request_data = {
|
||||
"model": "azure-search",
|
||||
"call_type": CallTypes.pass_through.value,
|
||||
"litellm_params": {"api_base": api_base, "metadata": {}},
|
||||
}
|
||||
user_api_key_dict = UserAPIKeyAuth(
|
||||
api_key="test_api_key",
|
||||
request_route="/azure-search",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_update_database:
|
||||
await logger.async_post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=Exception("Provider error"),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
mock_update_database.assert_called_once()
|
||||
call_args = mock_update_database.call_args
|
||||
assert call_args.kwargs["kwargs"]["litellm_params"]["api_base"] == api_base
|
||||
assert call_args.kwargs["kwargs"]["litellm_params"]["metadata"]["status"] == "failure"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_post_call_failure_hook_releases_budget_reservation_before_route_skip():
|
||||
logger = _ProxyDBLogger()
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import os
|
|||
import sys
|
||||
from collections.abc import Callable
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
|
|
@ -26,6 +27,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
|||
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
|
||||
HttpPassThroughEndpointHelpers,
|
||||
InitPassThroughEndpointHelpers,
|
||||
_build_passthrough_failure_request_payload,
|
||||
_registered_pass_through_routes,
|
||||
chat_completion_pass_through_endpoint,
|
||||
create_pass_through_route,
|
||||
|
|
@ -44,10 +46,43 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
|||
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
)
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
MESSAGE_START_SSE_FRAME = b'event: message_start\ndata: {"type": "message_start"}\n\n'
|
||||
|
||||
|
||||
def test_build_passthrough_failure_request_payload_carries_masked_api_base():
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="gpt-4o",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type=CallTypes.pass_through.value,
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-call-id",
|
||||
function_id="test-function-id",
|
||||
)
|
||||
logging_obj.pre_call(
|
||||
input={},
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": {},
|
||||
"api_base": "http://upstream.example/search?api-version=1",
|
||||
"headers": {},
|
||||
},
|
||||
)
|
||||
kwargs = {"litellm_params": {"metadata": {}}}
|
||||
|
||||
payload = _build_passthrough_failure_request_payload(
|
||||
parsed_body={},
|
||||
kwargs=kwargs,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=None,
|
||||
)
|
||||
|
||||
assert payload["litellm_params"]["api_base"] == logging_obj.model_call_details["litellm_params"]["api_base"]
|
||||
assert "api_base" not in kwargs["litellm_params"]
|
||||
|
||||
|
||||
def test_with_trace_context_without_opentelemetry(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setitem(sys.modules, "litellm.integrations.otel.plumbing.context", None)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue