mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
Merge pull request #41943 from BerriAI/litellm_unknown_model_spend_logs_outside_router
fix(proxy): keep the raw client model out of spend logs for rejections outside the router
This commit is contained in:
Mateo Wang
GitHub
commit
f15dba2e2f
6 changed files with 532 additions and 16 deletions
|
|
@ -367,6 +367,8 @@ def get_credentials_for_model(
|
|||
"""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
if llm_router is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
|
|
@ -376,9 +378,8 @@ def get_credentials_for_model(
|
|||
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
|
||||
|
||||
if credentials is None:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": f"Model '{model_id}' not found in model_list. Please check your config.yaml."},
|
||||
raise ProxyModelNotFoundError(
|
||||
route=operation_context, model_name=model_id, retryable_with_model_read_through=False
|
||||
)
|
||||
|
||||
return credentials
|
||||
|
|
@ -683,7 +684,7 @@ async def handle_model_based_routing(
|
|||
llm_router=llm_router,
|
||||
model_id=model_from_id,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
operation_context=f"file operation (file created with model '{model_from_id}')",
|
||||
operation_context="file operation (file created with model)",
|
||||
)
|
||||
original_file_id: Final = get_original_file_id(file_id)
|
||||
return True, model_from_id, original_file_id, credentials
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
LiteLLMProxyRequestSetup,
|
||||
_get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above
|
||||
)
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
from litellm.proxy.utils import normalize_route_for_root_path
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
@ -281,9 +282,8 @@ async def chat_completion_pass_through_endpoint(
|
|||
elif user_model is not None: # `litellm --model <your-model-name>`
|
||||
llm_response = asyncio.create_task(litellm.aadapter_completion(**data))
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={"error": "completion: Invalid model name passed in model=" + data.get("model", "")},
|
||||
raise ProxyModelNotFoundError(
|
||||
route="completion", model_name=data.get("model", ""), retryable_with_model_read_through=False
|
||||
)
|
||||
|
||||
# Await the llm_response task
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime as dt
|
||||
from functools import reduce
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable
|
||||
|
||||
|
|
@ -380,11 +382,81 @@ def _model_group_provider(model_group: str, llm_router: "Router | None") -> str
|
|||
return next(iter(providers)) if len(providers) == 1 else None
|
||||
|
||||
|
||||
def _is_configured_model_group(model_group: str, llm_router: "Router | None") -> bool:
|
||||
if llm_router is None or not model_group:
|
||||
return False
|
||||
return llm_router.is_recognized_model(model_group) or model_group in llm_router.team_public_model_names
|
||||
|
||||
|
||||
def _looks_like_model_name(model: str) -> bool:
|
||||
candidate: Final = model.removeprefix(MCP_SPEND_LOG_MODEL_PREFIX)
|
||||
return len(candidate) <= MAX_SPEND_LOG_MODEL_NAME_LENGTH and not any(char.isspace() for char in candidate)
|
||||
|
||||
|
||||
_TRUNCATION_MARKER: Final = re.compile(
|
||||
rf"\.\.\. \({re.escape(LITELLM_TRUNCATED_PAYLOAD_FIELD)} skipped \d+ chars\. "
|
||||
rf"{re.escape(LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE)}\) \.\.\."
|
||||
)
|
||||
_SCRUBBED_ERROR_TEXT_FIELDS: Final = frozenset(("error_message", "traceback"))
|
||||
|
||||
|
||||
def _raw_model_spellings(raw_model: str) -> tuple[str, ...]:
|
||||
return tuple(dict.fromkeys((raw_model, repr(raw_model)[1:-1], json.dumps(raw_model)[1:-1])))
|
||||
|
||||
|
||||
def _overlap_at_end(text: str, spelling: str) -> int:
|
||||
lengths: Final = range(min(len(text), len(spelling) - 1), 0, -1)
|
||||
return next((length for length in lengths if text.endswith(spelling[:length])), 0)
|
||||
|
||||
|
||||
def _overlap_at_start(text: str, spelling: str) -> int:
|
||||
lengths: Final = range(min(len(text), len(spelling) - 1), 0, -1)
|
||||
return next((length for length in lengths if text.startswith(spelling[-length:])), 0)
|
||||
|
||||
|
||||
def _scrub_raw_model_split_by_truncation(text: str, spellings: tuple[str, ...]) -> str:
|
||||
marker: Final = _TRUNCATION_MARKER.search(text)
|
||||
if marker is None:
|
||||
return text
|
||||
head: Final = text[: marker.start()]
|
||||
tail: Final = text[marker.end() :]
|
||||
head_cut: Final = max(_overlap_at_end(head, spelling) for spelling in spellings)
|
||||
tail_cut: Final = max(_overlap_at_start(tail, spelling) for spelling in spellings)
|
||||
return "".join(
|
||||
(
|
||||
head[: len(head) - head_cut],
|
||||
UNKNOWN_MODEL_SPEND_LOG_MODEL if head_cut else "",
|
||||
marker.group(0),
|
||||
UNKNOWN_MODEL_SPEND_LOG_MODEL if tail_cut else "",
|
||||
tail[tail_cut:],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _scrub_raw_model_from_error_text(text: str, spellings: tuple[str, ...]) -> str:
|
||||
whole_occurrences_scrubbed: Final = reduce(
|
||||
lambda scrubbed, spelling: scrubbed.replace(spelling, UNKNOWN_MODEL_SPEND_LOG_MODEL), spellings, text
|
||||
)
|
||||
return _scrub_raw_model_split_by_truncation(whole_occurrences_scrubbed, spellings)
|
||||
|
||||
|
||||
def _scrub_raw_model_from_error_information(
|
||||
error_information: StandardLoggingPayloadErrorInformation | None, raw_model: str
|
||||
) -> StandardLoggingPayloadErrorInformation | None:
|
||||
if error_information is None or not raw_model:
|
||||
return error_information
|
||||
spellings: Final = _raw_model_spellings(raw_model)
|
||||
return cast(
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
{
|
||||
key: _scrub_raw_model_from_error_text(value, spellings)
|
||||
if key in _SCRUBBED_ERROR_TEXT_FIELDS and isinstance(value, str)
|
||||
else value
|
||||
for key, value in error_information.items()
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def get_logging_payload(
|
||||
kwargs: dict | None,
|
||||
response_obj: object,
|
||||
|
|
@ -502,14 +574,29 @@ def get_logging_payload(
|
|||
)
|
||||
failed_with_prompt_shaped_model: Final = (
|
||||
_get_status_for_spend_log(metadata=metadata) == "failure"
|
||||
and not _model_group
|
||||
and not _model_id
|
||||
and not _looks_like_model_name(resolved_model)
|
||||
and not _is_configured_model_group(_model_group, llm_router)
|
||||
)
|
||||
model_name: Final = (
|
||||
UNKNOWN_MODEL_SPEND_LOG_MODEL
|
||||
if rejected_as_unknown_model or failed_with_prompt_shaped_model or model_is_malformed
|
||||
else resolved_model
|
||||
)
|
||||
model_is_placeholdered: Final = model_name == UNKNOWN_MODEL_SPEND_LOG_MODEL
|
||||
persisted_model_group: Final = (
|
||||
""
|
||||
if model_is_placeholdered and _model_group == raw_model and not _looks_like_model_name(raw_model)
|
||||
else _model_group
|
||||
)
|
||||
persisted_metadata: Final = (
|
||||
{
|
||||
**metadata,
|
||||
"error_information": _scrub_raw_model_from_error_information(metadata.get("error_information"), raw_model),
|
||||
}
|
||||
if model_is_placeholdered
|
||||
else metadata
|
||||
)
|
||||
litellm_call_id: Final = cast(
|
||||
str | None,
|
||||
kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),
|
||||
|
|
@ -517,7 +604,7 @@ def get_logging_payload(
|
|||
|
||||
# clean up litellm metadata
|
||||
clean_metadata = _get_spend_logs_metadata(
|
||||
metadata,
|
||||
persisted_metadata,
|
||||
applied_guardrails=(
|
||||
standard_logging_payload["metadata"].get("applied_guardrails", None)
|
||||
if standard_logging_payload is not None
|
||||
|
|
@ -576,7 +663,7 @@ def get_logging_payload(
|
|||
litellm_call_id=litellm_call_id,
|
||||
router_metadata=_get_router_metadata_for_spend_log(
|
||||
metadata=metadata,
|
||||
requested_model=_model_group,
|
||||
requested_model=persisted_model_group,
|
||||
selected_model=model_name,
|
||||
selected_provider=custom_llm_provider,
|
||||
router_correlation_id=litellm_call_id,
|
||||
|
|
@ -658,7 +745,7 @@ def get_logging_payload(
|
|||
request_tags=request_tags,
|
||||
end_user=end_user_id or "",
|
||||
api_base=_api_base,
|
||||
model_group=_model_group,
|
||||
model_group=persisted_model_group,
|
||||
model_id=_model_id,
|
||||
mcp_namespaced_tool_name=mcp_namespaced_tool_name,
|
||||
agent_id=agent_id,
|
||||
|
|
@ -669,7 +756,13 @@ def get_logging_payload(
|
|||
),
|
||||
response=_get_response_for_spend_logs_payload(payload=standard_logging_payload, kwargs=kwargs),
|
||||
proxy_server_request=_get_proxy_server_request_for_spend_logs_payload(
|
||||
metadata=metadata, litellm_params=litellm_params, kwargs=kwargs
|
||||
metadata=metadata,
|
||||
litellm_params=(
|
||||
_placeholder_stored_request_body(litellm_params, persisted_model_group, raw_model)
|
||||
if model_is_placeholdered
|
||||
else litellm_params
|
||||
),
|
||||
kwargs=kwargs,
|
||||
),
|
||||
session_id=_get_session_id_for_spend_log(
|
||||
kwargs=kwargs,
|
||||
|
|
@ -975,7 +1068,7 @@ def _sanitize_request_body_for_spend_logs_payload(
|
|||
visited.add(obj_id)
|
||||
|
||||
def _sanitize_value(value: object) -> object:
|
||||
if isinstance(value, dict):
|
||||
if isinstance(value, Mapping):
|
||||
return _sanitize_request_body_for_spend_logs_payload(value, visited, max_string_length_prompt_in_db)
|
||||
elif isinstance(value, list):
|
||||
return [_sanitize_value(item) for item in value]
|
||||
|
|
@ -1329,9 +1422,65 @@ def _convert_mapping_to_json_serializable(obj: Mapping[str, object]) -> dict[str
|
|||
return dict(obj)
|
||||
|
||||
|
||||
def _placeholder_stored_request_body_metadata(
|
||||
request_body: Mapping[str, object], persisted_model_group: str, raw_model: str
|
||||
) -> Mapping[str, object]:
|
||||
body_metadata: Final = request_body.get("metadata")
|
||||
if not isinstance(body_metadata, Mapping):
|
||||
return request_body
|
||||
error_information: Final = body_metadata.get("error_information")
|
||||
placeholdered_fields: Final = MappingProxyType(
|
||||
{
|
||||
"model_group": persisted_model_group,
|
||||
"error_information": _scrub_raw_model_from_error_information(
|
||||
cast(StandardLoggingPayloadErrorInformation, error_information), raw_model
|
||||
)
|
||||
if isinstance(error_information, Mapping)
|
||||
else error_information,
|
||||
}
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
**request_body,
|
||||
"metadata": MappingProxyType(
|
||||
{key: placeholdered_fields.get(key, value) for key, value in body_metadata.items()}
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _placeholder_stored_request_body(
|
||||
litellm_params: Mapping[str, object], persisted_model_group: str, raw_model: str
|
||||
) -> Mapping[str, object]:
|
||||
proxy_server_request: Final = litellm_params.get("proxy_server_request")
|
||||
if not isinstance(proxy_server_request, Mapping):
|
||||
return litellm_params
|
||||
request_body: Final = proxy_server_request.get("body")
|
||||
if not isinstance(request_body, Mapping):
|
||||
return litellm_params
|
||||
model_placeholdered: Final = (
|
||||
MappingProxyType({**request_body, "model": UNKNOWN_MODEL_SPEND_LOG_MODEL})
|
||||
if "model" in request_body
|
||||
else request_body
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
**litellm_params,
|
||||
"proxy_server_request": MappingProxyType(
|
||||
{
|
||||
**proxy_server_request,
|
||||
"body": _placeholder_stored_request_body_metadata(
|
||||
model_placeholdered, persisted_model_group, raw_model
|
||||
),
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _get_proxy_server_request_for_spend_logs_payload(
|
||||
metadata: dict,
|
||||
litellm_params: dict,
|
||||
litellm_params: Mapping[str, object],
|
||||
kwargs: dict | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -6,10 +7,31 @@ import pytest
|
|||
|
||||
from litellm.proxy.openai_files_endpoints.common_utils import (
|
||||
apply_unified_file_ids,
|
||||
get_credentials_for_model,
|
||||
map_raw_file_ids_to_unified,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
from litellm.proxy.utils import handle_exception_on_proxy
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
_RAW_MODEL_WITH_PROMPT: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes"
|
||||
|
||||
|
||||
def test_get_credentials_for_model_rejects_an_unknown_model_without_persisting_the_raw_model():
|
||||
llm_router: Final = MagicMock()
|
||||
llm_router.get_deployment_credentials_with_provider.return_value = None
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError) as raised:
|
||||
get_credentials_for_model(
|
||||
llm_router=llm_router, model_id=_RAW_MODEL_WITH_PROMPT, operation_context="file upload"
|
||||
)
|
||||
|
||||
assert (raised.value.status_code, handle_exception_on_proxy(raised.value).code) == (400, "400")
|
||||
assert _RAW_MODEL_WITH_PROMPT in raised.value.detail["error"]
|
||||
assert raised.value.retryable_with_model_read_through is False
|
||||
assert raised.value.spend_log_error_message.startswith("file upload: ")
|
||||
assert "medical records" not in raised.value.spend_log_error_message
|
||||
|
||||
|
||||
def _batch(input_file_id, output_file_id, error_file_id) -> LiteLLMBatch:
|
||||
return LiteLLMBatch(
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from collections.abc import Callable
|
|||
from contextlib import ExitStack, contextmanager
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -38,6 +39,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
|||
from litellm.proxy.pass_through_endpoints.success_handler import (
|
||||
PassThroughEndpointLogging,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
|
|
@ -6443,6 +6445,46 @@ async def test_chat_completion_pass_through_endpoint_answers_an_openai_typed_err
|
|||
assert (raised.value.type, raised.value.param, raised.value.code) == ("invalid_request_error", None, "400")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_pass_through_endpoint_keeps_the_raw_model_out_of_the_spend_log_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes"
|
||||
proxy_logging: Final = MagicMock()
|
||||
proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
|
||||
proxy_logging.post_call_failure_hook = AsyncMock()
|
||||
|
||||
async def fake_add_litellm_data_to_request(**kwargs: object) -> object:
|
||||
return kwargs["data"]
|
||||
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.add_litellm_data_to_request", fake_add_litellm_data_to_request)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.user_model", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
|
||||
|
||||
request: Final = MagicMock(spec=Request)
|
||||
request.body = AsyncMock(
|
||||
return_value=json.dumps({"model": raw_model, "messages": [{"role": "user", "content": "hi"}]}).encode()
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as raised:
|
||||
await chat_completion_pass_through_endpoint(
|
||||
fastapi_response=Response(),
|
||||
request=request,
|
||||
adapter_id="anthropic",
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
)
|
||||
|
||||
logged_exception: Final = proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"]
|
||||
assert isinstance(logged_exception, ProxyModelNotFoundError)
|
||||
assert logged_exception.retryable_with_model_read_through is False
|
||||
assert logged_exception.spend_log_error_message.startswith("completion: ")
|
||||
assert "medical records" not in logged_exception.spend_log_error_message
|
||||
assert (raised.value.type, raised.value.param, raised.value.code) == ("invalid_request_error", None, "400")
|
||||
assert raw_model in logged_exception.detail["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_pass_through_endpoint_failure_carries_the_callers_litellm_call_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
|
|||
_sanitize_error_information_for_spend_logs,
|
||||
_sanitize_guardrail_information_for_spend_logs,
|
||||
_sanitize_request_body_for_spend_logs_payload,
|
||||
_scrub_raw_model_from_error_information,
|
||||
get_logging_payload,
|
||||
get_spend_logs_id,
|
||||
should_store_prompts_and_responses_in_spend_logs,
|
||||
|
|
@ -50,6 +51,7 @@ from litellm.types.utils import (
|
|||
StandardLoggingMetadata,
|
||||
StandardLoggingModelInformation,
|
||||
StandardLoggingPayload,
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1075,13 +1077,18 @@ def test_get_logging_payload_replaces_a_non_string_model_with_the_placeholder(
|
|||
[
|
||||
({"user_api_key": "sk-test"}, litellm.ModelResponse(id="chatcmpl-test", choices=[])),
|
||||
(
|
||||
{"user_api_key": "sk-test", "model_group": "team alias", "status": "failure"},
|
||||
{
|
||||
"user_api_key": "sk-test",
|
||||
"model_group": "team alias",
|
||||
"model_info": {"id": "team-alias-deployment"},
|
||||
"status": "failure",
|
||||
},
|
||||
ValueError("provider timed out"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_logging_payload_keeps_a_whitespace_model_name_on_success_or_a_routed_failure(
|
||||
metadata: dict[str, str], response_obj: litellm.ModelResponse | Exception
|
||||
metadata: dict[str, object], response_obj: litellm.ModelResponse | Exception
|
||||
):
|
||||
kwargs: Final = {
|
||||
"model": _RAW_MODEL_WITH_PROMPT,
|
||||
|
|
@ -1100,6 +1107,301 @@ def test_get_logging_payload_keeps_a_whitespace_model_name_on_success_or_a_route
|
|||
assert payload["model"] == _RAW_MODEL_WITH_PROMPT
|
||||
|
||||
|
||||
@pytest.mark.parametrize("redact_messages", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("metadata", "expected_stored_model"),
|
||||
[
|
||||
({"user_api_key": "sk-test", "status": "failure"}, UNKNOWN_MODEL_SPEND_LOG_MODEL),
|
||||
(
|
||||
{"user_api_key": "sk-test", "status": "failure", "model_info": {"id": "routed-deployment"}},
|
||||
_RAW_MODEL_WITH_PROMPT,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_logging_payload_placeholders_the_stored_request_body_model_only_when_the_row_is_placeholdered(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
metadata: dict[str, object],
|
||||
expected_stored_model: str,
|
||||
redact_messages: bool,
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
|
||||
kwargs: Final = {
|
||||
"model": _RAW_MODEL_WITH_PROMPT,
|
||||
"call_type": "amoderation",
|
||||
"standard_callback_dynamic_params": {"turn_off_message_logging": redact_messages},
|
||||
"litellm_params": {
|
||||
"metadata": metadata,
|
||||
"proxy_server_request": {
|
||||
"url": "http://localhost:4000/v1/moderations",
|
||||
"body": {"input": "hi", "model": _RAW_MODEL_WITH_PROMPT},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
payload: Final = get_logging_payload(
|
||||
kwargs=kwargs,
|
||||
response_obj=ValueError("Invalid value for 'model'"),
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
stored_request_body: Final = json.loads(payload["proxy_server_request"])
|
||||
assert stored_request_body["model"] == expected_stored_model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_info", "expected_stored_model_group", "expected_stored_error_message"),
|
||||
[
|
||||
({}, "", f"Invalid value for 'model' = {UNKNOWN_MODEL_SPEND_LOG_MODEL}"),
|
||||
(
|
||||
{"model_info": {"id": "routed-deployment"}},
|
||||
_RAW_MODEL_WITH_PROMPT,
|
||||
f"Invalid value for 'model' = {_RAW_MODEL_WITH_PROMPT}",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_logging_payload_placeholders_the_metadata_copied_into_the_stored_request_body(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_info: dict[str, object],
|
||||
expected_stored_model_group: str,
|
||||
expected_stored_error_message: str,
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
|
||||
metadata: Final = {
|
||||
"user_api_key": "sk-test",
|
||||
"status": "failure",
|
||||
"model_group": _RAW_MODEL_WITH_PROMPT,
|
||||
"error_information": {
|
||||
"error_code": "400",
|
||||
"error_class": "BadRequestError",
|
||||
"llm_provider": "openai",
|
||||
"error_message": f"Invalid value for 'model' = {_RAW_MODEL_WITH_PROMPT}",
|
||||
"traceback": "",
|
||||
},
|
||||
**deployment_info,
|
||||
}
|
||||
kwargs: Final = {
|
||||
"model": _RAW_MODEL_WITH_PROMPT,
|
||||
"call_type": "amoderation",
|
||||
"litellm_params": {
|
||||
"metadata": metadata,
|
||||
"proxy_server_request": {
|
||||
"url": "http://localhost:4000/v1/moderations",
|
||||
"body": {"input": "hi", "model": _RAW_MODEL_WITH_PROMPT, "metadata": metadata},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
payload: Final = get_logging_payload(
|
||||
kwargs=kwargs,
|
||||
response_obj=ValueError("Invalid value for 'model'"),
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
stored_request_body: Final = json.loads(payload["proxy_server_request"])
|
||||
assert stored_request_body["metadata"]["model_group"] == expected_stored_model_group
|
||||
assert stored_request_body["metadata"]["error_information"]["error_message"] == expected_stored_error_message
|
||||
assert stored_request_body["metadata"]["user_api_key"] == "sk-test"
|
||||
assert ("medical records" in payload["proxy_server_request"]) == bool(deployment_info)
|
||||
|
||||
|
||||
_WHITESPACE_MODEL_GROUP: Final = "Broken GPT Mini"
|
||||
_WHITESPACE_MODEL_GROUP_ALIAS: Final = "Broken GPT Alias"
|
||||
_COOLDOWN_ERROR_MESSAGE: Final = (
|
||||
f"No deployments available for selected model. Passed model={_WHITESPACE_MODEL_GROUP}. Try again in 300 seconds"
|
||||
)
|
||||
|
||||
|
||||
def _router_serving_the_whitespace_model_group() -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": _WHITESPACE_MODEL_GROUP,
|
||||
"litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-test"},
|
||||
}
|
||||
],
|
||||
model_group_alias={_WHITESPACE_MODEL_GROUP_ALIAS: _WHITESPACE_MODEL_GROUP},
|
||||
)
|
||||
|
||||
|
||||
def _router_serving_only_a_wildcard() -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[{"model_name": "*", "litellm_params": {"model": "openai/*", "api_key": "sk-test"}}]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("requested_model", "llm_router", "expected_model", "expected_model_group", "expected_error_message"),
|
||||
[
|
||||
(
|
||||
_WHITESPACE_MODEL_GROUP,
|
||||
_router_serving_the_whitespace_model_group(),
|
||||
_WHITESPACE_MODEL_GROUP,
|
||||
_WHITESPACE_MODEL_GROUP,
|
||||
_COOLDOWN_ERROR_MESSAGE,
|
||||
),
|
||||
(
|
||||
_WHITESPACE_MODEL_GROUP_ALIAS,
|
||||
_router_serving_the_whitespace_model_group(),
|
||||
_WHITESPACE_MODEL_GROUP_ALIAS,
|
||||
_WHITESPACE_MODEL_GROUP_ALIAS,
|
||||
_COOLDOWN_ERROR_MESSAGE,
|
||||
),
|
||||
(
|
||||
_WHITESPACE_MODEL_GROUP,
|
||||
_router_serving_only_a_wildcard(),
|
||||
UNKNOWN_MODEL_SPEND_LOG_MODEL,
|
||||
"",
|
||||
_COOLDOWN_ERROR_MESSAGE.replace(_WHITESPACE_MODEL_GROUP, UNKNOWN_MODEL_SPEND_LOG_MODEL),
|
||||
),
|
||||
(
|
||||
_WHITESPACE_MODEL_GROUP,
|
||||
None,
|
||||
UNKNOWN_MODEL_SPEND_LOG_MODEL,
|
||||
"",
|
||||
_COOLDOWN_ERROR_MESSAGE.replace(_WHITESPACE_MODEL_GROUP, UNKNOWN_MODEL_SPEND_LOG_MODEL),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_logging_payload_keeps_a_configured_whitespace_model_group_that_failed_before_a_deployment_was_picked(
|
||||
requested_model: str,
|
||||
llm_router: litellm.Router | None,
|
||||
expected_model: str,
|
||||
expected_model_group: str,
|
||||
expected_error_message: str,
|
||||
):
|
||||
kwargs: Final = {
|
||||
"model": requested_model,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"call_type": "acompletion",
|
||||
"litellm_params": {
|
||||
"metadata": {
|
||||
"user_api_key": "sk-test",
|
||||
"model_group": requested_model,
|
||||
"status": "failure",
|
||||
"error_information": {"error_message": _COOLDOWN_ERROR_MESSAGE, "error_class": "RateLimitError"},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload: Final = get_logging_payload(
|
||||
kwargs=kwargs,
|
||||
response_obj=litellm.RateLimitError(message=_COOLDOWN_ERROR_MESSAGE, model=requested_model, llm_provider=""),
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
persisted_error: Final = json.loads(payload["metadata"])["error_information"]
|
||||
assert (payload["model"], payload["model_group"], persisted_error["error_message"]) == (
|
||||
expected_model,
|
||||
expected_model_group,
|
||||
expected_error_message,
|
||||
)
|
||||
|
||||
|
||||
def _openai_invalid_model_error_message(model: str) -> str:
|
||||
body: Final = {
|
||||
"error": {
|
||||
"message": f"Invalid value for 'model' = {model}. Please check the OpenAI documentation and try again.",
|
||||
"type": "invalid_request_error",
|
||||
"param": "model",
|
||||
"code": None,
|
||||
}
|
||||
}
|
||||
return f"Error code: 400 - {body}"
|
||||
|
||||
|
||||
def test_get_logging_payload_persists_no_raw_model_for_a_prompt_shaped_moderation_rejected_by_the_provider():
|
||||
provider_rejection: Final = litellm.BadRequestError(
|
||||
message=_openai_invalid_model_error_message(_RAW_MODEL_WITH_PROMPT),
|
||||
model=_RAW_MODEL_WITH_PROMPT,
|
||||
llm_provider="openai",
|
||||
)
|
||||
error_information: Final = _sanitize_error_information_for_spend_logs(
|
||||
StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=provider_rejection,
|
||||
traceback_str=(
|
||||
f"Traceback (most recent call last):\n ...\nlitellm.exceptions.BadRequestError: {provider_rejection}"
|
||||
),
|
||||
),
|
||||
original_exception=provider_rejection,
|
||||
)
|
||||
kwargs: Final = {
|
||||
"model": _RAW_MODEL_WITH_PROMPT,
|
||||
"input": "hi",
|
||||
"call_type": "",
|
||||
"litellm_params": {
|
||||
"metadata": {
|
||||
"user_api_key": "sk-test",
|
||||
"model_group": _RAW_MODEL_WITH_PROMPT,
|
||||
"status": "failure",
|
||||
"error_information": error_information,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload: Final = get_logging_payload(
|
||||
kwargs=kwargs,
|
||||
response_obj=provider_rejection,
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
persisted_error: Final = json.loads(payload["metadata"])["error_information"]
|
||||
scrubbed_message: Final = (
|
||||
f"litellm.BadRequestError: {_openai_invalid_model_error_message(UNKNOWN_MODEL_SPEND_LOG_MODEL)}"
|
||||
)
|
||||
assert (payload["model"], payload["model_group"]) == (UNKNOWN_MODEL_SPEND_LOG_MODEL, "")
|
||||
assert persisted_error["error_message"] == scrubbed_message
|
||||
assert persisted_error["traceback"].endswith(scrubbed_message)
|
||||
assert "medical records" not in payload["metadata"]
|
||||
|
||||
|
||||
_TRUNCATION_MARKER_TEXT: Final = (
|
||||
f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped 10 chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error_text", "expected"),
|
||||
[
|
||||
(f"Invalid model {_RAW_MODEL_WITH_PROMPT}", f"Invalid model {UNKNOWN_MODEL_SPEND_LOG_MODEL}"),
|
||||
(
|
||||
f"OpenAIException - {{'message': {_RAW_MODEL_WITH_PROMPT!r}}}",
|
||||
f"OpenAIException - {{'message': '{UNKNOWN_MODEL_SPEND_LOG_MODEL}'}}",
|
||||
),
|
||||
(
|
||||
(
|
||||
f"Invalid model {_RAW_MODEL_WITH_PROMPT[:20]}{_TRUNCATION_MARKER_TEXT}"
|
||||
f"{_RAW_MODEL_WITH_PROMPT[30:]} rejected"
|
||||
),
|
||||
(
|
||||
f"Invalid model {UNKNOWN_MODEL_SPEND_LOG_MODEL}{_TRUNCATION_MARKER_TEXT}"
|
||||
f"{UNKNOWN_MODEL_SPEND_LOG_MODEL} rejected"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_scrub_raw_model_from_error_information_covers_literal_escaped_and_truncation_split_spellings(
|
||||
error_text: str, expected: str
|
||||
):
|
||||
scrubbed: Final = _scrub_raw_model_from_error_information(
|
||||
cast(
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
{"error_message": error_text, "traceback": error_text, "error_class": "BadRequestError"},
|
||||
),
|
||||
_RAW_MODEL_WITH_PROMPT,
|
||||
)
|
||||
|
||||
assert scrubbed == {"error_message": expected, "traceback": expected, "error_class": "BadRequestError"}
|
||||
|
||||
|
||||
@patch("litellm.proxy.proxy_server.master_key", None)
|
||||
@patch("litellm.proxy.proxy_server.general_settings", {})
|
||||
def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_none():
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue