diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 3d6c72de09f..b6b7c0585d7 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -367,6 +367,8 @@ def get_credentials_for_model( """ from fastapi import HTTPException + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + if llm_router is None: raise HTTPException( status_code=500, @@ -376,9 +378,8 @@ def get_credentials_for_model( credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id=model_id) if credentials is None: - raise HTTPException( - status_code=400, - detail={"error": f"Model '{model_id}' not found in model_list. Please check your config.yaml."}, + raise ProxyModelNotFoundError( + route=operation_context, model_name=model_id, retryable_with_model_read_through=False ) return credentials @@ -683,7 +684,7 @@ async def handle_model_based_routing( llm_router=llm_router, model_id=model_from_id, user_api_key_dict=user_api_key_dict, - operation_context=f"file operation (file created with model '{model_from_id}')", + operation_context="file operation (file created with model)", ) original_file_id: Final = get_original_file_id(file_id) return True, model_from_id, original_file_id, credentials diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index ae1c543de56..79a328f5199 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -95,6 +95,7 @@ from litellm.proxy.litellm_pre_call_utils import ( LiteLLMProxyRequestSetup, _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above ) +from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.utils import normalize_route_for_root_path from litellm.repositories.team_repository import TeamRepository from litellm.secret_managers.main import get_secret_str @@ -281,9 +282,8 @@ async def chat_completion_pass_through_endpoint( elif user_model is not None: # `litellm --model ` llm_response = asyncio.create_task(litellm.aadapter_completion(**data)) else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": "completion: Invalid model name passed in model=" + data.get("model", "")}, + raise ProxyModelNotFoundError( + route="completion", model_name=data.get("model", ""), retryable_with_model_read_through=False ) # Await the llm_response task diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 5a3a3f6c2f4..9756844b587 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -1,9 +1,11 @@ +import json import os import re import secrets from collections.abc import Mapping, Sequence from datetime import datetime, timezone from datetime import datetime as dt +from functools import reduce from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable @@ -380,11 +382,81 @@ def _model_group_provider(model_group: str, llm_router: "Router | None") -> str return next(iter(providers)) if len(providers) == 1 else None +def _is_configured_model_group(model_group: str, llm_router: "Router | None") -> bool: + if llm_router is None or not model_group: + return False + return llm_router.is_recognized_model(model_group) or model_group in llm_router.team_public_model_names + + def _looks_like_model_name(model: str) -> bool: candidate: Final = model.removeprefix(MCP_SPEND_LOG_MODEL_PREFIX) return len(candidate) <= MAX_SPEND_LOG_MODEL_NAME_LENGTH and not any(char.isspace() for char in candidate) +_TRUNCATION_MARKER: Final = re.compile( + rf"\.\.\. \({re.escape(LITELLM_TRUNCATED_PAYLOAD_FIELD)} skipped \d+ chars\. " + rf"{re.escape(LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE)}\) \.\.\." +) +_SCRUBBED_ERROR_TEXT_FIELDS: Final = frozenset(("error_message", "traceback")) + + +def _raw_model_spellings(raw_model: str) -> tuple[str, ...]: + return tuple(dict.fromkeys((raw_model, repr(raw_model)[1:-1], json.dumps(raw_model)[1:-1]))) + + +def _overlap_at_end(text: str, spelling: str) -> int: + lengths: Final = range(min(len(text), len(spelling) - 1), 0, -1) + return next((length for length in lengths if text.endswith(spelling[:length])), 0) + + +def _overlap_at_start(text: str, spelling: str) -> int: + lengths: Final = range(min(len(text), len(spelling) - 1), 0, -1) + return next((length for length in lengths if text.startswith(spelling[-length:])), 0) + + +def _scrub_raw_model_split_by_truncation(text: str, spellings: tuple[str, ...]) -> str: + marker: Final = _TRUNCATION_MARKER.search(text) + if marker is None: + return text + head: Final = text[: marker.start()] + tail: Final = text[marker.end() :] + head_cut: Final = max(_overlap_at_end(head, spelling) for spelling in spellings) + tail_cut: Final = max(_overlap_at_start(tail, spelling) for spelling in spellings) + return "".join( + ( + head[: len(head) - head_cut], + UNKNOWN_MODEL_SPEND_LOG_MODEL if head_cut else "", + marker.group(0), + UNKNOWN_MODEL_SPEND_LOG_MODEL if tail_cut else "", + tail[tail_cut:], + ) + ) + + +def _scrub_raw_model_from_error_text(text: str, spellings: tuple[str, ...]) -> str: + whole_occurrences_scrubbed: Final = reduce( + lambda scrubbed, spelling: scrubbed.replace(spelling, UNKNOWN_MODEL_SPEND_LOG_MODEL), spellings, text + ) + return _scrub_raw_model_split_by_truncation(whole_occurrences_scrubbed, spellings) + + +def _scrub_raw_model_from_error_information( + error_information: StandardLoggingPayloadErrorInformation | None, raw_model: str +) -> StandardLoggingPayloadErrorInformation | None: + if error_information is None or not raw_model: + return error_information + spellings: Final = _raw_model_spellings(raw_model) + return cast( + StandardLoggingPayloadErrorInformation, + { + key: _scrub_raw_model_from_error_text(value, spellings) + if key in _SCRUBBED_ERROR_TEXT_FIELDS and isinstance(value, str) + else value + for key, value in error_information.items() + }, + ) + + def get_logging_payload( kwargs: dict | None, response_obj: object, @@ -502,14 +574,29 @@ def get_logging_payload( ) failed_with_prompt_shaped_model: Final = ( _get_status_for_spend_log(metadata=metadata) == "failure" - and not _model_group + and not _model_id and not _looks_like_model_name(resolved_model) + and not _is_configured_model_group(_model_group, llm_router) ) model_name: Final = ( UNKNOWN_MODEL_SPEND_LOG_MODEL if rejected_as_unknown_model or failed_with_prompt_shaped_model or model_is_malformed else resolved_model ) + model_is_placeholdered: Final = model_name == UNKNOWN_MODEL_SPEND_LOG_MODEL + persisted_model_group: Final = ( + "" + if model_is_placeholdered and _model_group == raw_model and not _looks_like_model_name(raw_model) + else _model_group + ) + persisted_metadata: Final = ( + { + **metadata, + "error_information": _scrub_raw_model_from_error_information(metadata.get("error_information"), raw_model), + } + if model_is_placeholdered + else metadata + ) litellm_call_id: Final = cast( str | None, kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), @@ -517,7 +604,7 @@ def get_logging_payload( # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( - metadata, + persisted_metadata, applied_guardrails=( standard_logging_payload["metadata"].get("applied_guardrails", None) if standard_logging_payload is not None @@ -576,7 +663,7 @@ def get_logging_payload( litellm_call_id=litellm_call_id, router_metadata=_get_router_metadata_for_spend_log( metadata=metadata, - requested_model=_model_group, + requested_model=persisted_model_group, selected_model=model_name, selected_provider=custom_llm_provider, router_correlation_id=litellm_call_id, @@ -658,7 +745,7 @@ def get_logging_payload( request_tags=request_tags, end_user=end_user_id or "", api_base=_api_base, - model_group=_model_group, + model_group=persisted_model_group, model_id=_model_id, mcp_namespaced_tool_name=mcp_namespaced_tool_name, agent_id=agent_id, @@ -669,7 +756,13 @@ def get_logging_payload( ), response=_get_response_for_spend_logs_payload(payload=standard_logging_payload, kwargs=kwargs), proxy_server_request=_get_proxy_server_request_for_spend_logs_payload( - metadata=metadata, litellm_params=litellm_params, kwargs=kwargs + metadata=metadata, + litellm_params=( + _placeholder_stored_request_body(litellm_params, persisted_model_group, raw_model) + if model_is_placeholdered + else litellm_params + ), + kwargs=kwargs, ), session_id=_get_session_id_for_spend_log( kwargs=kwargs, @@ -975,7 +1068,7 @@ def _sanitize_request_body_for_spend_logs_payload( visited.add(obj_id) def _sanitize_value(value: object) -> object: - if isinstance(value, dict): + if isinstance(value, Mapping): return _sanitize_request_body_for_spend_logs_payload(value, visited, max_string_length_prompt_in_db) elif isinstance(value, list): return [_sanitize_value(item) for item in value] @@ -1329,9 +1422,65 @@ def _convert_mapping_to_json_serializable(obj: Mapping[str, object]) -> dict[str return dict(obj) +def _placeholder_stored_request_body_metadata( + request_body: Mapping[str, object], persisted_model_group: str, raw_model: str +) -> Mapping[str, object]: + body_metadata: Final = request_body.get("metadata") + if not isinstance(body_metadata, Mapping): + return request_body + error_information: Final = body_metadata.get("error_information") + placeholdered_fields: Final = MappingProxyType( + { + "model_group": persisted_model_group, + "error_information": _scrub_raw_model_from_error_information( + cast(StandardLoggingPayloadErrorInformation, error_information), raw_model + ) + if isinstance(error_information, Mapping) + else error_information, + } + ) + return MappingProxyType( + { + **request_body, + "metadata": MappingProxyType( + {key: placeholdered_fields.get(key, value) for key, value in body_metadata.items()} + ), + } + ) + + +def _placeholder_stored_request_body( + litellm_params: Mapping[str, object], persisted_model_group: str, raw_model: str +) -> Mapping[str, object]: + proxy_server_request: Final = litellm_params.get("proxy_server_request") + if not isinstance(proxy_server_request, Mapping): + return litellm_params + request_body: Final = proxy_server_request.get("body") + if not isinstance(request_body, Mapping): + return litellm_params + model_placeholdered: Final = ( + MappingProxyType({**request_body, "model": UNKNOWN_MODEL_SPEND_LOG_MODEL}) + if "model" in request_body + else request_body + ) + return MappingProxyType( + { + **litellm_params, + "proxy_server_request": MappingProxyType( + { + **proxy_server_request, + "body": _placeholder_stored_request_body_metadata( + model_placeholdered, persisted_model_group, raw_model + ), + } + ), + } + ) + + def _get_proxy_server_request_for_spend_logs_payload( metadata: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], kwargs: dict | None = None, ) -> str: """ diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py index 87cd2aaff1f..ef8af7bdbd3 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py @@ -1,4 +1,5 @@ from types import MappingProxyType +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -6,10 +7,31 @@ import pytest from litellm.proxy.openai_files_endpoints.common_utils import ( apply_unified_file_ids, + get_credentials_for_model, map_raw_file_ids_to_unified, ) +from litellm.proxy.route_llm_request import ProxyModelNotFoundError +from litellm.proxy.utils import handle_exception_on_proxy from litellm.types.utils import LiteLLMBatch +_RAW_MODEL_WITH_PROMPT: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" + + +def test_get_credentials_for_model_rejects_an_unknown_model_without_persisting_the_raw_model(): + llm_router: Final = MagicMock() + llm_router.get_deployment_credentials_with_provider.return_value = None + + with pytest.raises(ProxyModelNotFoundError) as raised: + get_credentials_for_model( + llm_router=llm_router, model_id=_RAW_MODEL_WITH_PROMPT, operation_context="file upload" + ) + + assert (raised.value.status_code, handle_exception_on_proxy(raised.value).code) == (400, "400") + assert _RAW_MODEL_WITH_PROMPT in raised.value.detail["error"] + assert raised.value.retryable_with_model_read_through is False + assert raised.value.spend_log_error_message.startswith("file upload: ") + assert "medical records" not in raised.value.spend_log_error_message + def _batch(input_file_id, output_file_id, error_file_id) -> LiteLLMBatch: return LiteLLMBatch( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index bf8ef920bdc..fb89e3a6973 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -7,6 +7,7 @@ from collections.abc import Callable from contextlib import ExitStack, contextmanager from io import BytesIO from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -38,6 +39,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) +from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, @@ -6443,6 +6445,46 @@ async def test_chat_completion_pass_through_endpoint_answers_an_openai_typed_err assert (raised.value.type, raised.value.param, raised.value.code) == ("invalid_request_error", None, "400") +@pytest.mark.asyncio +async def test_chat_completion_pass_through_endpoint_keeps_the_raw_model_out_of_the_spend_log_error( + monkeypatch: pytest.MonkeyPatch, +): + raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" + proxy_logging: Final = MagicMock() + proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + proxy_logging.post_call_failure_hook = AsyncMock() + + async def fake_add_litellm_data_to_request(**kwargs: object) -> object: + return kwargs["data"] + + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging) + monkeypatch.setattr("litellm.proxy.proxy_server.add_litellm_data_to_request", fake_add_litellm_data_to_request) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + monkeypatch.setattr("litellm.proxy.proxy_server.user_model", None) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + + request: Final = MagicMock(spec=Request) + request.body = AsyncMock( + return_value=json.dumps({"model": raw_model, "messages": [{"role": "user", "content": "hi"}]}).encode() + ) + + with pytest.raises(ProxyException) as raised: + await chat_completion_pass_through_endpoint( + fastapi_response=Response(), + request=request, + adapter_id="anthropic", + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + + logged_exception: Final = proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"] + assert isinstance(logged_exception, ProxyModelNotFoundError) + assert logged_exception.retryable_with_model_read_through is False + assert logged_exception.spend_log_error_message.startswith("completion: ") + assert "medical records" not in logged_exception.spend_log_error_message + assert (raised.value.type, raised.value.param, raised.value.code) == ("invalid_request_error", None, "400") + assert raw_model in logged_exception.detail["error"] + + @pytest.mark.asyncio async def test_chat_completion_pass_through_endpoint_failure_carries_the_callers_litellm_call_id( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 7512bf5ad9c..0004711954a 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -39,6 +39,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _sanitize_error_information_for_spend_logs, _sanitize_guardrail_information_for_spend_logs, _sanitize_request_body_for_spend_logs_payload, + _scrub_raw_model_from_error_information, get_logging_payload, get_spend_logs_id, should_store_prompts_and_responses_in_spend_logs, @@ -50,6 +51,7 @@ from litellm.types.utils import ( StandardLoggingMetadata, StandardLoggingModelInformation, StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, ) @@ -1075,13 +1077,18 @@ def test_get_logging_payload_replaces_a_non_string_model_with_the_placeholder( [ ({"user_api_key": "sk-test"}, litellm.ModelResponse(id="chatcmpl-test", choices=[])), ( - {"user_api_key": "sk-test", "model_group": "team alias", "status": "failure"}, + { + "user_api_key": "sk-test", + "model_group": "team alias", + "model_info": {"id": "team-alias-deployment"}, + "status": "failure", + }, ValueError("provider timed out"), ), ], ) def test_get_logging_payload_keeps_a_whitespace_model_name_on_success_or_a_routed_failure( - metadata: dict[str, str], response_obj: litellm.ModelResponse | Exception + metadata: dict[str, object], response_obj: litellm.ModelResponse | Exception ): kwargs: Final = { "model": _RAW_MODEL_WITH_PROMPT, @@ -1100,6 +1107,301 @@ def test_get_logging_payload_keeps_a_whitespace_model_name_on_success_or_a_route assert payload["model"] == _RAW_MODEL_WITH_PROMPT +@pytest.mark.parametrize("redact_messages", [False, True]) +@pytest.mark.parametrize( + ("metadata", "expected_stored_model"), + [ + ({"user_api_key": "sk-test", "status": "failure"}, UNKNOWN_MODEL_SPEND_LOG_MODEL), + ( + {"user_api_key": "sk-test", "status": "failure", "model_info": {"id": "routed-deployment"}}, + _RAW_MODEL_WITH_PROMPT, + ), + ], +) +def test_get_logging_payload_placeholders_the_stored_request_body_model_only_when_the_row_is_placeholdered( + monkeypatch: pytest.MonkeyPatch, + metadata: dict[str, object], + expected_stored_model: str, + redact_messages: bool, +): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True}) + kwargs: Final = { + "model": _RAW_MODEL_WITH_PROMPT, + "call_type": "amoderation", + "standard_callback_dynamic_params": {"turn_off_message_logging": redact_messages}, + "litellm_params": { + "metadata": metadata, + "proxy_server_request": { + "url": "http://localhost:4000/v1/moderations", + "body": {"input": "hi", "model": _RAW_MODEL_WITH_PROMPT}, + }, + }, + } + + payload: Final = get_logging_payload( + kwargs=kwargs, + response_obj=ValueError("Invalid value for 'model'"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + stored_request_body: Final = json.loads(payload["proxy_server_request"]) + assert stored_request_body["model"] == expected_stored_model + + +@pytest.mark.parametrize( + ("deployment_info", "expected_stored_model_group", "expected_stored_error_message"), + [ + ({}, "", f"Invalid value for 'model' = {UNKNOWN_MODEL_SPEND_LOG_MODEL}"), + ( + {"model_info": {"id": "routed-deployment"}}, + _RAW_MODEL_WITH_PROMPT, + f"Invalid value for 'model' = {_RAW_MODEL_WITH_PROMPT}", + ), + ], +) +def test_get_logging_payload_placeholders_the_metadata_copied_into_the_stored_request_body( + monkeypatch: pytest.MonkeyPatch, + deployment_info: dict[str, object], + expected_stored_model_group: str, + expected_stored_error_message: str, +): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True}) + metadata: Final = { + "user_api_key": "sk-test", + "status": "failure", + "model_group": _RAW_MODEL_WITH_PROMPT, + "error_information": { + "error_code": "400", + "error_class": "BadRequestError", + "llm_provider": "openai", + "error_message": f"Invalid value for 'model' = {_RAW_MODEL_WITH_PROMPT}", + "traceback": "", + }, + **deployment_info, + } + kwargs: Final = { + "model": _RAW_MODEL_WITH_PROMPT, + "call_type": "amoderation", + "litellm_params": { + "metadata": metadata, + "proxy_server_request": { + "url": "http://localhost:4000/v1/moderations", + "body": {"input": "hi", "model": _RAW_MODEL_WITH_PROMPT, "metadata": metadata}, + }, + }, + } + + payload: Final = get_logging_payload( + kwargs=kwargs, + response_obj=ValueError("Invalid value for 'model'"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + stored_request_body: Final = json.loads(payload["proxy_server_request"]) + assert stored_request_body["metadata"]["model_group"] == expected_stored_model_group + assert stored_request_body["metadata"]["error_information"]["error_message"] == expected_stored_error_message + assert stored_request_body["metadata"]["user_api_key"] == "sk-test" + assert ("medical records" in payload["proxy_server_request"]) == bool(deployment_info) + + +_WHITESPACE_MODEL_GROUP: Final = "Broken GPT Mini" +_WHITESPACE_MODEL_GROUP_ALIAS: Final = "Broken GPT Alias" +_COOLDOWN_ERROR_MESSAGE: Final = ( + f"No deployments available for selected model. Passed model={_WHITESPACE_MODEL_GROUP}. Try again in 300 seconds" +) + + +def _router_serving_the_whitespace_model_group() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": _WHITESPACE_MODEL_GROUP, + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-test"}, + } + ], + model_group_alias={_WHITESPACE_MODEL_GROUP_ALIAS: _WHITESPACE_MODEL_GROUP}, + ) + + +def _router_serving_only_a_wildcard() -> litellm.Router: + return litellm.Router( + model_list=[{"model_name": "*", "litellm_params": {"model": "openai/*", "api_key": "sk-test"}}] + ) + + +@pytest.mark.parametrize( + ("requested_model", "llm_router", "expected_model", "expected_model_group", "expected_error_message"), + [ + ( + _WHITESPACE_MODEL_GROUP, + _router_serving_the_whitespace_model_group(), + _WHITESPACE_MODEL_GROUP, + _WHITESPACE_MODEL_GROUP, + _COOLDOWN_ERROR_MESSAGE, + ), + ( + _WHITESPACE_MODEL_GROUP_ALIAS, + _router_serving_the_whitespace_model_group(), + _WHITESPACE_MODEL_GROUP_ALIAS, + _WHITESPACE_MODEL_GROUP_ALIAS, + _COOLDOWN_ERROR_MESSAGE, + ), + ( + _WHITESPACE_MODEL_GROUP, + _router_serving_only_a_wildcard(), + UNKNOWN_MODEL_SPEND_LOG_MODEL, + "", + _COOLDOWN_ERROR_MESSAGE.replace(_WHITESPACE_MODEL_GROUP, UNKNOWN_MODEL_SPEND_LOG_MODEL), + ), + ( + _WHITESPACE_MODEL_GROUP, + None, + UNKNOWN_MODEL_SPEND_LOG_MODEL, + "", + _COOLDOWN_ERROR_MESSAGE.replace(_WHITESPACE_MODEL_GROUP, UNKNOWN_MODEL_SPEND_LOG_MODEL), + ), + ], +) +def test_get_logging_payload_keeps_a_configured_whitespace_model_group_that_failed_before_a_deployment_was_picked( + requested_model: str, + llm_router: litellm.Router | None, + expected_model: str, + expected_model_group: str, + expected_error_message: str, +): + kwargs: Final = { + "model": requested_model, + "messages": [{"role": "user", "content": "hi"}], + "call_type": "acompletion", + "litellm_params": { + "metadata": { + "user_api_key": "sk-test", + "model_group": requested_model, + "status": "failure", + "error_information": {"error_message": _COOLDOWN_ERROR_MESSAGE, "error_class": "RateLimitError"}, + } + }, + } + + payload: Final = get_logging_payload( + kwargs=kwargs, + response_obj=litellm.RateLimitError(message=_COOLDOWN_ERROR_MESSAGE, model=requested_model, llm_provider=""), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + llm_router=llm_router, + ) + + persisted_error: Final = json.loads(payload["metadata"])["error_information"] + assert (payload["model"], payload["model_group"], persisted_error["error_message"]) == ( + expected_model, + expected_model_group, + expected_error_message, + ) + + +def _openai_invalid_model_error_message(model: str) -> str: + body: Final = { + "error": { + "message": f"Invalid value for 'model' = {model}. Please check the OpenAI documentation and try again.", + "type": "invalid_request_error", + "param": "model", + "code": None, + } + } + return f"Error code: 400 - {body}" + + +def test_get_logging_payload_persists_no_raw_model_for_a_prompt_shaped_moderation_rejected_by_the_provider(): + provider_rejection: Final = litellm.BadRequestError( + message=_openai_invalid_model_error_message(_RAW_MODEL_WITH_PROMPT), + model=_RAW_MODEL_WITH_PROMPT, + llm_provider="openai", + ) + error_information: Final = _sanitize_error_information_for_spend_logs( + StandardLoggingPayloadSetup.get_error_information( + original_exception=provider_rejection, + traceback_str=( + f"Traceback (most recent call last):\n ...\nlitellm.exceptions.BadRequestError: {provider_rejection}" + ), + ), + original_exception=provider_rejection, + ) + kwargs: Final = { + "model": _RAW_MODEL_WITH_PROMPT, + "input": "hi", + "call_type": "", + "litellm_params": { + "metadata": { + "user_api_key": "sk-test", + "model_group": _RAW_MODEL_WITH_PROMPT, + "status": "failure", + "error_information": error_information, + } + }, + } + + payload: Final = get_logging_payload( + kwargs=kwargs, + response_obj=provider_rejection, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + persisted_error: Final = json.loads(payload["metadata"])["error_information"] + scrubbed_message: Final = ( + f"litellm.BadRequestError: {_openai_invalid_model_error_message(UNKNOWN_MODEL_SPEND_LOG_MODEL)}" + ) + assert (payload["model"], payload["model_group"]) == (UNKNOWN_MODEL_SPEND_LOG_MODEL, "") + assert persisted_error["error_message"] == scrubbed_message + assert persisted_error["traceback"].endswith(scrubbed_message) + assert "medical records" not in payload["metadata"] + + +_TRUNCATION_MARKER_TEXT: Final = ( + f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped 10 chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." +) + + +@pytest.mark.parametrize( + ("error_text", "expected"), + [ + (f"Invalid model {_RAW_MODEL_WITH_PROMPT}", f"Invalid model {UNKNOWN_MODEL_SPEND_LOG_MODEL}"), + ( + f"OpenAIException - {{'message': {_RAW_MODEL_WITH_PROMPT!r}}}", + f"OpenAIException - {{'message': '{UNKNOWN_MODEL_SPEND_LOG_MODEL}'}}", + ), + ( + ( + f"Invalid model {_RAW_MODEL_WITH_PROMPT[:20]}{_TRUNCATION_MARKER_TEXT}" + f"{_RAW_MODEL_WITH_PROMPT[30:]} rejected" + ), + ( + f"Invalid model {UNKNOWN_MODEL_SPEND_LOG_MODEL}{_TRUNCATION_MARKER_TEXT}" + f"{UNKNOWN_MODEL_SPEND_LOG_MODEL} rejected" + ), + ), + ], +) +def test_scrub_raw_model_from_error_information_covers_literal_escaped_and_truncation_split_spellings( + error_text: str, expected: str +): + scrubbed: Final = _scrub_raw_model_from_error_information( + cast( + StandardLoggingPayloadErrorInformation, + {"error_message": error_text, "traceback": error_text, "error_class": "BadRequestError"}, + ), + _RAW_MODEL_WITH_PROMPT, + ) + + assert scrubbed == {"error_message": expected, "traceback": expected, "error_class": "BadRequestError"} + + @patch("litellm.proxy.proxy_server.master_key", None) @patch("litellm.proxy.proxy_server.general_settings", {}) def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_none():