diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 38a907892b4..73d31745047 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -18,6 +18,7 @@ from typing import ( from litellm.batches.batch_utils import batch_cost_is_final from litellm.constants import MAX_FILE_LIST_LIMIT from litellm.proxy._types import ProxyException +from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.repositories.table_repositories import ( ManagedFileRepository, ManagedObjectRepository, @@ -372,9 +373,8 @@ def get_credentials_for_model( credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id=model_id) if credentials is None: - raise HTTPException( - status_code=400, - detail={"error": f"Model '{model_id}' not found in model_list. Please check your config.yaml."}, + raise ProxyModelNotFoundError( + route=operation_context, model_name=model_id, retryable_with_model_read_through=False ) return credentials @@ -610,7 +610,7 @@ def handle_model_based_routing( credentials = get_credentials_for_model( llm_router=llm_router, model_id=model_from_id, - operation_context=f"file operation (file created with model '{model_from_id}')", + operation_context="file operation (file created with model)", ) original_file_id: Final = get_original_file_id(file_id) return True, model_from_id, original_file_id, credentials diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index ae1c543de56..79a328f5199 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -95,6 +95,7 @@ from litellm.proxy.litellm_pre_call_utils import ( LiteLLMProxyRequestSetup, _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above ) +from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.utils import normalize_route_for_root_path from litellm.repositories.team_repository import TeamRepository from litellm.secret_managers.main import get_secret_str @@ -281,9 +282,8 @@ async def chat_completion_pass_through_endpoint( elif user_model is not None: # `litellm --model ` llm_response = asyncio.create_task(litellm.aadapter_completion(**data)) else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": "completion: Invalid model name passed in model=" + data.get("model", "")}, + raise ProxyModelNotFoundError( + route="completion", model_name=data.get("model", ""), retryable_with_model_read_through=False ) # Await the llm_response task diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 5a3a3f6c2f4..27a309eeb8f 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -1,9 +1,11 @@ +import json import os import re import secrets from collections.abc import Mapping, Sequence from datetime import datetime, timezone from datetime import datetime as dt +from functools import reduce from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable @@ -385,6 +387,70 @@ def _looks_like_model_name(model: str) -> bool: return len(candidate) <= MAX_SPEND_LOG_MODEL_NAME_LENGTH and not any(char.isspace() for char in candidate) +_TRUNCATION_MARKER: Final = re.compile( + rf"\.\.\. \({re.escape(LITELLM_TRUNCATED_PAYLOAD_FIELD)} skipped \d+ chars\. " + rf"{re.escape(LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE)}\) \.\.\." +) +_SCRUBBED_ERROR_TEXT_FIELDS: Final = frozenset(("error_message", "traceback")) + + +def _raw_model_spellings(raw_model: str) -> tuple[str, ...]: + return tuple(dict.fromkeys((raw_model, repr(raw_model)[1:-1], json.dumps(raw_model)[1:-1]))) + + +def _overlap_at_end(text: str, spelling: str) -> int: + lengths: Final = range(min(len(text), len(spelling) - 1), 0, -1) + return next((length for length in lengths if text.endswith(spelling[:length])), 0) + + +def _overlap_at_start(text: str, spelling: str) -> int: + lengths: Final = range(min(len(text), len(spelling) - 1), 0, -1) + return next((length for length in lengths if text.startswith(spelling[-length:])), 0) + + +def _scrub_raw_model_split_by_truncation(text: str, spellings: tuple[str, ...]) -> str: + marker: Final = _TRUNCATION_MARKER.search(text) + if marker is None: + return text + head: Final = text[: marker.start()] + tail: Final = text[marker.end() :] + head_cut: Final = max(_overlap_at_end(head, spelling) for spelling in spellings) + tail_cut: Final = max(_overlap_at_start(tail, spelling) for spelling in spellings) + return "".join( + ( + head[: len(head) - head_cut], + UNKNOWN_MODEL_SPEND_LOG_MODEL if head_cut else "", + marker.group(0), + UNKNOWN_MODEL_SPEND_LOG_MODEL if tail_cut else "", + tail[tail_cut:], + ) + ) + + +def _scrub_raw_model_from_error_text(text: str, spellings: tuple[str, ...]) -> str: + whole_occurrences_scrubbed: Final = reduce( + lambda scrubbed, spelling: scrubbed.replace(spelling, UNKNOWN_MODEL_SPEND_LOG_MODEL), spellings, text + ) + return _scrub_raw_model_split_by_truncation(whole_occurrences_scrubbed, spellings) + + +def _scrub_raw_model_from_error_information( + error_information: StandardLoggingPayloadErrorInformation | None, raw_model: str +) -> StandardLoggingPayloadErrorInformation | None: + if error_information is None or not raw_model: + return error_information + spellings: Final = _raw_model_spellings(raw_model) + return cast( + StandardLoggingPayloadErrorInformation, + { + key: _scrub_raw_model_from_error_text(value, spellings) + if key in _SCRUBBED_ERROR_TEXT_FIELDS and isinstance(value, str) + else value + for key, value in error_information.items() + }, + ) + + def get_logging_payload( kwargs: dict | None, response_obj: object, @@ -502,7 +568,7 @@ def get_logging_payload( ) failed_with_prompt_shaped_model: Final = ( _get_status_for_spend_log(metadata=metadata) == "failure" - and not _model_group + and not _model_id and not _looks_like_model_name(resolved_model) ) model_name: Final = ( @@ -510,6 +576,20 @@ def get_logging_payload( if rejected_as_unknown_model or failed_with_prompt_shaped_model or model_is_malformed else resolved_model ) + model_is_placeholdered: Final = model_name == UNKNOWN_MODEL_SPEND_LOG_MODEL + persisted_model_group: Final = ( + "" + if model_is_placeholdered and _model_group == raw_model and not _looks_like_model_name(raw_model) + else _model_group + ) + persisted_metadata: Final = ( + { + **metadata, + "error_information": _scrub_raw_model_from_error_information(metadata.get("error_information"), raw_model), + } + if model_is_placeholdered + else metadata + ) litellm_call_id: Final = cast( str | None, kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), @@ -517,7 +597,7 @@ def get_logging_payload( # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( - metadata, + persisted_metadata, applied_guardrails=( standard_logging_payload["metadata"].get("applied_guardrails", None) if standard_logging_payload is not None @@ -576,7 +656,7 @@ def get_logging_payload( litellm_call_id=litellm_call_id, router_metadata=_get_router_metadata_for_spend_log( metadata=metadata, - requested_model=_model_group, + requested_model=persisted_model_group, selected_model=model_name, selected_provider=custom_llm_provider, router_correlation_id=litellm_call_id, @@ -658,7 +738,7 @@ def get_logging_payload( request_tags=request_tags, end_user=end_user_id or "", api_base=_api_base, - model_group=_model_group, + model_group=persisted_model_group, model_id=_model_id, mcp_namespaced_tool_name=mcp_namespaced_tool_name, agent_id=agent_id, diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py index 87cd2aaff1f..77cd1358606 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py @@ -6,10 +6,29 @@ import pytest from litellm.proxy.openai_files_endpoints.common_utils import ( apply_unified_file_ids, + get_credentials_for_model, map_raw_file_ids_to_unified, ) +from litellm.proxy.route_llm_request import ProxyModelNotFoundError +from litellm.proxy.utils import handle_exception_on_proxy from litellm.types.utils import LiteLLMBatch +_RAW_MODEL_WITH_PROMPT = "opus-4.6 Please summarize my medical records\nPatient has diabetes" + + +def test_get_credentials_for_model_rejects_an_unknown_model_without_persisting_the_raw_model(): + llm_router = MagicMock() + llm_router.get_deployment_credentials_with_provider.return_value = None + + with pytest.raises(ProxyModelNotFoundError) as raised: + get_credentials_for_model(llm_router=llm_router, model_id=_RAW_MODEL_WITH_PROMPT, operation_context="file upload") + + assert (raised.value.status_code, handle_exception_on_proxy(raised.value).code) == (400, "400") + assert _RAW_MODEL_WITH_PROMPT in raised.value.detail["error"] + assert raised.value.retryable_with_model_read_through is False + assert raised.value.spend_log_error_message.startswith("file upload: ") + assert "medical records" not in raised.value.spend_log_error_message + def _batch(input_file_id, output_file_id, error_file_id) -> LiteLLMBatch: return LiteLLMBatch( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index bf8ef920bdc..81911665b62 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -38,6 +38,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) +from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, @@ -6443,6 +6444,46 @@ async def test_chat_completion_pass_through_endpoint_answers_an_openai_typed_err assert (raised.value.type, raised.value.param, raised.value.code) == ("invalid_request_error", None, "400") +@pytest.mark.asyncio +async def test_chat_completion_pass_through_endpoint_keeps_the_raw_model_out_of_the_spend_log_error( + monkeypatch: pytest.MonkeyPatch, +): + raw_model = "opus-4.6 Please summarize my medical records\nPatient has diabetes" + proxy_logging = MagicMock() + proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + proxy_logging.post_call_failure_hook = AsyncMock() + + async def fake_add_litellm_data_to_request(**kwargs: object) -> object: + return kwargs["data"] + + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging) + monkeypatch.setattr("litellm.proxy.proxy_server.add_litellm_data_to_request", fake_add_litellm_data_to_request) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + monkeypatch.setattr("litellm.proxy.proxy_server.user_model", None) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + + request = MagicMock(spec=Request) + request.body = AsyncMock( + return_value=json.dumps({"model": raw_model, "messages": [{"role": "user", "content": "hi"}]}).encode() + ) + + with pytest.raises(ProxyException) as raised: + await chat_completion_pass_through_endpoint( + fastapi_response=Response(), + request=request, + adapter_id="anthropic", + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + + logged_exception = proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"] + assert isinstance(logged_exception, ProxyModelNotFoundError) + assert logged_exception.retryable_with_model_read_through is False + assert logged_exception.spend_log_error_message.startswith("completion: ") + assert "medical records" not in logged_exception.spend_log_error_message + assert (raised.value.type, raised.value.param, raised.value.code) == ("invalid_request_error", None, "400") + assert raw_model in logged_exception.detail["error"] + + @pytest.mark.asyncio async def test_chat_completion_pass_through_endpoint_failure_carries_the_callers_litellm_call_id( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 7512bf5ad9c..cad1aebeb50 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -39,6 +39,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _sanitize_error_information_for_spend_logs, _sanitize_guardrail_information_for_spend_logs, _sanitize_request_body_for_spend_logs_payload, + _scrub_raw_model_from_error_information, get_logging_payload, get_spend_logs_id, should_store_prompts_and_responses_in_spend_logs, @@ -50,6 +51,7 @@ from litellm.types.utils import ( StandardLoggingMetadata, StandardLoggingModelInformation, StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, ) @@ -1075,13 +1077,18 @@ def test_get_logging_payload_replaces_a_non_string_model_with_the_placeholder( [ ({"user_api_key": "sk-test"}, litellm.ModelResponse(id="chatcmpl-test", choices=[])), ( - {"user_api_key": "sk-test", "model_group": "team alias", "status": "failure"}, + { + "user_api_key": "sk-test", + "model_group": "team alias", + "model_info": {"id": "team-alias-deployment"}, + "status": "failure", + }, ValueError("provider timed out"), ), ], ) def test_get_logging_payload_keeps_a_whitespace_model_name_on_success_or_a_routed_failure( - metadata: dict[str, str], response_obj: litellm.ModelResponse | Exception + metadata: dict[str, object], response_obj: litellm.ModelResponse | Exception ): kwargs: Final = { "model": _RAW_MODEL_WITH_PROMPT, @@ -1100,6 +1107,95 @@ def test_get_logging_payload_keeps_a_whitespace_model_name_on_success_or_a_route assert payload["model"] == _RAW_MODEL_WITH_PROMPT +def _openai_invalid_model_error_message(model: str) -> str: + body: Final = { + "error": { + "message": f"Invalid value for 'model' = {model}. Please check the OpenAI documentation and try again.", + "type": "invalid_request_error", + "param": "model", + "code": None, + } + } + return f"Error code: 400 - {body}" + + +def test_get_logging_payload_persists_no_raw_model_for_a_prompt_shaped_moderation_rejected_by_the_provider(): + provider_rejection: Final = litellm.BadRequestError( + message=_openai_invalid_model_error_message(_RAW_MODEL_WITH_PROMPT), + model=_RAW_MODEL_WITH_PROMPT, + llm_provider="openai", + ) + error_information: Final = _sanitize_error_information_for_spend_logs( + StandardLoggingPayloadSetup.get_error_information( + original_exception=provider_rejection, + traceback_str=f"Traceback (most recent call last):\n ...\nlitellm.exceptions.BadRequestError: {provider_rejection}", + ), + original_exception=provider_rejection, + ) + kwargs: Final = { + "model": _RAW_MODEL_WITH_PROMPT, + "input": "hi", + "call_type": "", + "litellm_params": { + "metadata": { + "user_api_key": "sk-test", + "model_group": _RAW_MODEL_WITH_PROMPT, + "status": "failure", + "error_information": error_information, + } + }, + } + + payload: Final = get_logging_payload( + kwargs=kwargs, + response_obj=provider_rejection, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + persisted_error: Final = json.loads(payload["metadata"])["error_information"] + scrubbed_message: Final = ( + f"litellm.BadRequestError: {_openai_invalid_model_error_message(UNKNOWN_MODEL_SPEND_LOG_MODEL)}" + ) + assert (payload["model"], payload["model_group"]) == (UNKNOWN_MODEL_SPEND_LOG_MODEL, "") + assert persisted_error["error_message"] == scrubbed_message + assert persisted_error["traceback"].endswith(scrubbed_message) + assert "medical records" not in payload["metadata"] + + +_TRUNCATION_MARKER_TEXT: Final = ( + f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped 10 chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." +) + + +@pytest.mark.parametrize( + ("error_text", "expected"), + [ + (f"Invalid model {_RAW_MODEL_WITH_PROMPT}", f"Invalid model {UNKNOWN_MODEL_SPEND_LOG_MODEL}"), + ( + f"OpenAIException - {{'message': {_RAW_MODEL_WITH_PROMPT!r}}}", + f"OpenAIException - {{'message': '{UNKNOWN_MODEL_SPEND_LOG_MODEL}'}}", + ), + ( + f"Invalid model {_RAW_MODEL_WITH_PROMPT[:20]}{_TRUNCATION_MARKER_TEXT}{_RAW_MODEL_WITH_PROMPT[30:]} rejected", + f"Invalid model {UNKNOWN_MODEL_SPEND_LOG_MODEL}{_TRUNCATION_MARKER_TEXT}{UNKNOWN_MODEL_SPEND_LOG_MODEL} rejected", + ), + ], +) +def test_scrub_raw_model_from_error_information_covers_literal_escaped_and_truncation_split_spellings( + error_text: str, expected: str +): + scrubbed: Final = _scrub_raw_model_from_error_information( + cast( + StandardLoggingPayloadErrorInformation, + {"error_message": error_text, "traceback": error_text, "error_class": "BadRequestError"}, + ), + _RAW_MODEL_WITH_PROMPT, + ) + + assert scrubbed == {"error_message": expected, "traceback": expected, "error_class": "BadRequestError"} + + @patch("litellm.proxy.proxy_server.master_key", None) @patch("litellm.proxy.proxy_server.general_settings", {}) def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_none():