From 392e80717200b36b083ce859c866af75d7721569 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:56:50 -0700 Subject: [PATCH] feat(logging): add normalized_error cluster key to error_information (#41715) * feat(logging): add normalized_error cluster key to error_information Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): stop classifying parameter length errors as context window errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): assert failure spend rows share normalized_error across provider wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): map agent model access denials and ignore non-string proxy error types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(logging): cover budget exceeded errors with custom wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): cluster router no-healthy and provider-budget wording correctly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): let the exception class win over router fallback wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): cluster peer closed connection errors as provider connection errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): cluster tag routing denials as 403_MODEL_ACCESS_DENIED Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: shivam Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng --- .../litellm_core_utils/error_normalization.py | 195 +++++++++++++++ litellm/litellm_core_utils/litellm_logging.py | 2 + litellm/types/utils.py | 1 + .../coverage_registry/quota_management.yaml | 1 + tests/e2e/models.py | 8 + .../spend_tracking/test_spend_tracking_e2e.py | 41 ++++ .../test_error_normalization.py | 231 ++++++++++++++++++ 7 files changed, 479 insertions(+) create mode 100644 litellm/litellm_core_utils/error_normalization.py create mode 100644 tests/test_litellm/litellm_core_utils/test_error_normalization.py diff --git a/litellm/litellm_core_utils/error_normalization.py b/litellm/litellm_core_utils/error_normalization.py new file mode 100644 index 00000000000..2a9ff748899 --- /dev/null +++ b/litellm/litellm_core_utils/error_normalization.py @@ -0,0 +1,195 @@ +""" +Map any exception litellm logs to one stable ``normalized_error`` code so dashboards can cluster +failures without parsing free-text messages that embed team names, token counts, model names, etc. +""" + +import re +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Protocol, runtime_checkable + +from litellm.exceptions import ( + APIConnectionError, + AuthenticationError, + BadGatewayError, + BadRequestError, + BlockedPiiEntityError, + BudgetExceededError, + ContentPolicyViolationError, + ContextWindowExceededError, + GuardrailRaisedException, + InternalServerError, + MidStreamFallbackError, + NotFoundError, + PermissionDeniedError, + RateLimitError, + RateLimitType, + ServiceUnavailableError, + Timeout, + UnprocessableEntityError, + UnsupportedParamsError, +) + +RATE_LIMIT_EXCEEDED: Final = "429_RATE_LIMIT_EXCEEDED" +BUDGET_EXCEEDED: Final = "429_BUDGET_EXCEEDED" +NO_HEALTHY_DEPLOYMENTS: Final = "429_NO_HEALTHY_DEPLOYMENTS" +AUTHENTICATION_FAILED: Final = "401_AUTHENTICATION_FAILED" +MODEL_ACCESS_DENIED: Final = "403_MODEL_ACCESS_DENIED" +PERMISSION_DENIED: Final = "403_PERMISSION_DENIED" +MISSING_REQUIRED_PARAMETER: Final = "400_MISSING_REQUIRED_PARAMETER" +INVALID_PARAMETER_VALUE: Final = "400_INVALID_PARAMETER_VALUE" +CONTEXT_WINDOW_EXCEEDED: Final = "400_CONTEXT_WINDOW_EXCEEDED" +CONTENT_POLICY_VIOLATION: Final = "400_CONTENT_POLICY_VIOLATION" +INVALID_REQUEST: Final = "400_INVALID_REQUEST" +RESOURCE_NOT_FOUND: Final = "404_RESOURCE_NOT_FOUND" +UPSTREAM_TIMEOUT: Final = "408_UPSTREAM_TIMEOUT" +PROVIDER_CONNECTION_ERROR: Final = "500_PROVIDER_CONNECTION_ERROR" +PROVIDER_OVERLOADED: Final = "503_PROVIDER_OVERLOADED" +PROVIDER_INTERNAL_ERROR: Final = "500_PROVIDER_INTERNAL_ERROR" +ROUTER_NO_FALLBACK: Final = "500_ROUTER_NO_FALLBACK" +ROUTER_FALLBACK_FAILURE: Final = "500_ROUTER_FALLBACK_FAILURE" +UPSTREAM_PASSTHROUGH: Final = "500_UPSTREAM_PASSTHROUGH" +UNSUPPORTED_OPERATION: Final = "500_UNSUPPORTED_OPERATION" +INTERNAL_STATE_ERROR: Final = "500_INTERNAL_STATE_ERROR" +UNCLASSIFIED: Final = "UNCLASSIFIED" + + +@runtime_checkable +class _HasProxyErrorType(Protocol): + type: str + + +_MESSAGE_PATTERNS: Final[tuple[tuple[re.Pattern[str], str], ...]] = ( + ( + re.compile(r"budget has been exceeded|max budget|exceeded.*budget|crossed budget", re.IGNORECASE), + BUDGET_EXCEEDED, + ), + (re.compile(r"no healthy deployments?|no deployments available", re.IGNORECASE), NO_HEALTHY_DEPLOYMENTS), + (re.compile(r"not allowed to access model due to tags configuration", re.IGNORECASE), MODEL_ACCESS_DENIED), + (re.compile(r"upstream passthrough request failed", re.IGNORECASE), UPSTREAM_PASSTHROUGH), + (re.compile(r"is not supported for provider|not implemented", re.IGNORECASE), UNSUPPORTED_OPERATION), + ( + re.compile(r"context window|context length|(prompt|input) is too long|tokens? ?> ?\d+ ?maximum", re.IGNORECASE), + CONTEXT_WINDOW_EXCEEDED, + ), + (re.compile(r"missing required parameter|field required", re.IGNORECASE), MISSING_REQUIRED_PARAMETER), + (re.compile(r"overloaded|unable to process your request", re.IGNORECASE), PROVIDER_OVERLOADED), + ( + re.compile( + r"connection error|APIConnectionError|TransferEncodingError|payload is not completed|connection reset" + r"|peer closed connection|incomplete chunked read", + re.IGNORECASE, + ), + PROVIDER_CONNECTION_ERROR, + ), + (re.compile(r"timed? ?out", re.IGNORECASE), UPSTREAM_TIMEOUT), +) + +_ROUTER_WRAPPER_PATTERNS: Final[tuple[tuple[re.Pattern[str], str], ...]] = ( + (re.compile(r"no fallback model group found", re.IGNORECASE), ROUTER_NO_FALLBACK), + (re.compile(r"error doing the fallback|MidStreamFallbackError", re.IGNORECASE), ROUTER_FALLBACK_FAILURE), +) + +_PROXY_ERROR_TYPE_MAP: Final[Mapping[str, str]] = MappingProxyType( + { + "budget_exceeded": BUDGET_EXCEEDED, + "auth_error": AUTHENTICATION_FAILED, + "expired_key": AUTHENTICATION_FAILED, + "token_not_found_in_db": AUTHENTICATION_FAILED, + "auth_provider_unavailable": AUTHENTICATION_FAILED, + "key_model_access_denied": MODEL_ACCESS_DENIED, + "team_model_access_denied": MODEL_ACCESS_DENIED, + "user_model_access_denied": MODEL_ACCESS_DENIED, + "org_model_access_denied": MODEL_ACCESS_DENIED, + "project_model_access_denied": MODEL_ACCESS_DENIED, + "agent_model_access_denied": MODEL_ACCESS_DENIED, + "key_vector_store_access_denied": PERMISSION_DENIED, + "team_vector_store_access_denied": PERMISSION_DENIED, + "org_vector_store_access_denied": PERMISSION_DENIED, + "tool_access_denied": PERMISSION_DENIED, + "team_member_permission_error": PERMISSION_DENIED, + "not_found_error": RESOURCE_NOT_FOUND, + } +) + +_STATUS_CODE_MAP: Final[Mapping[str, str]] = MappingProxyType( + { + "400": INVALID_REQUEST, + "401": AUTHENTICATION_FAILED, + "403": PERMISSION_DENIED, + "404": RESOURCE_NOT_FOUND, + "408": UPSTREAM_TIMEOUT, + "422": INVALID_PARAMETER_VALUE, + "429": RATE_LIMIT_EXCEEDED, + "500": PROVIDER_INTERNAL_ERROR, + "502": PROVIDER_INTERNAL_ERROR, + "503": PROVIDER_OVERLOADED, + "504": UPSTREAM_TIMEOUT, + } +) + +_INTERNAL_STATE_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = ( + TypeError, + KeyError, + AttributeError, + IndexError, + RuntimeError, + AssertionError, + ZeroDivisionError, +) + +_CLASS_CODE_TABLE: Final[tuple[tuple[tuple[type[BaseException], ...], str], ...]] = ( + ((AuthenticationError,), AUTHENTICATION_FAILED), + ((PermissionDeniedError,), PERMISSION_DENIED), + ((ContextWindowExceededError,), CONTEXT_WINDOW_EXCEEDED), + ((ContentPolicyViolationError, GuardrailRaisedException, BlockedPiiEntityError), CONTENT_POLICY_VIOLATION), + ((UnsupportedParamsError,), INVALID_PARAMETER_VALUE), + ((NotFoundError,), RESOURCE_NOT_FOUND), + ((Timeout,), UPSTREAM_TIMEOUT), + ((MidStreamFallbackError,), ROUTER_FALLBACK_FAILURE), + ((APIConnectionError,), PROVIDER_CONNECTION_ERROR), + ((ServiceUnavailableError,), PROVIDER_OVERLOADED), + ((InternalServerError, BadGatewayError), PROVIDER_INTERNAL_ERROR), + ((BadRequestError, UnprocessableEntityError), INVALID_REQUEST), + ((NotImplementedError,), UNSUPPORTED_OPERATION), +) + + +def _classify_by_message(message: str, patterns: tuple[tuple[re.Pattern[str], str], ...]) -> str | None: + return next((code for pattern, code in patterns if pattern.search(message)), None) + + +def _classify_by_class(exc: Exception) -> str | None: + if isinstance(exc, BudgetExceededError): + return BUDGET_EXCEEDED + if isinstance(exc, RateLimitError): + return BUDGET_EXCEEDED if exc.rate_limit_type == RateLimitType.BUDGET.value else RATE_LIMIT_EXCEEDED + for exc_types, code in _CLASS_CODE_TABLE: + if isinstance(exc, exc_types): + return code + if isinstance(exc, _INTERNAL_STATE_EXCEPTIONS): + return INTERNAL_STATE_ERROR + return None + + +def normalize_error(exc: Exception | None, status_code: str, message: str) -> str | None: + """ + Return a stable cluster key for ``exc``. ``status_code`` and ``message`` are the values + ``get_error_information`` already extracted, so the same exception always yields the same code. + """ + if exc is None: + return None + proxy_type: Final = exc.type if isinstance(exc, _HasProxyErrorType) else None + by_proxy_type: Final = _PROXY_ERROR_TYPE_MAP.get(proxy_type) if isinstance(proxy_type, str) else None + if by_proxy_type is not None: + return by_proxy_type + by_message: Final = _classify_by_message(message, _MESSAGE_PATTERNS) + if by_message is not None: + return by_message + by_class: Final = _classify_by_class(exc) + if by_class is not None: + return by_class + by_router_wrapper: Final = _classify_by_message(message, _ROUTER_WRAPPER_PATTERNS) + if by_router_wrapper is not None: + return by_router_wrapper + return _STATUS_CODE_MAP.get(status_code, UNCLASSIFIED) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 6e5a37a7226..8e28a0d543d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -76,6 +76,7 @@ from litellm.litellm_core_utils.core_helpers import ( reconstruct_model_name, set_response_cost_in_hidden_params, ) +from litellm.litellm_core_utils.error_normalization import normalize_error from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.internal_call_metadata import ( MODEL_ACCESS_GROUP_METADATA_KEY, @@ -6100,6 +6101,7 @@ class StandardLoggingPayloadSetup: error_budget_entity_id=budget_error.entity_id if budget_error else None, error_budget_limit=budget_error.max_budget if budget_error else None, error_budget_spend=budget_error.current_cost if budget_error else None, + normalized_error=normalize_error(original_exception, error_status, error_message), ) @staticmethod diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f2c8f0e7044..dfc98a9d89d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3256,6 +3256,7 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_budget_entity_id: str | None error_budget_limit: float | None error_budget_spend: float | None + normalized_error: ReadOnly[str | None] class GuardrailMode(TypedDict, total=False): diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 5ea48fdcd9d..5740a878608 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -51,6 +51,7 @@ - {id: quota_management.spend_tracking.end_user.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: end_user, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "user= attribution lands the end-user id on the spend row"} - {id: quota_management.spend_tracking.per_model.writes_own_rows, module: quota_management, tier: P2, behavior: spend_tracking, variant: per_model, assertions: [writes_own_rows], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Each model on a shared key gets its own spend row"} - {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"} +- {id: quota_management.spend_tracking.failure.writes_normalized_error, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_normalized_error], exercised_on: [chat_completions], source: "litellm_core_utils/error_normalization.py", rationale: "Failure rows carry a stable metadata.error_information.normalized_error key next to the unchanged error_message, so two upstream auth failures with different provider wording share one cluster key a dashboard can group by"} - {id: quota_management.spend_tracking.failure.attributes_provider, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [attributes_provider], exercised_on: [chat_completions], source: "proxy/utils.py", rationale: "A request rejected in pre_call_hook (rate limit, guardrail) still lands its single deployment's provider and model_id on the failure spend row"} - {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"} - {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index bc81d1015ba..bd7f5171172 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -937,10 +937,18 @@ class GuardrailRunRecord(BaseModel): guardrail_response: object | None = None +class SpendLogErrorInformation(BaseModel): + error_code: str | None = None + error_class: str | None = None + error_message: str | None = None + normalized_error: str | None = None + + class SpendLogMetadata(BaseModel): user_api_key_alias: str | None = None applied_guardrails: list[str] | None = None guardrail_information: list[GuardrailRunRecord] | None = None + error_information: SpendLogErrorInformation | None = None class SpendLogRow(BaseModel): diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index 6ca6f8cee55..286421e2e3f 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -499,6 +499,47 @@ def test_failure_call_writes_failure_status_row( assert (failure_row.spend or 0) == 0.0, "failed call must not be charged" +@pytest.mark.covers("quota_management.spend_tracking.failure.writes_normalized_error") +def test_failure_rows_share_normalized_error_across_provider_wording( + client: SpendClient, resources: ResourceManager, scoped_key: str +) -> None: + """Two upstream auth failures with different provider wording land as failure rows + whose metadata.error_information keeps each provider's own error_message and + carries the same stable normalized_error cluster key.""" + marker = unique_marker() + deployments: Final = ( + (f"e2e-norm-openai-{marker}", "openai/gpt-5.5"), + (f"e2e-norm-anthropic-{marker}", "anthropic/claude-haiku-4-5"), + ) + for name, provider_model in deployments: + model_id = client.proxy.create_model( + name, LiteLLMParamsBody(model=provider_model, api_key=f"sk-invalid-{marker}") + ) + resources.defer(lambda model_id=model_id: client.proxy.delete_model(model_id)) + result = client.chat(scoped_key, name, f"normalize failure {marker}", max_tokens=1) + assert not is_ok(result), f"{name}: invalid upstream key must fail the call, got {result}" + + rows = client.poll_logs_for_key( + scoped_key, + min_rows=2, + predicate=lambda rs: sum(1 for r in rs if r.status == "failure") >= 2, + ) + failure_rows = [r for r in rows if r.status == "failure"] + assert len(failure_rows) == 2, f"expected one failure row per deployment: {_summarize(rows)}" + + infos = [r.metadata.error_information if r.metadata else None for r in failure_rows] + assert all(info is not None for info in infos), ( + f"failure rows must carry metadata.error_information: {[r.model_dump() for r in failure_rows]}" + ) + messages = {info.error_message for info in infos if info is not None} + assert len(messages) == 2, f"provider wording must stay distinct in error_message: {messages}" + normalized = {info.normalized_error for info in infos if info is not None} + assert normalized == {"401_AUTHENTICATION_FAILED"}, ( + f"both auth failures must share one normalized_error cluster key; saw {normalized} " + f"for messages {messages}" + ) + + @pytest.mark.covers("quota_management.spend_tracking.failure.attributes_provider") def test_pre_call_rejection_row_attributes_provider_and_model_id( client: SpendClient, resources: ResourceManager diff --git a/tests/test_litellm/litellm_core_utils/test_error_normalization.py b/tests/test_litellm/litellm_core_utils/test_error_normalization.py new file mode 100644 index 00000000000..9d5469ddb4c --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_error_normalization.py @@ -0,0 +1,231 @@ +import httpx +import pytest + +import litellm +from litellm.exceptions import MidStreamFallbackError +from litellm.litellm_core_utils.error_normalization import normalize_error +from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.types.router import RouterErrors + +_RESPONSE = httpx.Response(status_code=500, request=httpx.Request("POST", "https://example.invalid")) + + +def _proxy_exc(message: str, error_type: str, code: int) -> ProxyException: + return ProxyException(message=message, type=error_type, param=None, code=code) + + +@pytest.mark.parametrize( + ("messages", "expected"), + [ + ( + ( + _proxy_exc("Rate limit exceeded for team X. Reset at 10:01", "rate_limit_error", 429), + _proxy_exc("Rate limit exceeded for team Y. Reset at 10:02", "rate_limit_error", 429), + ), + "429_RATE_LIMIT_EXCEEDED", + ), + ( + ( + litellm.BudgetExceededError(current_cost=3501.85, max_budget=3500), + _proxy_exc( + "User=abc, Current cost=1000.03, Max budget=1000", ProxyErrorTypes.budget_exceeded.value, 400 + ), + litellm.RateLimitError( + "budget", + llm_provider="openai", + model="gpt", + rate_limit_type=litellm.exceptions.RateLimitType.BUDGET, + ), + ), + "429_BUDGET_EXCEEDED", + ), + ( + ( + _proxy_exc("Token Expired", ProxyErrorTypes.expired_key.value, 401), + _proxy_exc("Malformed API Key", ProxyErrorTypes.auth_error.value, 401), + litellm.AuthenticationError("Signature verification failed", llm_provider="azure", model="gpt"), + ), + "401_AUTHENTICATION_FAILED", + ), + ( + ( + _proxy_exc("No team has access to gpt-5.5-mini", ProxyErrorTypes.team_model_access_denied.value, 401), + _proxy_exc("key not allowed to access claude", ProxyErrorTypes.key_model_access_denied.value, 401), + ValueError( + "Not allowed to access model due to tags configuration. Passed model=gpt-5.5 and tags=['team-a']" + ), + ), + "403_MODEL_ACCESS_DENIED", + ), + ( + ( + _proxy_exc("Missing required parameter: messages", ProxyErrorTypes.bad_request_error.value, 400), + litellm.BadRequestError("Missing required parameter: input", llm_provider="openai", model="gpt"), + ), + "400_MISSING_REQUIRED_PARAMETER", + ), + ( + ( + litellm.ContextWindowExceededError( + "1002823 tokens > 1000000 maximum", model="g", llm_provider="vertex" + ), + litellm.BadRequestError("Input is too long for requested model", llm_provider="anthropic", model="c"), + ), + "400_CONTEXT_WINDOW_EXCEEDED", + ), + ( + ( + litellm.NotFoundError("Response id xxx not found", llm_provider="openai", model="gpt"), + _proxy_exc("No vector store found with id abc", ProxyErrorTypes.not_found_error.value, 404), + ), + "404_RESOURCE_NOT_FOUND", + ), + ( + ( + litellm.APIConnectionError("Connection error", llm_provider="openai", model="gpt"), + litellm.InternalServerError("TransferEncodingError", llm_provider="openai", model="gpt"), + litellm.APIError(500, "Response payload is not completed", llm_provider="openai", model="gpt"), + httpx.RemoteProtocolError( + "peer closed connection without sending complete message body (incomplete chunked read)" + ), + ), + "500_PROVIDER_CONNECTION_ERROR", + ), + ( + ( + litellm.ServiceUnavailableError("server_is_overloaded", llm_provider="anthropic", model="c"), + litellm.InternalServerError( + "Bedrock is unable to process your request", llm_provider="bedrock", model="c" + ), + litellm.APIError(529, "Overloaded", llm_provider="anthropic", model="c"), + ), + "503_PROVIDER_OVERLOADED", + ), + ( + ( + litellm.InternalServerError( + "The server had an error while processing your request", llm_provider="openai", model="gpt" + ), + litellm.APIError(500, "server_error", llm_provider="openai", model="gpt"), + ), + "500_PROVIDER_INTERNAL_ERROR", + ), + ( + ( + _proxy_exc("No fallback model group found for gpt-5.6", "internal_server_error", 500), + _proxy_exc("No fallback model group found for claude-46-sonnet", "internal_server_error", 500), + ), + "500_ROUTER_NO_FALLBACK", + ), + ( + ( + _proxy_exc("Error doing the fallback: RateLimitError", "internal_server_error", 500), + MidStreamFallbackError( + "stream died", model="gpt", llm_provider="openai", original_exception=ValueError("boom") + ), + ), + "500_ROUTER_FALLBACK_FAILURE", + ), + ( + ( + TypeError("cannot pickle '_thread.RLock' object"), + RuntimeError("dictionary changed size during iteration"), + TypeError("'NoneType' object is not iterable"), + ), + "500_INTERNAL_STATE_ERROR", + ), + ( + ( + litellm.Timeout("Timeout on reading data from socket", model="gpt", llm_provider="openai"), + litellm.APIError(504, "Request timed out", llm_provider="openai", model="gpt"), + ), + "408_UPSTREAM_TIMEOUT", + ), + ( + ( + _proxy_exc("500: Upstream passthrough request failed", "internal_server_error", 500), + _proxy_exc("503: Upstream passthrough request failed", "internal_server_error", 503), + ), + "500_UPSTREAM_PASSTHROUGH", + ), + ( + ( + _proxy_exc("OCR is not supported for provider openai", "internal_server_error", 500), + NotImplementedError("rerank"), + ), + "500_UNSUPPORTED_OPERATION", + ), + ], +) +def test_variants_of_one_failure_share_a_normalized_error(messages: tuple[Exception, ...], expected: str) -> None: + normalized = {StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] for exc in messages} + assert normalized == {expected} + + +def test_router_no_healthy_deployment_wording_clusters_as_no_healthy_deployments() -> None: + for message in (RouterErrors.no_healthy_deployments.value, "No healthy deployments found."): + exc = litellm.BadRequestError(message, llm_provider="openai", model="gpt-4o") + assert normalize_error(exc, "400", message) == "429_NO_HEALTHY_DEPLOYMENTS", message + + +def test_provider_budget_routing_wording_clusters_as_budget_exceeded() -> None: + message = RouterErrors.no_deployments_with_provider_budget_routing.value + exc = litellm.BadRequestError(message, llm_provider="openai", model="gpt-4o") + assert normalize_error(exc, "400", message) == "429_BUDGET_EXCEEDED" + + +def test_router_fallback_wording_does_not_hide_the_wrapped_exception_class() -> None: + provider_message = "litellm.AuthenticationError: OpenAIException - Incorrect API key provided" + exc = litellm.AuthenticationError( + provider_message + "\nNo fallback model group found for lookup_groups=['x']", + llm_provider="openai", + model="gpt", + ) + assert normalize_error(exc, "401", str(exc)) == "401_AUTHENTICATION_FAILED" + wrapped = litellm.AuthenticationError( + "Error doing the fallback: " + provider_message, llm_provider="openai", model="gpt" + ) + assert normalize_error(wrapped, "401", str(wrapped)) == "401_AUTHENTICATION_FAILED" + + +def test_parameter_length_error_is_not_a_context_window_error() -> None: + exc = litellm.BadRequestError("string too long: 'user' max 64 chars", llm_provider="openai", model="gpt") + assert StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] == "400_INVALID_REQUEST" + + +def test_no_exception_has_no_normalized_error() -> None: + assert StandardLoggingPayloadSetup.get_error_information(None)["normalized_error"] is None + + +def test_unknown_exception_falls_back_to_status_then_unclassified() -> None: + assert normalize_error(Exception("x"), "429", "x") == "429_RATE_LIMIT_EXCEEDED" + assert normalize_error(Exception("x"), "", "x") == "UNCLASSIFIED" + + +def test_budget_exceeded_error_with_custom_wording_is_still_a_budget_error() -> None: + exc = litellm.BudgetExceededError(current_cost=2.0, max_budget=1.0, message="Spending cap reached for key") + assert StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] == "429_BUDGET_EXCEEDED" + + +def test_every_model_access_denied_proxy_type_shares_one_cluster() -> None: + access_denied_types = tuple(t for t in ProxyErrorTypes if t.value.endswith("_model_access_denied")) + assert len(access_denied_types) >= 6, access_denied_types + codes = {normalize_error(_proxy_exc("denied", t.value, 403), "403", "denied") for t in access_denied_types} + assert codes == {"403_MODEL_ACCESS_DENIED"}, codes + + +def test_non_string_type_attribute_falls_through_to_status() -> None: + class _OddType(Exception): + type = {"kind": "odd"} + + assert normalize_error(_OddType("odd"), "500", "odd") == "500_PROVIDER_INTERNAL_ERROR" + + +def test_normalized_error_never_embeds_dynamic_parts() -> None: + exc = _proxy_exc( + "No team has access to anthropic.claude-sonnet-4-5", ProxyErrorTypes.team_model_access_denied.value, 401 + ) + info = StandardLoggingPayloadSetup.get_error_information(exc) + assert info["error_message"] == "No team has access to anthropic.claude-sonnet-4-5" + assert "claude" not in (info["normalized_error"] or "")