From f005afa1460385a218be8ef1fdfa49998bf93523 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 22 Aug 2026 22:59:02 -0700 Subject: [PATCH] test(exception-mapping): pin the status and error-shape table every provider maps to (#37807) `exception_type` decides the class and status a caller sees for every provider failure, across 190 raise sites, and the tests for it were written one incident at a time. Nothing said what a plain 401 from any given provider should be, so mutating a raise site went unnoticed: swapping the class at each of the 190 in turn, the mapped test file caught 15. Adds two tables asserted end to end through `exception_type`: 25 providers by the 9 upstream statuses, and the three error shapes the router branches on (a full context window, a content policy block, a timeout). The same 190 mutants now fail 97 of them. The tables record today's behavior, uneven where it is uneven. cloudflare, ollama and vllm map no status at all, so every failure reaches the caller as a 500. A full context window is recognised by 15 of the 25, and a content policy block by 11, which bounds where `context_window_fallbacks` and the content policy retry policy can fire. --- .../test_exception_mapping_utils.py | 285 ++++++++++++++++++ 1 file changed, 285 insertions(+) diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index cc0a52247a4..599ad016827 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -1,5 +1,6 @@ import httpx +import openai import pytest import litellm @@ -759,6 +760,290 @@ def test_azure_404_with_invalid_request_error_type_maps_to_not_found(): assert "Response with id 'resp_abc' not found." in excinfo.value.message +class _UpstreamHTTPError(Exception): + def __init__(self, status_code: int) -> None: + super().__init__("upstream failure") + self.message = "upstream failure" + self.status_code = status_code + self.request = httpx.Request("POST", "https://api.example.com/v1/chat/completions") + self.response = httpx.Response( + status_code=status_code, request=self.request, text="upstream failure" + ) + + +UPSTREAM_STATUS_CODES = (400, 401, 403, 404, 408, 422, 429, 500, 503) + +OPENAI_SHAPED = { + 400: (litellm.BadRequestError, 400), + 401: (litellm.AuthenticationError, 401), + 403: (litellm.APIError, 403), + 404: (litellm.NotFoundError, 404), + 408: (litellm.Timeout, 408), + 422: (litellm.BadRequestError, 422), + 429: (litellm.RateLimitError, 429), + 500: (litellm.InternalServerError, 500), + 503: (litellm.ServiceUnavailableError, 503), +} + +UPSTREAM_STATUS_DISCARDED = (litellm.APIConnectionError, 500) + +PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS = ("cloudflare", "ollama", "vllm") + +DEVIATIONS_FROM_THE_OPENAI_SHAPE = { + "anthropic": {403: UPSTREAM_STATUS_DISCARDED, 422: UPSTREAM_STATUS_DISCARDED}, + "azure": {500: (litellm.APIError, 500)}, + "bedrock": { + 403: UPSTREAM_STATUS_DISCARDED, + 500: (litellm.ServiceUnavailableError, 503), + }, + "cohere": { + 401: UPSTREAM_STATUS_DISCARDED, + 403: UPSTREAM_STATUS_DISCARDED, + 404: UPSTREAM_STATUS_DISCARDED, + 422: UPSTREAM_STATUS_DISCARDED, + 429: UPSTREAM_STATUS_DISCARDED, + 503: UPSTREAM_STATUS_DISCARDED, + }, + "databricks": { + 403: (litellm.AuthenticationError, 401), + 422: (litellm.BadRequestError, 400), + }, + "gemini": { + 403: (litellm.PermissionDeniedError, 403), + 422: UPSTREAM_STATUS_DISCARDED, + }, + "huggingface": { + 404: (litellm.APIError, 404), + 422: (litellm.APIError, 422), + 500: (litellm.APIError, 500), + }, + "nlp_cloud": { + 403: (litellm.AuthenticationError, 403), + 404: (litellm.APIError, 404), + 408: (litellm.APIError, 408), + 500: (litellm.APIError, 500), + 503: (litellm.APIError, 503), + }, + "openrouter": {500: (litellm.APIError, 500)}, + "replicate": { + 403: (litellm.APIError, 500), + 404: (litellm.APIError, 500), + 422: (litellm.UnprocessableEntityError, 422), + 500: (litellm.ServiceUnavailableError, 503), + 503: (litellm.APIError, 500), + }, + "sagemaker": { + 403: UPSTREAM_STATUS_DISCARDED, + 500: (litellm.ServiceUnavailableError, 503), + }, + "vertex_ai": { + 403: (litellm.PermissionDeniedError, 403), + 422: UPSTREAM_STATUS_DISCARDED, + }, + **{ + provider: dict.fromkeys(UPSTREAM_STATUS_CODES, UPSTREAM_STATUS_DISCARDED) + for provider in PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS + }, +} + +PROVIDERS_WITH_A_HANDLER = ( + "ai21", + "anthropic", + "azure", + "azure_ai", + "bedrock", + "cloudflare", + "cohere", + "databricks", + "deepseek", + "fireworks_ai", + "gemini", + "groq", + "huggingface", + "mistral", + "nlp_cloud", + "ollama", + "openai", + "openrouter", + "perplexity", + "replicate", + "sagemaker", + "together_ai", + "vertex_ai", + "vllm", + "xai", +) + + +def _expected_for(provider: str, status_code: int) -> tuple[type[Exception], int]: + return DEVIATIONS_FROM_THE_OPENAI_SHAPE.get(provider, {}).get( + status_code, OPENAI_SHAPED[status_code] + ) + + +@pytest.fixture +def quiet_exception_mapping(monkeypatch): + monkeypatch.setattr(litellm, "suppress_debug_info", True) + + +@pytest.mark.parametrize("status_code", UPSTREAM_STATUS_CODES) +@pytest.mark.parametrize("provider", PROVIDERS_WITH_A_HANDLER) +def test_an_upstream_status_maps_to_one_exception_per_provider( + provider, status_code, quiet_exception_mapping +): + expected_class, expected_status = _expected_for(provider, status_code) + + with pytest.raises(openai.APIError) as raised: + exception_type( + model="test-model", + original_exception=_UpstreamHTTPError(status_code=status_code), + custom_llm_provider=provider, + ) + + assert type(raised.value) is expected_class + assert raised.value.status_code == expected_status + + +@pytest.mark.parametrize("status_code", UPSTREAM_STATUS_CODES) +@pytest.mark.parametrize("provider", PROVIDERS_WITH_A_HANDLER) +def test_a_mapped_exception_keeps_the_provider_and_model_it_came_from( + provider, status_code, quiet_exception_mapping +): + with pytest.raises(openai.APIError) as raised: + exception_type( + model="test-model", + original_exception=_UpstreamHTTPError(status_code=status_code), + custom_llm_provider=provider, + ) + + assert raised.value.llm_provider == provider + assert raised.value.model == "test-model" + + +@pytest.mark.parametrize("provider", PROVIDERS_WITH_A_HANDLER) +def test_an_already_mapped_litellm_exception_passes_through_untouched( + provider, quiet_exception_mapping +): + already_mapped = litellm.RateLimitError( + message="already mapped", llm_provider=provider, model="test-model" + ) + + returned = exception_type( + model="test-model", + original_exception=already_mapped, + custom_llm_provider=provider, + ) + + assert returned is already_mapped + + +CONTEXT_WINDOW_MESSAGE = "This model's maximum context length is 4096 tokens." +CONTENT_POLICY_MESSAGE = ( + '{"error": {"type": "invalid_request_error", "code": "content_policy_violation"}}' +) +TIMEOUT_MESSAGE = "Request timed out." + +PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW = ( + "ai21", + "anthropic", + "azure", + "azure_ai", + "databricks", + "deepseek", + "fireworks_ai", + "gemini", + "groq", + "mistral", + "openai", + "perplexity", + "together_ai", + "vertex_ai", + "xai", +) + +PROVIDERS_THAT_RECOGNISE_A_CONTENT_POLICY_BLOCK = ( + "ai21", + "azure", + "azure_ai", + "deepseek", + "fireworks_ai", + "groq", + "mistral", + "openai", + "perplexity", + "together_ai", + "xai", +) + + +class _UpstreamErrorWithMessage(_UpstreamHTTPError): + def __init__(self, message: str, status_code: int) -> None: + super().__init__(status_code=status_code) + self.args = (message,) + self.message = message + self.response = httpx.Response( + status_code=status_code, request=self.request, text=message + ) + + +@pytest.mark.parametrize("provider", PROVIDERS_WITH_A_HANDLER) +def test_a_full_context_window_reaches_the_caller_as_the_router_needs_it( + provider, quiet_exception_mapping +): + if provider in PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS: + expected_class, expected_status = UPSTREAM_STATUS_DISCARDED + elif provider in PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW: + expected_class, expected_status = litellm.ContextWindowExceededError, 400 + else: + expected_class, expected_status = litellm.BadRequestError, 400 + + with pytest.raises(openai.APIError) as raised: + exception_type( + model="test-model", + original_exception=_UpstreamErrorWithMessage(CONTEXT_WINDOW_MESSAGE, 400), + custom_llm_provider=provider, + ) + + assert type(raised.value) is expected_class + assert raised.value.status_code == expected_status + + +@pytest.mark.parametrize("provider", PROVIDERS_WITH_A_HANDLER) +def test_a_content_policy_block_reaches_the_caller_as_the_router_needs_it( + provider, quiet_exception_mapping +): + if provider in PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS: + expected_class, expected_status = UPSTREAM_STATUS_DISCARDED + elif provider in PROVIDERS_THAT_RECOGNISE_A_CONTENT_POLICY_BLOCK: + expected_class, expected_status = litellm.ContentPolicyViolationError, 400 + else: + expected_class, expected_status = litellm.BadRequestError, 400 + + with pytest.raises(openai.APIError) as raised: + exception_type( + model="test-model", + original_exception=_UpstreamErrorWithMessage(CONTENT_POLICY_MESSAGE, 400), + custom_llm_provider=provider, + ) + + assert type(raised.value) is expected_class + assert raised.value.status_code == expected_status + + +@pytest.mark.parametrize("provider", PROVIDERS_WITH_A_HANDLER) +def test_a_timed_out_request_is_a_timeout_for_every_provider( + provider, quiet_exception_mapping +): + with pytest.raises(litellm.Timeout) as raised: + exception_type( + model="test-model", + original_exception=_UpstreamErrorWithMessage(TIMEOUT_MESSAGE, 408), + custom_llm_provider=provider, + ) + + assert raised.value.status_code == 408 + + def test_bedrock_mantle_400_maps_to_bad_request(): from litellm.llms.base_llm.chat.transformation import BaseLLMException