From 43fad507dec1bf729a35270dc433e2aadc235e9f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 19:17:32 -0700 Subject: [PATCH 1/3] fix(responses): map all documented in-stream error codes to real HTTP statuses --- litellm/responses/streaming_iterator.py | 45 ++++++++---- .../test_streaming_iterator_error_events.py | 69 +++++++++++++++++++ 2 files changed, 102 insertions(+), 12 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 357a7ecefe6..e85a758269f 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -7,7 +7,8 @@ import traceback import uuid from datetime import datetime from functools import lru_cache -from typing import Any, Dict, List, Literal, Optional +from types import MappingProxyType +from typing import Any, Dict, List, Literal, Mapping, Optional import httpx from openai._streaming import SSEDecoder @@ -48,13 +49,32 @@ def _log_background_task_failure(task: "asyncio.Task[Any]", *, task_name: str) - verbose_logger.error("%s failed: %s", task_name, exception) -_CLIENT_ERROR_CODES: frozenset[str] = frozenset( - ( - "invalid_request_error", - "context_length_exceeded", - "content_policy_violation", - "model_not_found", - ) +_ERROR_CODE_HTTP_STATUS: Mapping[str, int] = MappingProxyType( + { + "server_error": 500, + "rate_limit_exceeded": 429, + "insufficient_quota": 429, + "vector_store_timeout": 504, + "invalid_prompt": 400, + "invalid_image": 400, + "invalid_image_format": 400, + "invalid_base64_image": 400, + "invalid_image_url": 400, + "image_too_large": 400, + "image_too_small": 400, + "image_parse_error": 400, + "image_content_policy_violation": 400, + "invalid_image_mode": 400, + "image_file_too_large": 400, + "unsupported_image_media_type": 400, + "empty_image_file": 400, + "failed_to_download_image": 400, + "image_file_not_found": 400, + "invalid_request_error": 400, + "context_length_exceeded": 400, + "content_policy_violation": 400, + "model_not_found": 400, + } ) @@ -78,12 +98,13 @@ def _error_event_fields(error_obj: object) -> tuple[str, Optional[str], Optional def _status_code_for_error_fields(error_type: Optional[str], error_code: Optional[str]) -> int: - fields = tuple(field for field in (error_type, error_code) if field is not None) + fields = tuple(field for field in (error_code, error_type) if field is not None) if any(field.startswith("rate_limit") or field == "insufficient_quota" for field in fields): return 429 - if any(field in _CLIENT_ERROR_CODES for field in fields): - return 400 - return 500 + return next( + (_ERROR_CODE_HTTP_STATUS[field] for field in fields if field in _ERROR_CODE_HTTP_STATUS), + 500, + ) class BaseResponsesAPIStreamingIterator: diff --git a/tests/test_litellm/responses/test_streaming_iterator_error_events.py b/tests/test_litellm/responses/test_streaming_iterator_error_events.py index 1a2dcd0fcb7..3b87246ebdb 100644 --- a/tests/test_litellm/responses/test_streaming_iterator_error_events.py +++ b/tests/test_litellm/responses/test_streaming_iterator_error_events.py @@ -28,9 +28,11 @@ from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.streaming_iterator import ( + _ERROR_CODE_HTTP_STATUS, BaseResponsesAPIStreamingIterator, ResponsesAPIStreamingIterator, SyncResponsesAPIStreamingIterator, + _status_code_for_error_fields, ) from litellm.types.llms.openai import ( ErrorEvent, @@ -355,3 +357,70 @@ def test_sync_iterator_raises_mid_stream_fallback_on_rate_limit_error_event(): pass assert exc_info.value.status_code == 429 assert isinstance(exc_info.value.original_exception, litellm.APIError) + + +def test_every_openai_sdk_response_error_code_has_explicit_status_mapping(): + from typing import get_args + + from openai.types.responses.response_error import ResponseError + + sdk_codes = set(get_args(ResponseError.model_fields["code"].annotation)) + unmapped = sdk_codes - set(_ERROR_CODE_HTTP_STATUS) + assert unmapped == set(), ( + f"OpenAI SDK ResponseError codes missing from _ERROR_CODE_HTTP_STATUS: {sorted(unmapped)}; " + "classify each new code with an explicit HTTP status instead of letting it default to 500" + ) + + +@pytest.mark.parametrize( + "code,expected_status", + [ + ("server_error", 500), + ("rate_limit_exceeded", 429), + ("insufficient_quota", 429), + ("vector_store_timeout", 504), + ("invalid_prompt", 400), + ("invalid_image", 400), + ("invalid_image_format", 400), + ("invalid_base64_image", 400), + ("invalid_image_url", 400), + ("image_too_large", 400), + ("image_too_small", 400), + ("image_parse_error", 400), + ("image_content_policy_violation", 400), + ("invalid_image_mode", 400), + ("image_file_too_large", 400), + ("unsupported_image_media_type", 400), + ("empty_image_file", 400), + ("failed_to_download_image", 400), + ("image_file_not_found", 400), + ("totally_unknown_future_code", 500), + ], +) +def test_status_code_for_documented_response_error_codes(code: str, expected_status: int): + assert _status_code_for_error_fields(None, code) == expected_status + + +def test_specific_error_code_wins_over_generic_error_type(): + assert _status_code_for_error_fields("server_error", "invalid_image") == 400 + + +def test_maybe_raise_for_response_failed_event_maps_image_code_to_400(): + iterator = _make_iterator() + mock_response_obj = Mock() + mock_response_obj.error = {"code": "image_content_policy_violation", "message": "image rejected"} + chunk = Mock() + chunk.type = "response.failed" + chunk.response = mock_response_obj + with pytest.raises(litellm.APIError) as exc_info: + iterator._maybe_raise_for_error_event(chunk) + assert exc_info.value.status_code == 400 + assert not isinstance(exc_info.value, MidStreamFallbackError) + + +def test_maybe_raise_for_error_event_maps_vector_store_timeout_to_retriable_504(): + iterator = _make_iterator() + chunk = _make_error_chunk("server_error", "vector_store_timeout", "vector store timed out") + with pytest.raises(MidStreamFallbackError) as exc_info: + iterator._maybe_raise_for_error_event(chunk) + assert exc_info.value.status_code == 504 From 18b9e90d123d581c3de238012acf44ad421c598d Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 31 Jul 2026 04:04:04 +0000 Subject: [PATCH 2/3] fix(cost): bill the fast service tier at the priority rate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/llm_cost_calc/utils.py | 22 ++++--- litellm/types/utils.py | 1 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 63 +++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 85ed0665ebf..fbc06b76c72 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -2,7 +2,8 @@ ## Helper utilities for cost_per_token() from dataclasses import dataclass -from typing import Any, Literal, Optional, Tuple, TypedDict, cast +from types import MappingProxyType +from typing import Any, Literal, Mapping, Optional, Tuple, TypedDict, cast import litellm from litellm._logging import verbose_logger @@ -39,6 +40,14 @@ _VALID_DATA_RESIDENCIES = frozenset(r.value for r in DataResidency) # of being rebuilt for every model_info key on every call. _SERVICE_TIER_SUFFIXES: tuple[str, ...] = tuple(f"_{st.value}" for st in ServiceTier) +_SERVICE_TIER_TO_COST_KEY_SUFFIX: Mapping[str, str] = MappingProxyType( + { + ServiceTier.FLEX.value: ServiceTier.FLEX.value, + ServiceTier.PRIORITY.value: ServiceTier.PRIORITY.value, + ServiceTier.FAST.value: ServiceTier.PRIORITY.value, + } +) + def _get_token_detail_value(details: object, key: str) -> Optional[int]: if isinstance(details, dict): @@ -177,7 +186,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> st Args: base_key: The base cost key (e.g., "input_cost_per_token") - service_tier: The service tier ("flex", "priority", or None for standard) + service_tier: The service tier ("flex", "priority", "fast", or None for standard) Returns: str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token") @@ -185,12 +194,11 @@ def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> st if service_tier is None: return base_key - # Only use service tier specific keys for "flex" and "priority" - if service_tier.lower() in [ServiceTier.FLEX.value, ServiceTier.PRIORITY.value]: - return f"{base_key}_{service_tier.lower()}" + suffix = _SERVICE_TIER_TO_COST_KEY_SUFFIX.get(service_tier.lower()) + if suffix is None: + return base_key - # For any other service tier, use standard pricing - return base_key + return f"{base_key}_{suffix}" def _parse_above_token_threshold(key: str) -> float: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 056592bbf93..18991f53e6f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3847,6 +3847,7 @@ class ServiceTier(Enum): AUTO = "auto" FLEX = "flex" PRIORITY = "priority" + FAST = "fast" class DataResidency(Enum): diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 3454f160cfa..866f8f71484 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2447,3 +2447,66 @@ def test_generic_cost_per_token_gemini_35_flash_lite(): ) assert prompt_cost == pytest.approx(0.0003) assert completion_cost == pytest.approx(0.00125) + + +def test_fast_service_tier_bills_at_the_priority_rate(_local_model_cost_map): + """Regression: OpenAI's Fast mode replaced Priority Processing and costs 2x standard. + + Before the fix "fast" fell through to standard pricing, so a Fast mode request + was billed at half of what it actually costs.""" + from litellm.types.utils import Usage + + usage = Usage( + prompt_tokens=1_000, + completion_tokens=500, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200), + ) + + standard = generic_cost_per_token( + model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier=None + ) + priority = generic_cost_per_token( + model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="priority" + ) + fast = generic_cost_per_token( + model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast" + ) + + expected_prompt = 800 * 1e-05 + 200 * 1e-06 + expected_completion = 500 * 6e-05 + + assert fast == priority + assert fast[0] == pytest.approx(expected_prompt, rel=1e-9) + assert fast[1] == pytest.approx(expected_completion, rel=1e-9) + assert fast[0] == pytest.approx(standard[0] * 2, rel=1e-9) + assert fast[1] == pytest.approx(standard[1] * 2, rel=1e-9) + + +def test_fast_service_tier_is_case_insensitive(_local_model_cost_map): + from litellm.types.utils import Usage + + usage = Usage(prompt_tokens=1_000, completion_tokens=500) + + assert generic_cost_per_token( + model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="FAST" + ) == generic_cost_per_token( + model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast" + ) + + +def test_fast_service_tier_matches_priority_above_the_context_threshold(_local_model_cost_map): + """The above-threshold branch resolves its own cost keys, so the alias has to hold there too.""" + from litellm.types.utils import Usage + + usage = Usage(prompt_tokens=300_000, completion_tokens=1_000) + + fast = generic_cost_per_token( + model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast" + ) + priority = generic_cost_per_token( + model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="priority" + ) + + assert fast == priority + assert fast[0] == pytest.approx(300_000 * 1e-05, rel=1e-9) + assert fast[1] == pytest.approx(1_000 * 4.5e-05, rel=1e-9) From 9545b109b18d2e19e8812e514f7ddda486c2ca53 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:58:23 -0700 Subject: [PATCH 3/3] fix(responses): suppress LIT002 on error-code map frozen by MappingProxyType --- litellm/responses/streaming_iterator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index e85a758269f..dab666ff0d9 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -50,7 +50,7 @@ def _log_background_task_failure(task: "asyncio.Task[Any]", *, task_name: str) - _ERROR_CODE_HTTP_STATUS: Mapping[str, int] = MappingProxyType( - { + { # mutable-ok: immediately frozen by MappingProxyType "server_error": 500, "rate_limit_exceeded": 429, "insufficient_quota": 429,