Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_/coverage-collector-skip-markers-d3f666

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -2,7 +2,8 @@
 ## Helper utilities for cost_per_token()
 
 from dataclasses import dataclass
-from typing import Any, Literal, Optional, Tuple, TypedDict, cast
+from types import MappingProxyType
+from typing import Any, Literal, Mapping, Optional, Tuple, TypedDict, cast
 
 import litellm
 from litellm._logging import verbose_logger
@@ -39,6 +40,14 @@ _VALID_DATA_RESIDENCIES = frozenset(r.value for r in DataResidency)
 # of being rebuilt for every model_info key on every call.
 _SERVICE_TIER_SUFFIXES: tuple[str, ...] = tuple(f"_{st.value}" for st in ServiceTier)
 
+_SERVICE_TIER_TO_COST_KEY_SUFFIX: Mapping[str, str] = MappingProxyType(
+    {
+        ServiceTier.FLEX.value: ServiceTier.FLEX.value,
+        ServiceTier.PRIORITY.value: ServiceTier.PRIORITY.value,
+        ServiceTier.FAST.value: ServiceTier.PRIORITY.value,
+    }
+)
+
 
 def _get_token_detail_value(details: object, key: str) -> Optional[int]:
     if isinstance(details, dict):
@@ -177,7 +186,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> st
 
     Args:
         base_key: The base cost key (e.g., "input_cost_per_token")
-        service_tier: The service tier ("flex", "priority", or None for standard)
+        service_tier: The service tier ("flex", "priority", "fast", or None for standard)
 
     Returns:
         str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")
@@ -185,12 +194,11 @@ def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> st
     if service_tier is None:
         return base_key
 
-    # Only use service tier specific keys for "flex" and "priority"
-    if service_tier.lower() in [ServiceTier.FLEX.value, ServiceTier.PRIORITY.value]:
-        return f"{base_key}_{service_tier.lower()}"
+    suffix = _SERVICE_TIER_TO_COST_KEY_SUFFIX.get(service_tier.lower())
+    if suffix is None:
+        return base_key
 
-    # For any other service tier, use standard pricing
-    return base_key
+    return f"{base_key}_{suffix}"
 
 
 def _parse_above_token_threshold(key: str) -> float:

litellm/responses/streaming_iterator.py

@@ -7,7 +7,8 @@ import traceback
 import uuid
 from datetime import datetime
 from functools import lru_cache
-from typing import Any, Dict, List, Literal, Optional
+from types import MappingProxyType
+from typing import Any, Dict, List, Literal, Mapping, Optional
 
 import httpx
 from openai._streaming import SSEDecoder
@@ -48,13 +49,32 @@ def _log_background_task_failure(task: "asyncio.Task[Any]", *, task_name: str) -
         verbose_logger.error("%s failed: %s", task_name, exception)
 
 
-_CLIENT_ERROR_CODES: frozenset[str] = frozenset(
-    (
-        "invalid_request_error",
-        "context_length_exceeded",
-        "content_policy_violation",
-        "model_not_found",
-    )
+_ERROR_CODE_HTTP_STATUS: Mapping[str, int] = MappingProxyType(
+    {  # mutable-ok: immediately frozen by MappingProxyType
+        "server_error": 500,
+        "rate_limit_exceeded": 429,
+        "insufficient_quota": 429,
+        "vector_store_timeout": 504,
+        "invalid_prompt": 400,
+        "invalid_image": 400,
+        "invalid_image_format": 400,
+        "invalid_base64_image": 400,
+        "invalid_image_url": 400,
+        "image_too_large": 400,
+        "image_too_small": 400,
+        "image_parse_error": 400,
+        "image_content_policy_violation": 400,
+        "invalid_image_mode": 400,
+        "image_file_too_large": 400,
+        "unsupported_image_media_type": 400,
+        "empty_image_file": 400,
+        "failed_to_download_image": 400,
+        "image_file_not_found": 400,
+        "invalid_request_error": 400,
+        "context_length_exceeded": 400,
+        "content_policy_violation": 400,
+        "model_not_found": 400,
+    }
 )
 
 
@@ -78,12 +98,13 @@ def _error_event_fields(error_obj: object) -> tuple[str, Optional[str], Optional
 
 
 def _status_code_for_error_fields(error_type: Optional[str], error_code: Optional[str]) -> int:
-    fields = tuple(field for field in (error_type, error_code) if field is not None)
+    fields = tuple(field for field in (error_code, error_type) if field is not None)
     if any(field.startswith("rate_limit") or field == "insufficient_quota" for field in fields):
         return 429
-    if any(field in _CLIENT_ERROR_CODES for field in fields):
-        return 400
-    return 500
+    return next(
+        (_ERROR_CODE_HTTP_STATUS[field] for field in fields if field in _ERROR_CODE_HTTP_STATUS),
+        500,
+    )
 
 
 class BaseResponsesAPIStreamingIterator:

litellm/types/utils.py

@@ -3847,6 +3847,7 @@ class ServiceTier(Enum):
     AUTO = "auto"
     FLEX = "flex"
     PRIORITY = "priority"
+    FAST = "fast"
 
 
 class DataResidency(Enum):

tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py

@@ -2447,3 +2447,66 @@ def test_generic_cost_per_token_gemini_35_flash_lite():
     )
     assert prompt_cost == pytest.approx(0.0003)
     assert completion_cost == pytest.approx(0.00125)
+
+
+def test_fast_service_tier_bills_at_the_priority_rate(_local_model_cost_map):
+    """Regression: OpenAI's Fast mode replaced Priority Processing and costs 2x standard.
+
+    Before the fix "fast" fell through to standard pricing, so a Fast mode request
+    was billed at half of what it actually costs."""
+    from litellm.types.utils import Usage
+
+    usage = Usage(
+        prompt_tokens=1_000,
+        completion_tokens=500,
+        prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200),
+    )
+
+    standard = generic_cost_per_token(
+        model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier=None
+    )
+    priority = generic_cost_per_token(
+        model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="priority"
+    )
+    fast = generic_cost_per_token(
+        model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast"
+    )
+
+    expected_prompt = 800 * 1e-05 + 200 * 1e-06
+    expected_completion = 500 * 6e-05
+
+    assert fast == priority
+    assert fast[0] == pytest.approx(expected_prompt, rel=1e-9)
+    assert fast[1] == pytest.approx(expected_completion, rel=1e-9)
+    assert fast[0] == pytest.approx(standard[0] * 2, rel=1e-9)
+    assert fast[1] == pytest.approx(standard[1] * 2, rel=1e-9)
+
+
+def test_fast_service_tier_is_case_insensitive(_local_model_cost_map):
+    from litellm.types.utils import Usage
+
+    usage = Usage(prompt_tokens=1_000, completion_tokens=500)
+
+    assert generic_cost_per_token(
+        model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="FAST"
+    ) == generic_cost_per_token(
+        model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast"
+    )
+
+
+def test_fast_service_tier_matches_priority_above_the_context_threshold(_local_model_cost_map):
+    """The above-threshold branch resolves its own cost keys, so the alias has to hold there too."""
+    from litellm.types.utils import Usage
+
+    usage = Usage(prompt_tokens=300_000, completion_tokens=1_000)
+
+    fast = generic_cost_per_token(
+        model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast"
+    )
+    priority = generic_cost_per_token(
+        model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="priority"
+    )
+
+    assert fast == priority
+    assert fast[0] == pytest.approx(300_000 * 1e-05, rel=1e-9)
+    assert fast[1] == pytest.approx(1_000 * 4.5e-05, rel=1e-9)

tests/test_litellm/responses/test_streaming_iterator_error_events.py

@@ -28,9 +28,11 @@ from litellm.exceptions import MidStreamFallbackError
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
 from litellm.responses.streaming_iterator import (
+    _ERROR_CODE_HTTP_STATUS,
     BaseResponsesAPIStreamingIterator,
     ResponsesAPIStreamingIterator,
     SyncResponsesAPIStreamingIterator,
+    _status_code_for_error_fields,
 )
 from litellm.types.llms.openai import (
     ErrorEvent,
@@ -355,3 +357,70 @@ def test_sync_iterator_raises_mid_stream_fallback_on_rate_limit_error_event():
             pass
     assert exc_info.value.status_code == 429
     assert isinstance(exc_info.value.original_exception, litellm.APIError)
+
+
+def test_every_openai_sdk_response_error_code_has_explicit_status_mapping():
+    from typing import get_args
+
+    from openai.types.responses.response_error import ResponseError
+
+    sdk_codes = set(get_args(ResponseError.model_fields["code"].annotation))
+    unmapped = sdk_codes - set(_ERROR_CODE_HTTP_STATUS)
+    assert unmapped == set(), (
+        f"OpenAI SDK ResponseError codes missing from _ERROR_CODE_HTTP_STATUS: {sorted(unmapped)}; "
+        "classify each new code with an explicit HTTP status instead of letting it default to 500"
+    )
+
+
+@pytest.mark.parametrize(
+    "code,expected_status",
+    [
+        ("server_error", 500),
+        ("rate_limit_exceeded", 429),
+        ("insufficient_quota", 429),
+        ("vector_store_timeout", 504),
+        ("invalid_prompt", 400),
+        ("invalid_image", 400),
+        ("invalid_image_format", 400),
+        ("invalid_base64_image", 400),
+        ("invalid_image_url", 400),
+        ("image_too_large", 400),
+        ("image_too_small", 400),
+        ("image_parse_error", 400),
+        ("image_content_policy_violation", 400),
+        ("invalid_image_mode", 400),
+        ("image_file_too_large", 400),
+        ("unsupported_image_media_type", 400),
+        ("empty_image_file", 400),
+        ("failed_to_download_image", 400),
+        ("image_file_not_found", 400),
+        ("totally_unknown_future_code", 500),
+    ],
+)
+def test_status_code_for_documented_response_error_codes(code: str, expected_status: int):
+    assert _status_code_for_error_fields(None, code) == expected_status
+
+
+def test_specific_error_code_wins_over_generic_error_type():
+    assert _status_code_for_error_fields("server_error", "invalid_image") == 400
+
+
+def test_maybe_raise_for_response_failed_event_maps_image_code_to_400():
+    iterator = _make_iterator()
+    mock_response_obj = Mock()
+    mock_response_obj.error = {"code": "image_content_policy_violation", "message": "image rejected"}
+    chunk = Mock()
+    chunk.type = "response.failed"
+    chunk.response = mock_response_obj
+    with pytest.raises(litellm.APIError) as exc_info:
+        iterator._maybe_raise_for_error_event(chunk)
+    assert exc_info.value.status_code == 400
+    assert not isinstance(exc_info.value, MidStreamFallbackError)
+
+
+def test_maybe_raise_for_error_event_maps_vector_store_timeout_to_retriable_504():
+    iterator = _make_iterator()
+    chunk = _make_error_chunk("server_error", "vector_store_timeout", "vector store timed out")
+    with pytest.raises(MidStreamFallbackError) as exc_info:
+        iterator._maybe_raise_for_error_event(chunk)
+    assert exc_info.value.status_code == 504