diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 9570b4d8cc8..68823d99519 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -227,6 +227,7 @@ jobs: tests/unit/llms/test_oss_decision.py tests/unit/llms/test_polling_url_origin_match.py tests/unit/llms/test_predibase_transformation.py + tests/unit/llms/test_subscription_accounts.py tests/unit/llms/a2a tests/unit/llms/aiml tests/unit/llms/aiohttp_openai diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index 2a7522cf34e..2d3ccf40175 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( "/cache_settings", "/coordination_redis/", "/cost_tracking", + "/subscription_accounts/", "/cost_optimization/", "/cost/", "/credentials", diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261028000000_subscription_accounts/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261028000000_subscription_accounts/migration.sql new file mode 100644 index 00000000000..e2788c85be9 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261028000000_subscription_accounts/migration.sql @@ -0,0 +1,21 @@ +SET LOCAL lock_timeout = '10s'; + +-- CreateTable +CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" ( + "subscription_account_id" TEXT NOT NULL, + "custom_llm_provider" TEXT NOT NULL, + "account_id" TEXT NOT NULL, + "label" TEXT, + "monthly_fee" DOUBLE PRECISION NOT NULL, + "currency" TEXT NOT NULL, + "billing_period_start" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT NOT NULL, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_by" TEXT NOT NULL, + + CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id") +); + +-- CreateIndex +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 2965174114b..179b741fe7c 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery { @@id([event_id, target, version]) @@index([target, state, next_attempt_at]) } + +model LiteLLM_SubscriptionAccountTable { + subscription_account_id String @id @default(uuid()) + custom_llm_provider String + account_id String + label String? + monthly_fee Float + currency String + billing_period_start String + created_at DateTime @default(now()) @map("created_at") + created_by String + updated_at DateTime @default(now()) @updatedAt @map("updated_at") + updated_by String + + @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") +} diff --git a/litellm/constants.py b/litellm/constants.py index e7c9c5dcbfb..c586a5f767a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -68,6 +68,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "retry_after", "fallbacks", "context_window_fallbacks", + "content_policy_fallbacks", "retry_policy", "model_group_retry_policy", "model_group_alias", @@ -420,6 +421,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float( "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0 ) BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0) +REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0 +REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 @@ -897,6 +900,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset( } ) +SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"}) A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"}) LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [ diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 45c82589d2d..d816bc661f3 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -116,12 +116,12 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict)) -def _served_provider_for_model(model: str) -> str | None: +def _served_provider_for_model(model: str, api_base: object = None) -> str | None: from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider try: - _, provider, _, _ = get_llm_provider(model=model) + _, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None) except BadRequestError: return None return provider @@ -536,7 +536,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): if model is None: return False hosted_flag: Final = _hosted_openai_dialect_flag( - model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider + model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base) ) if hosted_flag is not None: return hosted_flag @@ -559,8 +559,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) @staticmethod - def _resolve_provider(model: str) -> str | None: - return _served_provider_for_model(model) + def _resolve_provider(model: str, api_base: object = None) -> str | None: + return _served_provider_for_model(model, api_base) @staticmethod def count_request_cache_breakpoints( diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 4907bd4368d..320231430d2 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -361,6 +361,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_LensReview", "LiteLLM_LensIngestionKey", "LiteLLM_LensWorker", + "LiteLLM_SubscriptionAccountTable", "LiteLLM_LensSignalConfig", "LiteLLM_LensTraceSignal", ) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 28f45d2bb2c..2f81154ab83 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -982,25 +982,36 @@ def _usage_field(usage: object, name: str) -> object: return getattr(usage, name, None) -def _cost_number(value: object) -> float | None: - if isinstance(value, bool) or not isinstance(value, int | float): +def finite_cost(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value): return None return float(value) +def single_log_line(value: object) -> str: + return str(value).replace("\r", "").replace("\n", "") + + +def finite_reported_cost(value: object) -> float | None: + cost: Final = finite_cost(value) + if cost is None and isinstance(value, float): + verbose_logger.warning("Ignoring a non-finite provider-reported cost %s", value) + return cost + + def _reported_cost_value(value: object) -> float | None: """A reported cost is a number, or a breakdown whose total lives under ``total_cost``.""" raw: Final[object] = value if isinstance(value, Mapping): - return _cost_number(_usage_field(raw, "total_cost")) - return _cost_number(value) + return finite_reported_cost(_usage_field(raw, "total_cost")) + return finite_reported_cost(value) def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None: additional_headers: Final[object] = response.hidden_params.get("additional_headers") if not isinstance(additional_headers, Mapping): return None - return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER)) + return finite_reported_cost(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER)) def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None: diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 5a27f5446ad..33b6d4e1857 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string +from litellm.llms.bedrock.common_utils import bedrock_model_reaches_converse_by_default from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error from litellm.types.utils import LlmProviders @@ -938,13 +939,22 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = ( ) -def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response: +def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response: original_response: Final = getattr(original_exception, "response", None) if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None: - return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")) - if original_response.status_code == 400: + return httpx.Response( + status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/") + ) + if original_response.status_code == status_code: return original_response - return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request) + return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request) + + +def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> bool: + original_response: Final = getattr(original_exception, "response", None) + if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None: + return False + return original_response.request.url.path.endswith(("/converse", "/converse-stream")) def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: @@ -980,7 +990,7 @@ def _map_bedrock_exception( message=mantle_context_window_message, model=model, llm_provider=custom_llm_provider, - response=_bedrock_mantle_context_window_response(original_exception), + response=_bedrock_response_with_status(original_exception, 400), ) if ( "too many tokens" in error_str @@ -1003,6 +1013,18 @@ def _map_bedrock_exception( llm_provider="bedrock", response=getattr(original_exception, "response", None), ) + elif ( + "The provided model identifier is invalid" in error_str + and bool(model) + and _came_from_bedrock_converse(original_exception) + and bedrock_model_reaches_converse_by_default(model) + ): + raise NotFoundError( + message=f"BedrockException - {error_str}", + model=model, + llm_provider="bedrock", + response=_bedrock_response_with_status(original_exception, 404), + ) elif "Malformed input request" in error_str: raise BadRequestError( message=f"BedrockException - {error_str}", diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 8792fd0f58a..29f96d61976 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -85,6 +85,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "github_copilot_user_session", "fireworks_forward_user_id", "responses_api_version", + "allowed_openai_params", PROVIDER_AFFINITY_HEADER_KWARG_KEY, } ) diff --git a/litellm/litellm_core_utils/internal_call_metadata.py b/litellm/litellm_core_utils/internal_call_metadata.py index f3dbd612bee..453d4358b35 100644 --- a/litellm/litellm_core_utils/internal_call_metadata.py +++ b/litellm/litellm_core_utils/internal_call_metadata.py @@ -35,6 +35,12 @@ from litellm.types.utils import ( BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"}) +ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved" +OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved" +ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key" +OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key" +IO_TOKEN_RESERVATION_KEYS: Final = frozenset({ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, ITPM_CACHE_KEY, OTPM_CACHE_KEY}) + MODEL_ACCESS_GROUP_METADATA_KEY: Final = "user_api_key_matched_model_access_groups" """Where auth records the model access groups that authorized the request, for the spend writer. diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4dd33fa5565..cfe8c1739c6 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -76,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import ( is_classifier_call, ) from litellm.litellm_core_utils.core_helpers import ( + finite_cost, get_provider_response_headers_from_hidden_params, is_expected_client_error, native_messages_reported_cost, @@ -84,6 +85,7 @@ from litellm.litellm_core_utils.core_helpers import ( proxy_stamped_used_client_oauth_token, reconstruct_model_name, set_response_cost_in_hidden_params, + single_log_line, ) from litellm.litellm_core_utils.error_normalization import normalize_error from litellm.litellm_core_utils.get_litellm_params import get_litellm_params @@ -568,6 +570,11 @@ def _resolve_mantle_region_for_cost( _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"}) _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _REDACTED_RESULT: Final = TypeAdapter(object) +_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" + + +def _without_guardrail_information(metadata: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in metadata.items() if key != _GUARDRAIL_INFORMATION_KEY} def _provider_response_id(source: object) -> str | None: @@ -2088,7 +2095,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_model_name=litellm_model_name, router_model_id=router_model_id or hidden_params.get("model_id"), ) - return hidden_params["response_cost"] + return self._recorded_response_cost(hidden_params["response_cost"]) elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set router_model_id = hidden_params["model_id"] @@ -2188,7 +2195,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_model_name=litellm_model_name, router_model_id=router_model_id, ) - return total_response_cost + return self._recorded_response_cost(total_response_cost) except Exception as e: # error calculating cost debug_info = StandardLoggingModelCostFailureDebugInformation( error_str=str(e), @@ -2214,6 +2221,15 @@ class Logging(LiteLLMLoggingBaseClass): _response_cost_calculator = response_cost_calculator + def _recorded_response_cost(self, cost: float | None) -> float | None: + if cost is None or finite_cost(cost) is not None: + return cost + verbose_logger.warning( + "Recording a non-finite response_cost as 0 for litellm_call_id=%s", single_log_line(self.litellm_call_id) + ) + self.cost_breakdown = None + return 0.0 + def uncached_price(self, result: object) -> UncachedPrice: previous_cost_breakdown: Final = self.cost_breakdown previous_billed_token_rates: Final = self.billed_token_rates @@ -2566,6 +2582,23 @@ class Logging(LiteLLMLoggingBaseClass): attempt.model_call_details = dict(self.model_call_details) return attempt + def guardrail_scoped_copy(self) -> "Logging": + request_params: Final = _LITELLM_PARAMS.validate_python( + self.litellm_params or {} # pyright: ignore[reportUnknownMemberType] # Logging.litellm_params is untyped + ) + litellm_params: Final = { + **request_params, + **{ + key: _without_guardrail_information(_METADATA_MAPPING.validate_python(value)) + for key, value in request_params.items() + if key in _PER_CALL_LITELLM_PARAM_DICTS and isinstance(value, dict) + }, + } + scoped: Final = copy.copy(self) + scoped.litellm_params = litellm_params + scoped.model_call_details = {**self.model_call_details, "litellm_params": litellm_params} + return scoped + def dispatch_attempt_failure_handlers( self, exception: Exception, traceback_exception: str ) -> Coroutine[None, None, None]: @@ -7560,7 +7593,15 @@ def get_standard_logging_object_payload( ) custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj) raw_response_cost: Final = kwargs.get("response_cost") - llm_response_cost: Final[float] = raw_response_cost or 0.0 + numeric_response_cost: Final = raw_response_cost if isinstance(raw_response_cost, int | float) else None + priced_response_cost: Final = finite_cost(numeric_response_cost) + if numeric_response_cost is not None and priced_response_cost is None: + verbose_logger.warning( + "Logging a non-finite response_cost=%s as 0 for litellm_call_id=%s", + numeric_response_cost, + single_log_line(logging_obj.litellm_call_id), + ) + llm_response_cost: Final[float] = priced_response_cost or 0.0 guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information")) finalized_cost: Final = _finalized_cost_of(logging_obj) response_cost: Final[float] = ( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b3f89b33d1d..862e61cb631 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content( """ from litellm.litellm_core_utils.token_counter import get_image_type + media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None + # If we have a valid content type that's not generic, use it - if current_content_type and current_content_type not in [ + if media_type and media_type not in [ "binary/octet-stream", "application/octet-stream", ]: - return current_content_type + return media_type # Extension to MIME type mapping # Supports images, documents, and other common file types diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 834d26b4fe8..de896cde4dc 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -912,9 +912,9 @@ def create_anthropic_image_param( image_url_input: str | dict, format: str | None = None, is_bedrock_invoke: bool = False, -) -> AnthropicMessagesImageParam: +) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: """ - Create an AnthropicMessagesImageParam from an image URL input. + Create an Anthropic content block from an image URL input. Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding. """ @@ -933,14 +933,7 @@ def create_anthropic_image_param( if is_bedrock_invoke or image_url.startswith("http://"): base64_url: Final = convert_url_to_base64(url=image_url) image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) - return AnthropicMessagesImageParam( - type="image", - source=AnthropicContentParamSource( - type="base64", - media_type=image_chunk["media_type"], - data=image_chunk["data"], - ), - ) + return _anthropic_content_element_factory(image_chunk) else: # HTTPS URL - pass directly for regular Anthropic return AnthropicMessagesImageParam( @@ -953,14 +946,7 @@ def create_anthropic_image_param( else: # Convert to base64 for data URIs or other formats image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format) - return AnthropicMessagesImageParam( - type="image", - source=AnthropicContentParamSource( - type="base64", - media_type=image_chunk["media_type"], - data=image_chunk["data"], - ), - ) + return _anthropic_content_element_factory(image_chunk) # The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex @@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result( anthropic_content_element=_anthropic_image_param, original_content_element=content, ) - anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param)) + anthropic_content_list.append( + cast( + AnthropicMessagesImageParam | AnthropicMessagesDocumentParam, + _anthropic_image_param, + ) + ) elif content["type"] == "tool_reference": anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"])) elif content["type"] == "file": diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 484bacdc72c..4db22e9d5bf 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,5 +1,6 @@ import asyncio import json +import time import traceback from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence from dataclasses import dataclass @@ -15,13 +16,19 @@ from litellm._logging import redact_internal_details_from_client_message, verbos from litellm._uuid import uuid from litellm.constants import ( BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, + REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS, + REALTIME_LAZY_SETUP_DEADLINE_SECONDS, REALTIME_METERED_USAGE_KEY, REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError -from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend +from litellm.llms.base_llm.realtime.transformation import ( + BaseRealtimeConfig, + RealtimeBackend, + RealtimeSessionUpdateRejection, +) from litellm.types.llms.openai import ( OpenAIRealtimeEvents, OpenAIRealtimeOutputItemDone, @@ -125,6 +132,65 @@ class _LoggingWorker(Protocol): _DECODED_JSON: Final = TypeAdapter(object) +_SESSION_FIELDS: Final = TypeAdapter(dict[str, object]) +_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) +_SESSION_MERGE_MAX_DEPTH: Final = 8 + + +class _SessionUpdateFrame(TypedDict): + type: ReadOnly[str] + session: ReadOnly[Mapping[str, object]] + + +def _merge_session_fields( + base: Mapping[str, object], update: Mapping[str, object], depth: int = 0 +) -> Mapping[str, object]: + def merged_value(key: str) -> object: + if key not in update: + return base[key] + value: Final = update[key] + base_value: Final = base.get(key) + if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping): + return _merge_session_fields( + _SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1 + ) + return update[key] + + return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))}) + + +def _session_update_frame(session: Mapping[str, object]) -> str: + frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session} + return json.dumps(frame, default=_SESSION_FIELDS.validate_python) + + +_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection( + code="session_update_too_large", + message=( + "The session.update frames sent before the first conversation content add up to more than 10 MB, " + "so this session.update was not applied" + ), + param="session", +) + + +def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, object]: + return { + "type": "error", + "event_id": f"event_{uuid.uuid4()}", + "error": { + "type": "invalid_request_error", + "code": "audio_input_not_screenable", + "message": ( + "A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it " + "starts answering it, so audio input was not sent. Send text with conversation.item.create instead" + ), + "param": None, + "event_id": client_event_id if isinstance(client_event_id, str) else None, + }, + } + + def _decode_json_object(payload: str) -> Mapping[str, object]: """Decode a realtime frame into its top-level field mapping.""" return json.loads(payload) @@ -243,8 +309,17 @@ class RealTimeStreaming: event_normalizer: RealtimeEventNormalizer | None = None, usage_meter: RealtimeUsageMeter | None = None, logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER, + lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS, + lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS, + clock: Callable[[], float] = time.monotonic, ): self.websocket: _ClientWebSocket = websocket + self._clock: Final = clock + self._lazy_setup_deadline: Final = clock() + lazy_setup_deadline_seconds + self._lazy_setup_content_grace_seconds: Final = lazy_setup_content_grace_seconds + self._held_client_content: tuple[str, ...] = () + self._held_client_content_bytes: int = 0 + self._held_client_content_deadline: float | None = None self._replay_scope: Final = client_websocket_scope(websocket) self.backend_ws = backend_ws self.logging_obj = logging_obj @@ -300,6 +375,10 @@ class RealTimeStreaming: # Gemini Live rejects a follow-up BidiGenerateContentSetup once any # content (realtimeInput / clientContent / toolResponse) has been sent. self._content_sent_after_setup: bool = False + self._held_session_update: Mapping[str, object] | None = None + self._applied_session_fields: Mapping[str, object] | None = None + self._held_session_update_bytes: int = 0 + self._unscreened_audio_refused: bool = False # Whether this is a transcription-only session (session.type == "transcription", # e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and # their input_audio_transcription.completed usage drives duration-based cost. @@ -337,6 +416,17 @@ class RealTimeStreaming: ] ) _CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"]) + _CLIENT_CONTENT_TYPES = frozenset( + [ + "conversation.item.create", + "response.create", + "input_audio_buffer.append", + "input_audio_buffer.commit", + "input_audio_buffer.end", + ] + ) + _HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create", "input_audio_buffer.append"]) + _DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"]) _AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = { "pcm16": {"type": "audio/pcm", "rate": 24000}, "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000}, @@ -377,7 +467,7 @@ class RealTimeStreaming: message_obj: dict[str, Any] = cast(dict[str, Any], message) else: message_obj = cast(dict[str, Any], json.loads(message)) - self._collect_tool_calls_from_response_done(cast(dict, message_obj)) + self._collect_tool_calls_from_response_done(message_obj) if not self._should_store_message(message_obj): return try: @@ -502,7 +592,7 @@ class RealTimeStreaming: self.store_message(flush_event) self._capture_transcription_usage(flush_event) - def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None: + def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None: """Extract function_call items from response.done events for spend logging.""" try: if event_obj.get("type") != "response.done": @@ -607,6 +697,167 @@ class RealTimeStreaming: self._usage_meter.observe_client_event(sent_event) return True + def _builds_setup_lazily(self) -> bool: + if self.provider_config is None: + return False + return self.provider_config.builds_setup_from_session_update() + + def _holds_session_updates_until_first_content(self) -> bool: + return self.session_configuration_request is None and self._builds_setup_lazily() + + async def _rejected_setup_session_update(self, message: str) -> bool: + provider_config: Final = self.provider_config + if provider_config is None or not provider_config.builds_setup_from_session_update(): + return False + client_event: Final = _decode_json_object(message) + session: Final = client_event.get("session") + update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS + if self._leaves_applied_setup_unchanged(update): + await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS) + return True + rejection: Final = provider_config.setup_session_update_rejection( + self.model, update, setup_sent=self.session_configuration_request is not None + ) + if rejection is None: + return False + await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id")))) + return True + + def _held_session_update_message(self) -> str: + return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS) + + async def _hold_session_update(self, message: str) -> None: + client_event: Final = _decode_json_object(message) + if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES: + await self.websocket.send_text( + json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id"))) + ) + return + session: Final = client_event.get("session") + update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS + held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update) + self._held_session_update = held + self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8")) + await self._acknowledge_session_setup(held) + + def _setup_preview(self, session: Mapping[str, object]) -> str | None: + if self.provider_config is None: + return None + preview: Final = self.provider_config.transform_realtime_request( + _session_update_frame(session), self.model, None + ) + return next( + (frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None + ) + + def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool: + applied: Final = self._applied_session_fields + if self.session_configuration_request is None or applied is None: + return False + applied_setup: Final = self._setup_preview(applied) + merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update)) + return ( + applied_setup is not None + and merged_setup is not None + and _decode_json_object(applied_setup) == _decode_json_object(merged_setup) + ) + + async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None: + if self.provider_config is None: + return + setup: Final = self._setup_preview(session) + if setup is None: + return + acknowledged: Final = self.provider_config.transform_session_created_event( + self.model, str(self.logging_obj.litellm_trace_id), setup + ) + if acknowledged is None: + return + event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"} + event_str: Final = json.dumps(event) + self.store_message(event_str) + await self._send_event_to_client(event, event_str) + + async def _send_held_session_update_as_setup(self) -> None: + held: Final = self._held_session_update + if held is None: + return + message: Final = self._held_session_update_message() + self._held_session_update = None + self._held_session_update_bytes = 0 + if await self._send_to_backend(message): + self._applied_session_fields = held + await self._send_held_client_content() + + def _holds_client_content(self, msg_type: str | None, message: str) -> bool: + held_types: Final = ( + RealTimeStreaming._DEFERRED_SETUP_HELD_CONTENT_TYPES + if self._uses_deferred_backend_setup() + else RealTimeStreaming._HELD_CONTENT_TYPES + ) + return ( + msg_type in held_types + and self._held_session_update is not None + and self._holds_session_updates_until_first_content() + and not self._pending_messages_until_setup + and len(self._held_client_content) < RealTimeStreaming._MAX_BUFFERED_MESSAGES + and self._held_client_content_bytes + len(message.encode("utf-8")) <= RealTimeStreaming._MAX_BUFFERED_BYTES + ) + + def _hold_client_content(self, message: str) -> None: + if self._held_client_content_deadline is None: + self._held_client_content_deadline = self._clock() + self._lazy_setup_content_grace_seconds + self._held_client_content = (*self._held_client_content, message) + self._held_client_content_bytes += len(message.encode("utf-8")) + + async def _send_held_client_content(self) -> None: + held: Final = self._held_client_content + self._held_client_content = () + self._held_client_content_bytes = 0 + self._held_client_content_deadline = None + for message in held: + await self._send_to_backend(message) + + def _setup_deadline(self) -> float: + content_deadline: Final = self._held_client_content_deadline + return ( + self._lazy_setup_deadline if content_deadline is None else min(self._lazy_setup_deadline, content_deadline) + ) + + async def _send_setup_before_content(self, msg_type: str | None) -> None: + if self._holds_session_updates_until_first_content(): + await self._send_held_session_update_as_setup() + await self._send_default_setup_before_first_client_message(msg_type) + + async def _receive_client_message_before_setup_deadline(self) -> str | None: + if not self._holds_session_updates_until_first_content(): + return await self._receive_client_message() + receive: Final = asyncio.ensure_future(self._receive_client_message()) + try: + done, _ = await asyncio.wait((receive,), timeout=self._setup_deadline() - self._clock()) + if not done: + await self._send_setup_before_content(None) + return await receive + finally: + receive.cancel() + + async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None: + if self.session_configuration_request is not None or self.provider_config is None: + return + if msg_type == "session.update" or ( + self._uses_deferred_backend_setup() + and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup) + ): + return + default_setup: Final = self.provider_config.session_configuration_request(self.model) + if not default_setup: + return + setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup) + await self.backend_ws.send(setup) + self._cache_session_configuration_request(setup) + self._applied_session_fields = _NO_SESSION_FIELDS + await self._send_held_client_content() + def _enforce_transcription_session_model(self, message: str) -> str: """Force client transcription session updates to the authorized model. @@ -851,6 +1102,20 @@ class RealTimeStreaming: self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id} await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id})) + async def _send_blocked_turn_to_backend(self, pre_block_backend_message: str | None, item_id: str | None) -> None: + if pre_block_backend_message is None and self._is_transcription_session: + return + await self._send_setup_before_content("conversation.item.create") + # Deliver any caller-supplied backend message FIRST so that + # protocol contracts requiring a specific ordering (e.g. + # Gemini Live's mandatory ``toolResponse`` after a + # ``toolCall``) are honored before the guardrail's own + # clientContent / cancel messages are sent. + if pre_block_backend_message is not None: + await self._send_to_backend(pre_block_backend_message) + if not self._is_transcription_session: + await self._withdraw_blocked_turn(item_id) + async def _withdraw_blocked_turn(self, item_id: str | None) -> None: await self._send_guardrail_cancel() if item_id is not None and self._deletes_conversation_items: @@ -958,6 +1223,21 @@ class RealTimeStreaming: """Transcription-only sessions have no assistant turn to gate and reject realtime session updates.""" return not self._is_transcription_session and self._has_audio_transcription_guardrails() + def _refuses_unscreened_audio(self, msg_type: str | None) -> bool: + return ( + msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES + and self._builds_setup_lazily() + and self._should_disable_vad_auto_response() + ) + + async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None: + if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES: + return + self._unscreened_audio_refused = True + await self.websocket.send_text( + json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id"))) + ) + async def run_realtime_guardrails( self, transcript: str, @@ -1030,15 +1310,7 @@ class RealTimeStreaming: # Use realtime_violation_message if configured; fall back to guardrail error text. error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg - # Deliver any caller-supplied backend message FIRST so that - # protocol contracts requiring a specific ordering (e.g. - # Gemini Live's mandatory ``toolResponse`` after a - # ``toolCall``) are honored before the guardrail's own - # clientContent / cancel messages are sent. - if pre_block_backend_message is not None: - await self._send_to_backend(pre_block_backend_message) - if not self._is_transcription_session: - await self._withdraw_blocked_turn(item_id) + await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id) # Send the policy violation hint (shows as small gray status text in UI). await self.websocket.send_text( json.dumps( @@ -1136,6 +1408,9 @@ class RealTimeStreaming: break finally: self._flushing_pending_messages_until_setup = False + if self.session_created_sent_to_client and self._builds_setup_lazily(): + await self._maybe_send_guardrail_turn_detection_update() + continue if self.session_created_sent_to_client: # A synthetic session.created (with placeholder defaults) was # already forwarded to the client when we connected. The @@ -1295,7 +1570,11 @@ class RealTimeStreaming: return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket") def _backend_refused_session(self, close: BackendClose) -> bool: - return close.code != 1000 and not self._backend_sent_frames + return ( + close.code != 1000 + and not self._backend_sent_frames + and not self._holds_session_updates_until_first_content() + ) async def _log_session_end(self, transport_refusal: Exception | None) -> None: provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal() @@ -1488,7 +1767,7 @@ class RealTimeStreaming: client_event: _ClientEventFrame try: while True: - message = await self._receive_client_message() + message = await self._receive_client_message_before_setup_deadline() if message is None: return ClientLoopExit.CLIENT_DISCONNECTED @@ -1600,6 +1879,7 @@ class RealTimeStreaming: if ( msg_type == "session.update" and self.session_configuration_request is None + and self._held_session_update is None and not self._guardrail_turn_detection_update_sent and self._should_disable_vad_auto_response() and ( @@ -1663,12 +1943,25 @@ class RealTimeStreaming: except (json.JSONDecodeError, AttributeError): pass + if self._refuses_unscreened_audio(msg_type): + await self._refuse_unscreened_audio(msg_type, message) + continue + ## LOGGING # Log after any in-place modifications (GA remap, guardrail # turn_detection injection) so audit logs reflect what we # actually forward to the backend. self.store_input(message=message) + if self._holds_client_content(msg_type, message): + self._hold_client_content(message) + continue + if self._held_client_content and msg_type != "session.update": + await self._send_setup_before_content(msg_type) + + if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup(): + await self._send_setup_before_content(msg_type) + if self._should_buffer_client_message_until_setup(message): self._buffer_pending_message_until_setup(message) continue @@ -1697,6 +1990,15 @@ class RealTimeStreaming: # ``_maybe_send_guardrail_turn_detection_update`` would retry. if msg_type == "session.update" and self.provider_config is None: await self._send_session_gate_once() + if msg_type == "session.update" and await self._rejected_setup_session_update(message): + continue + if msg_type == "session.update" and self._holds_session_updates_until_first_content(): + await self._hold_session_update(message) + if self._pending_messages_until_setup: + await self._send_held_session_update_as_setup() + continue + if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES: + await self._send_setup_before_content(msg_type) try: sent = await self._send_to_backend(message) except RealtimeTranscriptionProtocolError as e: diff --git a/litellm/litellm_core_utils/reasoning_effort_utils.py b/litellm/litellm_core_utils/reasoning_effort_utils.py index acd20aeb37d..e9c94e32f8f 100644 --- a/litellm/litellm_core_utils/reasoning_effort_utils.py +++ b/litellm/litellm_core_utils/reasoning_effort_utils.py @@ -1,6 +1,12 @@ +import reprlib +from collections.abc import Collection, Mapping +from dataclasses import dataclass from typing import Final, Literal +from pydantic import TypeAdapter + import litellm +from litellm._logging import verbose_logger from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, @@ -11,6 +17,54 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"] +_DROP_WARNING: Final = "Dropping %s=%s for model=%r because %s" +_DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%s for model=%r (drop_params=True) because %s" +_NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level." +_NON_STRING_EFFORT_REASON: Final = ( + "it is not a string. Send reasoning_effort as a level name such as 'high' to set a reasoning level." +) +_REASONING_PARAMS: Final = ("reasoning", "reasoning_effort", "thinking") +OPAQUE_VALUE: Final = TypeAdapter(object) + + +def warn_dropped_reasoning( + param: str, value: object, model: str | None, reason: str, *, drop_params_gated: bool +) -> None: + verbose_logger.warning( + _DROP_PARAMS_DROP_WARNING if drop_params_gated else _DROP_WARNING, param, reprlib.repr(value), model, reason + ) + + +def warn_non_string_reasoning_effort(value: object, model: str | None) -> None: + effort: Final = as_str_mapping(value) + requested_level: Final = effort.get("effort") if effort is not None else value + if requested_level is None or isinstance(value, str): + return + warn_dropped_reasoning("reasoning_effort", value, model, _NON_STRING_EFFORT_REASON, drop_params_gated=False) + + +def warn_dropped_reasoning_params( + dropped: Collection[object], + request: Mapping[str, object], + model: str | None, + reason: str, + *, + drop_params_gated: bool, +) -> None: + for param in _REASONING_PARAMS: + if param in dropped and param in request: + warn_dropped_reasoning(param, request[param], model, reason, drop_params_gated=drop_params_gated) + + +@dataclass(frozen=True, slots=True) +class DroppedThinkingBudget: + budget_tokens: object + + def warn(self, model: str) -> None: + warn_dropped_reasoning( + "thinking.budget_tokens", self.budget_tokens, model, _NON_NUMERIC_BUDGET_REASON, drop_params_gated=True + ) + def reasoning_effort_from_thinking_budget( budget_tokens: float, @@ -29,7 +83,9 @@ def reasoning_effort_from_thinking_budget( return "minimal" -def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> OpenAIStyleReasoningEffort | None: +def _effort_or_dropped_budget( + thinking: object, *, drop_params: bool +) -> OpenAIStyleReasoningEffort | DroppedThinkingBudget | None: fields: Final = as_str_mapping(thinking) if fields is None or fields.get("type") != "enabled": return None @@ -37,10 +93,27 @@ def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> Op if isinstance(budget, (int, float)) and not isinstance(budget, bool): return reasoning_effort_from_thinking_budget(budget) if drop_params or litellm.drop_params is True: - return None + return DroppedThinkingBudget(budget) raise non_numeric_thinking_budget_error(budget) +def reasoning_effort_from_thinking( + thinking: object, *, model: str, drop_params: bool +) -> OpenAIStyleReasoningEffort | None: + effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params) + if not isinstance(effort, DroppedThinkingBudget): + return effort + effort.warn(model) + return None + + +def reasoning_effort_from_thinking_silently( + thinking: object, *, drop_params: bool +) -> OpenAIStyleReasoningEffort | None: + effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params) + return None if isinstance(effort, DroppedThinkingBudget) else effort + + def non_numeric_thinking_budget_error(budget: object) -> UnsupportedParamsError: return UnsupportedParamsError( message=( diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 24c401281ef..909e1222ae6 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -1,4 +1,5 @@ import json +import math from collections.abc import Callable from typing import Any, Final @@ -14,16 +15,23 @@ def strip_null_bytes(value: str) -> str: return value.replace("\x00", "") +def _finite_or_none(value: float | None) -> float | None: + return None if isinstance(value, float) and not math.isfinite(value) else value + + def safe_json_structure( data: object, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH, value_transform: Callable[[str | None, str], str] | None = None, key: str | None = None, + *, + non_finite_as_null: bool = False, ) -> object: """ Rebuild data out of JSON-native pieces while detecting circular references. If a circular reference is detected then a marker string is returned. - NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. + NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. With + non_finite_as_null, NaN and infinite floats become null, since jsonb rejects them. value_transform, when given, is applied to every string leaf (and to the str() fallback for non-serializable objects) with the mapping key the leaf @@ -43,7 +51,7 @@ def safe_json_structure( cleaned = obj.replace("\x00", "") if "\x00" in obj else obj return _transform(key, cleaned) if isinstance(obj, (int, float, bool, type(None))): - return obj + return _finite_or_none(obj) if non_finite_as_null else obj # Check for circular reference. if id(obj) in seen: return "CircularReference Detected" @@ -65,8 +73,9 @@ def safe_json_structure( seen.remove(id(obj)) return tuple_result elif isinstance(obj, set): - sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj] - set_result: Final = sorted(sortable_values) + members: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj] + present: Final = sorted(member for member in members if member is not None) + set_result: Final = present + [None] * members.count(None) seen.remove(id(obj)) return set_result elif isinstance(obj, BaseModel): @@ -88,6 +97,10 @@ def safe_dumps( data: object, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH, value_transform: Callable[[str | None, str], str] | None = None, + *, + non_finite_as_null: bool = False, ) -> str: """Serialize data to JSON text through safe_json_structure.""" - return json.dumps(safe_json_structure(data, max_depth, value_transform), default=str) + return json.dumps( + safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str + ) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 94351e72ab1..574f1103c1a 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -15,6 +15,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import ( PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS, chunk_usage, + finite_cost, provider_reported_usage_cost, stamp_litellm_bill_on_usage, ) @@ -1014,7 +1015,9 @@ class ChunkProcessor: prompt_tokens_details, cache_creation_token_details ) - if usage_chunk_dict["cost"] is not None: + if Usage.dropped_non_finite_cost(usage_chunk): + cost = None + elif usage_chunk_dict["cost"] is not None: cost = usage_chunk_dict["cost"] prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details) @@ -1444,8 +1447,8 @@ def _set_stream_builder_response_cost( def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance( - getattr(usage, "cost", None), (int, float) + if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and ( + finite_cost(getattr(usage, "cost", None)) is not None ): return computed_cost: Final = logging_obj.response_cost_calculator(result=response) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 5542249d2c7..098a404ea14 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -40,6 +40,7 @@ from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + CHOICE_LOGPROBS_OR_JSON, CacheCreationTokenDetails, CompletionTokensDetailsWrapper, Delta, @@ -1201,7 +1202,10 @@ class CustomStreamWrapper: choice_json.pop( "finish_reason", None ) # for mistral etc. which return a value in their last chunk (not-openai compatible). - choices.append(StreamingChoices(**choice_json)) + choice_json.pop("logprobs", None) + choices.append( + StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None)) + ) except Exception: choices.append(StreamingChoices()) setattr(model_response, "choices", choices) @@ -1605,7 +1609,7 @@ class CustomStreamWrapper: model_response.system_fingerprint = response_obj["original_chunk"].system_fingerprint self.system_fingerprint = response_obj["original_chunk"].system_fingerprint if response_obj["logprobs"] is not None: - model_response.choices[0].logprobs = response_obj["logprobs"] + model_response.choices[0].logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(response_obj["logprobs"]) if response_obj["usage"] is not None: if isinstance(response_obj["usage"], dict): @@ -2482,6 +2486,25 @@ class CustomStreamWrapper: def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None: self._earlier_rounds_spend = earlier + def partial_response(self) -> "ModelResponse | TextCompletionResponse | None": + partial: Final = ( + litellm.stream_chunk_builder( + chunks=self.chunks, + messages=self.messages if isinstance(self.messages, list) else None, + logging_obj=self.logging_obj, + count_prompt_tokens=self.count_prompt_tokens, + ) + if self.chunks + else None + ) + usage: Final[object] = getattr(partial, "usage", None) + if partial is None or not isinstance(usage, Usage): + return None + if self.model: + partial.model = self.model + backfill_missing_cache_usage_fields(usage) + return partial + def _record_partial_usage_for_failure(self) -> None: """ A stream that breaks mid-flight still billed the provider for the chunks @@ -2491,23 +2514,15 @@ class CustomStreamWrapper: moves it into terminal failure usage before opening the next attempt, so a recovered attempt logs its own usage. """ - if self.logging_obj is None or not self.chunks: + if self.logging_obj is None: return try: - partial_response: Final = litellm.stream_chunk_builder( - chunks=self.chunks, - messages=self.messages if isinstance(self.messages, list) else None, - logging_obj=self.logging_obj, - count_prompt_tokens=self.count_prompt_tokens, - ) + partial_response: Final = self.partial_response() if partial_response is None: return - usage: Final = cast(Usage | None, getattr(partial_response, "usage", None)) - if usage is None: + usage: Final[object] = getattr(partial_response, "usage", None) + if not isinstance(usage, Usage): return - if self.model: - partial_response.model = self.model - backfill_missing_cache_usage_fields(usage) partial_cost: Final = self.logging_obj.response_cost_calculator(result=partial_response) or 0.0 if self._earlier_rounds_spend is None: self.logging_obj.model_call_details["combined_usage_object"] = usage diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 865247afc63..d773e31c733 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import place_mid_conversation_system, split_leading_system_run, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort from litellm.llms.base_llm.base_utils import ( ResponseFormatSchema, as_json_object, @@ -1951,6 +1952,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if isinstance(effort_value, dict): effort_value = effort_value.get("effort") if not isinstance(effort_value, str): + warn_non_string_reasoning_effort(OPAQUE_VALUE.validate_python(value), model) continue mapped_thinking = AnthropicConfig._map_reasoning_effort( reasoning_effort=effort_value, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 82977f97854..6d1ac24af55 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -37,7 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) -from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error +from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.wif import ( aget_anthropic_wif_token, @@ -610,11 +610,11 @@ def _numeric_budget_tokens(budget_tokens: object) -> int | None: return None -def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | None: +def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | DroppedThinkingBudget: if (numeric := _numeric_budget_tokens(budget_tokens)) is not None: return numeric if drop_params or litellm.drop_params is True: - return None + return DroppedThinkingBudget(budget_tokens) raise non_numeric_thinking_budget_error(budget_tokens) @@ -1097,6 +1097,22 @@ class AnthropicModelInfo(BaseLLMModelInfo): custom_llm_provider: str, drop_params: bool, ) -> None: + dropped: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently( + model=model, + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + drop_params=drop_params, + ) + if dropped is not None: + dropped.warn(model) + + @staticmethod + def translate_legacy_thinking_for_adaptive_model_silently( + model: str, + optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers + custom_llm_provider: str, + drop_params: bool, + ) -> DroppedThinkingBudget | None: """Translate legacy ``thinking.type=enabled`` to adaptive for the adaptive-thinking models that reject it (4.7+ and the 5 families). Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the @@ -1105,12 +1121,12 @@ class AnthropicModelInfo(BaseLLMModelInfo): ``output_config.effort`` is never overridden. """ if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): - return + return None if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider): - return + return None thinking: Final = validated_object_mapping(optional_params.get("thinking")) if thinking is None or thinking.get("type") != "enabled": - return + return None budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params) display: Final = thinking.get("display") @@ -1119,8 +1135,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params ) optional_params["thinking"] = {**carried_keys, "type": "adaptive", **supported_display} - if budget_tokens is None: - return + if isinstance(budget_tokens, DroppedThinkingBudget): + return budget_tokens effort: Final = AnthropicModelInfo._legacy_budget_to_effort( model=model, budget_tokens=budget_tokens, custom_llm_provider=custom_llm_provider ) @@ -1129,6 +1145,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): "effort": effort, **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})), } + return None @staticmethod def _legacy_budget_to_effort(model: str, budget_tokens: int, custom_llm_provider: str) -> str: diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 704bec9056f..b3b84f721c0 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -647,12 +647,14 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data["output_config"] = extra_kwargs["output_config"] custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider") + api_base: Final = extra_kwargs.get("api_base") ( openai_request, tool_name_mapping, ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + api_base=api_base if isinstance(api_base, str) else None, drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")), ) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index ddda759384f..bc9649ed867 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -24,6 +24,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64 TOOL_NAME_HASH_LENGTH: Final = 8 TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55 PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"}) +PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"}) _COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue]) @@ -133,6 +134,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, + reasoning_effort_from_thinking_silently, ) from litellm.llms.anthropic.common_utils import ( eager_input_streaming_flag, @@ -255,7 +257,12 @@ class AnthropicAdapter: return result def translate_completion_input_params_with_tool_mapping( - self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True + self, + kwargs, + *, + custom_llm_provider: str | None = None, + api_base: str | None = None, + drop_params: bool = True, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -289,7 +296,9 @@ class AnthropicAdapter: ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, + api_base=api_base, drop_params=drop_params, + report_reasoning_drops=True, ) return translated_body, tool_name_mapping @@ -659,7 +668,9 @@ class LiteLLMAnthropicMessagesAdapter: def translate_anthropic_thinking_to_reasoning_effort( thinking: AnthropicThinkingParam, *, + model: str, drop_params: bool = True, + report_reasoning_drops: bool = False, ) -> str | None: """ Translate Anthropic's thinking parameter to OpenAI's reasoning_effort. @@ -678,7 +689,9 @@ class LiteLLMAnthropicMessagesAdapter: if thinking_type == "disabled": return "none" elif thinking_type == "enabled": - return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + if report_reasoning_drops: + return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) + return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params) elif thinking_type == "adaptive": # Adaptive thinking: effort is controlled by output_config.effort, # not budget_tokens. Return a default; caller should override with @@ -745,7 +758,7 @@ class LiteLLMAnthropicMessagesAdapter: return {"thinking": thinking} else: reasoning_effort: Final = LiteLLMAnthropicMessagesAdapter.translate_anthropic_thinking_to_reasoning_effort( - thinking + thinking, model=model ) if reasoning_effort: return { @@ -1001,7 +1014,9 @@ class LiteLLMAnthropicMessagesAdapter: ) @staticmethod - def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool: + def _supports_prompt_cache_key( + prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None + ) -> bool: if not model or not custom_llm_provider: return False if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: @@ -1009,7 +1024,17 @@ class LiteLLMAnthropicMessagesAdapter: supported_params: Final = litellm.get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider ) - return "prompt_cache_key" in (supported_params or ()) + if "prompt_cache_key" not in (supported_params or ()): + return False + if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: + return True + _, _, _, resolved_api_base = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider, api_base=api_base + ) + mistral_config: Final = litellm.MistralConfig() + return mistral_config.takes_prompt_cache_key_value( + prompt_cache_key + ) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base) @staticmethod def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool: @@ -1042,6 +1067,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs: ChatCompletionRequest, *, custom_llm_provider: str | None = None, + api_base: str | None = None, ) -> None: """Translate metadata fields from Anthropic request to OpenAI request.""" if "metadata" in anthropic_message_request: @@ -1050,7 +1076,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs["user"] = metadata["user_id"] prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"]) if prompt_cache_key is not None and self._supports_prompt_cache_key( - anthropic_message_request.get("model"), custom_llm_provider + prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base ): new_kwargs["prompt_cache_key"] = prompt_cache_key @@ -1127,6 +1153,7 @@ class LiteLLMAnthropicMessagesAdapter: *, custom_llm_provider: str | None = None, drop_params: bool = True, + report_reasoning_drops: bool = False, ) -> None: """Translate Anthropic thinking to either thinking or reasoning_effort. @@ -1187,7 +1214,10 @@ class LiteLLMAnthropicMessagesAdapter: return reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort( - cast(AnthropicThinkingParam, thinking), drop_params=drop_params + cast(AnthropicThinkingParam, thinking), + model=model, + drop_params=drop_params, + report_reasoning_drops=report_reasoning_drops, ) if not reasoning_effort: return @@ -1244,8 +1274,10 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request: AnthropicMessagesRequest, *, custom_llm_provider: str | None = None, + api_base: str | None = None, preserve_midturn_system: bool = False, drop_params: bool = True, + report_reasoning_drops: bool = False, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format. @@ -1282,6 +1314,7 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request=anthropic_message_request, new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, + api_base=api_base, ) ## CONVERT TOOL CHOICE self._translate_tool_choice_to_openai( @@ -1299,6 +1332,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, drop_params=drop_params, + report_reasoning_drops=report_reasoning_drops, ) ## CONVERT STOP_SEQUENCES self._translate_stop_sequences_to_openai( diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 6f97a019a96..1da7c2efdda 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.common_utils import ( + AnthropicModelInfo, strip_provider_specific_fields_from_anthropic_messages, tools_marked_for_a_tool_config_point, ) @@ -36,7 +37,6 @@ from litellm.router_utils.reasoning_effort_capability import ( keeps_effort, model_group_from_metadata, resolve_deployment_reasoning_effort, - with_adaptive_thinking_as_effort, ) from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata @@ -185,9 +185,11 @@ def _with_deployment_output_effort( ).value def translate(source: Mapping[str, object]) -> Mapping[str, object]: - return with_adaptive_thinking_as_effort( - source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + translated: Final = dict(source) + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params ) + return translated requested_output: Final = as_str_mapping(params.get("output_config")) caller_output: Final = None if requested_output is None else resolve(requested_output) diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index ef855eff180..c148a0defe8 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -21,6 +21,7 @@ from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards @@ -103,6 +104,7 @@ _MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): _workload_identity_eligible: ClassVar[bool] = True + _report_reasoning_drops: ClassVar[bool] = True @property def custom_llm_provider(self) -> str | None: @@ -695,12 +697,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ): anthropic_messages_optional_request_params.pop("output_format") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict + requested_reasoning_effort: Final = OPAQUE_VALUE.validate_python( + anthropic_messages_optional_request_params.get("reasoning_effort") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict + ) self._translate_reasoning_effort_to_anthropic( model=model, optional_params=anthropic_messages_optional_request_params, max_tokens=max_tokens, custom_llm_provider=self._resolved_provider, ) + if self._report_reasoning_drops: + warn_non_string_reasoning_effort(requested_reasoning_effort, model) AnthropicModelInfo.maybe_drop_disabled_thinking( model=model, @@ -708,12 +715,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): custom_llm_provider=self._resolved_provider, ) - AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + dropped_budget: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently( model=model, optional_params=anthropic_messages_optional_request_params, custom_llm_provider=self._resolved_provider, drop_params=getattr(litellm_params, "drop_params", None) is True, ) + if dropped_budget is not None and self._report_reasoning_drops: + dropped_budget.warn(model) self._translate_adaptive_effort_for_non_adaptive_model( model=model, diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 94c608f90e6..0630bef0311 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -534,6 +534,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: output_config: dict[str, object] | None = None, auto_summary_level: str = "detailed", *, + model: str, drop_params: bool = True, ) -> dict[str, object] | None: """ @@ -555,7 +556,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if isinstance(output_config, dict) and output_config.get("effort"): effort = output_config["effort"] elif thinking_type == "enabled": - budget_effort: Final = reasoning_effort_from_thinking(thinking, drop_params=drop_params) + budget_effort: Final = reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) if budget_effort is None: return None effort = budget_effort @@ -667,6 +668,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: thinking, output_config=cast(dict[str, object] | None, output_config), auto_summary_level=auto_summary_level, + model=model, drop_params=drop_params, ) if reasoning: diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 27e003ce8b6..2da071ebdd2 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -9,7 +9,7 @@ from dataclasses import dataclass, field from functools import reduce from itertools import accumulate, groupby from types import MappingProxyType -from typing import Annotated, Final, Literal, Protocol, TypeAlias +from typing import Annotated, ClassVar, Final, Literal, Protocol, TypeAlias import httpx from pydantic import ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError @@ -81,6 +81,10 @@ _DEPLOYMENT_OPTIONS: Final = frozenset( ) +class _BaselineAnthropicMessagesConfig(AnthropicMessagesConfig): + _report_reasoning_drops: ClassVar[bool] = False + + class _StrictModel(LiteLLMBaseModel): model_config = ConfigDict(extra="forbid", frozen=True, strict=True) @@ -661,7 +665,7 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), dict(options), ) - config: Final = AnthropicMessagesConfig() + config: Final = _BaselineAnthropicMessagesConfig() body: Final = config.transform_anthropic_messages_request( model=resolved_model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 09777da8669..2b757bb10d9 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -14,6 +14,7 @@ from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import * from litellm.types.responses.main import * @@ -92,6 +93,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def _effort_resolves_to_none(model: str, effort: str | None) -> bool: return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort) + def deployment_names_wrap_openai_models(self) -> bool: + return True + + def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None: + return SUMMARY_ONLY_REASONING_FIELD_VALUES if self._is_gpt_5_model(model) else None + def get_supported_openai_params(self, model: str) -> list: """ Azure Responses API does not support context_management (compaction). diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 5381447f28f..5214d31fb2e 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.azure_ai.common_utils import ( api_key_header_for_base, @@ -40,6 +41,14 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +_REQUEST_BODY: Final = TypeAdapter(dict[str, object]) + + +def _request_model(request: Mapping[str, object]) -> str | None: + model: Final = request.get("model") + return model if isinstance(model, str) else None + + class AzureFoundryErrorStrings(str, enum.Enum): SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'" @@ -510,7 +519,13 @@ class AzureAIStudioConfig(OpenAIConfig): request_data = self._drop_extra_params_from_request_data(request_data, error_text) if "Extra inputs are not permitted" in error_text and self._error_has_tool_level_extra_fields(error_text): request_data = self._drop_tool_level_extra_fields(request_data, error_text) - data: Final = drop_params_from_unprocessable_entity_error(e=e, data=request_data) + data: Final = drop_params_from_unprocessable_entity_error( + e=e, + data=request_data, + model=_request_model(_REQUEST_BODY.validate_python(request_data)), + drop_params_gated="Extra inputs are not permitted" in error_text + and not self._error_has_tool_level_extra_fields(error_text), + ) return data def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict: @@ -526,6 +541,14 @@ class AzureAIStudioConfig(OpenAIConfig): def _drop_extra_params_from_request_data(self, request_data: dict, error_text: str) -> dict: params_to_drop: Final = self._extract_params_to_drop_from_error_text(error_text) if params_to_drop: + request: Final = _REQUEST_BODY.validate_python(request_data) + warn_dropped_reasoning_params( + params_to_drop, + request, + _request_model(request), + "Azure AI rejected it as an extra parameter its deployment does not pass through.", + drop_params_gated=False, + ) for param in params_to_drop: if param in request_data: request_data.pop(param, None) diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index f1539d16b30..ff714bb908b 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -1,10 +1,12 @@ +import dataclasses from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence from types import TracebackType -from typing import TYPE_CHECKING, Any, Protocol +from typing import TYPE_CHECKING, Any, Literal, Protocol import httpx +from litellm._uuid import uuid from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents from litellm.types.realtime import ( RealtimeInputAudioTranscriptionUsage, @@ -22,6 +24,26 @@ else: LiteLLMLoggingObj = Any +@dataclasses.dataclass(frozen=True, slots=True) +class RealtimeSessionUpdateRejection: + code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"] + message: str + param: str + + def error_event(self, client_event_id: object) -> dict[str, object]: + return { + "type": "error", + "event_id": f"event_{uuid.uuid4()}", + "error": { + "type": "invalid_request_error", + "code": self.code, + "message": self.message, + "param": self.param, + "event_id": client_event_id if isinstance(client_event_id, str) else None, + }, + } + + class RealtimeBackend(Protocol): async def __aenter__(self) -> object: ... @@ -93,6 +115,14 @@ class BaseRealtimeConfig(ABC): def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session return None + def builds_setup_from_session_update(self) -> bool: + return False + + def setup_session_update_rejection( + self, model: str, session: Mapping[str, object], *, setup_sent: bool + ) -> RealtimeSessionUpdateRejection | None: + return None + def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: return None diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 792b14090b8..32d0e408aba 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -1,7 +1,7 @@ import types from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast, get_args import httpx @@ -27,6 +27,20 @@ else: BaseLLMException = Any ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"] +ReasoningFieldValues: TypeAlias = Mapping[str, frozenset[str | None] | None] +REASONING_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None)) +SUMMARY_ONLY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType( + {"summary": REASONING_SUMMARY_VALUES} +) +STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType( + {**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": frozenset({"standard"})} +) +STANDARD_MODE_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType( + {**STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, "context": None} +) +ANY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType( + {**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": None, "context": None} +) class BaseResponsesAPIConfig(ABC): @@ -100,6 +114,18 @@ class BaseResponsesAPIConfig(ABC): """The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for.""" return "detailed" + def reasoning_field_values(self) -> ReasoningFieldValues | None: + return None + + def deployment_names_wrap_openai_models(self) -> bool: + return False + + def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None: + return None + + def ignores_unsupported_reasoning_fields(self) -> bool: + return False + @abstractmethod def map_openai_params( self, diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 131d4afae5b..c3e0ae333fe 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( inline_remote_image_urls, inline_remote_media, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token @@ -43,6 +44,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_runtime_chat_completions_serves_reasoning_inline, bedrock_serves_service_tier, split_bedrock_region_path, + strip_bedrock_routing_prefix, ) from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig @@ -122,6 +124,13 @@ def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) - effort: Final = params.get("reasoning_effort") if not isinstance(effort, str) or effort not in chat_completions_reasoning_efforts_refused_for(model): return params + warn_dropped_reasoning( + "reasoning_effort", + effort, + strip_bedrock_routing_prefix(model), + "AWS's Chat Completions endpoint rejects that level for this model family, so AWS applies its default effort.", + drop_params_gated=False, + ) return _without_params(params, frozenset(("reasoning_effort",))) @@ -453,6 +462,14 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): ), status_code=400, ) + if malformed_effort: + warn_dropped_reasoning( + "reasoning_effort", + raw_params["reasoning_effort"], + strip_bedrock_routing_prefix(model), + "Bedrock's Chat Completions endpoint takes it only as a string, so AWS applies its default effort.", + drop_params_gated=True, + ) rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]() unsupported_tier: Final = _unsupported_service_tier(model, raw_params) if unsupported_tier and not (litellm.drop_params or drop_params): diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 7c266928212..5a6ae980ce2 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -50,6 +50,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, @@ -63,6 +64,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image from litellm.llms.bedrock.common_utils import ( bedrock_chat_uses_native_structured_output, bedrock_cleaned_output_schema, + bedrock_model_is_unknown, bedrock_model_supports_regex_lookaround, bedrock_native_output_format, bedrock_native_output_is_opt_in, @@ -187,6 +189,8 @@ def _tools_the_model_accepts( _OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock) +_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field." +_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort." def _has_native_text_format(output_config: object) -> bool: @@ -530,13 +534,14 @@ class AmazonConverseConfig(BaseConfig): """ Handle the reasoning_effort parameter based on the model type. - - GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields. + - GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged + via additionalModelRequestFields. - OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields. - Nova 2 models: transformed to reasoningConfig. - Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on adaptive Claude 4.6 / 4.7). """ - if "gpt-oss" in model or "deepseek" in model: + if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model): optional_params["reasoning_effort"] = reasoning_effort elif self._is_openai_gpt_reasoning_model(model): reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort} @@ -639,6 +644,10 @@ class AmazonConverseConfig(BaseConfig): or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider) ) + def _warn_native_reasoning_drop(self, param: str, value: object, model: str, reason: str) -> None: + if param in self.get_supported_openai_params(model): + warn_dropped_reasoning(param, value, model, reason, drop_params_gated=False) + def _model_rejects_reasoning_effort_param(self, model: str, base_model: str) -> bool: """Whether the model returns a 400 for every ``reasoning_effort`` shape on Converse. @@ -682,7 +691,7 @@ class AmazonConverseConfig(BaseConfig): "parallel_tool_calls", ] - if "arn" in model: # we can't infer the model from the arn, so just add all params + if "arn" in model or bedrock_model_is_unknown(model): supported_params.append("tools") supported_params.append("tool_choice") supported_params.append("thinking") @@ -1142,9 +1151,8 @@ class AmazonConverseConfig(BaseConfig): "tool_choice": {"type": "auto", "disable_parallel_tool_use": not value} } if param == "thinking" and drop_thinking_param: - verbose_logger.debug( - "Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively.", - model, + self._warn_native_reasoning_drop( + param, OPAQUE_VALUE.validate_python(value), model, _DEEPSEEK_THINKING_REASON ) elif param == "thinking" and not self._is_openai_gpt_reasoning_model(model): if ( @@ -1178,10 +1186,7 @@ class AmazonConverseConfig(BaseConfig): drop_params=drop_params, ) elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param: - verbose_logger.debug( - "Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.", - model, - ) + self._warn_native_reasoning_drop(param, value, model, _DEEPSEEK_R1_EFFORT_REASON) elif ( param == "reasoning_effort" and isinstance(value, str) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 852179e8caa..877fb4ca692 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -2,7 +2,7 @@ import copy import json import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, cast, get_args +from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config @@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = ( "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}" ) -_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python( - get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL) -) - def _response_header(headers: Mapping[str, str], name: str) -> str | None: return headers.get(name, None) @@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): """ return False - @staticmethod - def get_bedrock_invoke_provider( - model: str, - ) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None: - """ - Helper function to get the bedrock provider from the model - - handles 4 scenarios: - 1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic` - 2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic` - 3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama` - 4. model=us.amazon.nova-pro-v1:0 -> Returns `nova` - """ - if model.startswith("invoke/"): - model = model.replace("invoke/", "", 1) - - # Special case: Check for "nova" in model name first (before "amazon") - # This handles amazon.nova-* models which would otherwise match "amazon" (Titan) - if "nova" in model.lower(): - if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova") - - _split_model: Final = model.split(".")[0] - if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model) - - # If not a known provider, check for pattern with two slashes - provider = AmazonInvokeConfig._get_provider_from_model_path(model) - if provider is not None: - return provider - - for provider in _INVOKE_PROVIDERS: - if provider in model: - return provider - return None - - @staticmethod - def _get_provider_from_model_path( - model_path: str, - ) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None: - """ - Helper function to get the provider from a model path with format: provider/model-name - - Args: - model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name') - - Returns: - Optional[str]: The provider name, or None if no valid provider found - """ - parts: Final = model_path.split("/") - if len(parts) >= 1: - provider: Final = parts[0] - if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider) - return None - def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]: # handle anthropic prompts and amazon titan prompts prompt = "" diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index d37cdcb848d..fff66f33610 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1098,6 +1098,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None: return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw) +def _cost_map_row_names_a_provider(key: str) -> bool: + entry: Final = _model_cost_entry(key) + return entry is not None and entry.get("litellm_provider") is not None + + def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: return tuple( _model_cost_entry(key) @@ -1589,7 +1594,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ The deployment's ``model_info.supports_regex_lookaround`` wins, then the ``model_prices_and_context_window.json`` entry of its ``base_model``, then the - entry of ``model`` itself. A model nobody flagged keeps its schema as sent. + entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless + no candidate names a model LiteLLM knows, whatever route prefix it carries. """ params: Final = litellm_params or {} deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround") @@ -1598,7 +1604,41 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ base_model: Final = params.get("base_model") candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model) flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates) - return next((flag for flag in flags if flag is not None), True) + flag: Final = next((flag for flag in flags if flag is not None), None) + if flag is not None: + return flag + return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates) + + +def _bedrock_model_id_without_region(model: str) -> str: + routed_id: Final = model.removeprefix("bedrock/") + region, separator, regional_id = routed_id.partition("/") + return regional_id if separator and region in get_all_bedrock_regions() else routed_id + + +def bedrock_model_is_unknown(model: str) -> bool: + model_id: Final = _bedrock_model_id_without_region(model) + if "/" in model_id: + return False + base: Final = get_bedrock_base_model(model_id) + regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions()) + if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)): + return False + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None + + +def _names_no_invoke_provider(model: str) -> bool: + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + model_id: Final = _bedrock_model_id_without_region(model) + names_one_model: Final = bool(model_id) and ("/" not in model_id or model_id.startswith("arn:")) + return names_one_model and BaseAWSLLM.get_bedrock_invoke_provider(model) is None + + +def bedrock_model_reaches_converse_by_default(model: str) -> bool: + return BedrockModelInfo.named_bedrock_route(model, None) is None and _names_no_invoke_provider(model) _BedrockConverseModelFlag: TypeAlias = Literal[ @@ -1607,10 +1647,13 @@ _BedrockConverseModelFlag: TypeAlias = Literal[ ] -def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None: +def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]: base: Final = get_bedrock_base_model(model) - candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))) - for candidate in candidates: + return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))) + + +def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None: + for candidate in _bedrock_model_info_candidates(model): with contextlib.suppress(Exception): model_info = get_cached_model_info()( model=candidate, @@ -1788,8 +1831,19 @@ class BedrockModelInfo(BaseLLMModelInfo): default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in; ``request_params`` (the caller's chat params) sends such a request to Converse when it needs a feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family - model stays on Converse without the prefix. + model stays on Converse without the prefix. A model the cost map does not list as Converse goes to + Invoke only when its id names an Invoke provider, and to Converse otherwise. """ + named_route: Final = BedrockModelInfo.named_bedrock_route(model, request_params) + if named_route is not None: + return named_route + return "converse" if _names_no_invoke_provider(model) else "invoke" + + @staticmethod + def named_bedrock_route( + model: str, + request_params: Mapping[str, object] | None, + ) -> BedrockRoute | None: route_mappings: dict[ str, Literal[ @@ -1844,7 +1898,7 @@ class BedrockModelInfo(BaseLLMModelInfo): return "converse" if _OPENAI_FAMILY_MODEL_RE.search(base_model): return "converse" - return "invoke" + return None @staticmethod def _explicit_converse_route(model: str) -> bool: diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index e9b2f26e977..00622a1e287 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm._logging import redact_string, verbose_proxy_logger -from litellm._uuid import uuid from litellm.constants import ( BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS, BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY, @@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes +from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection from litellm.types.llms.bedrock import AwsAuthParams from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput from ..base_aws_llm import BaseAWSLLM, run_aws_signing from ..common_utils import BedrockError -from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities +from .transformation import BedrockRealtimeConfig, requested_modalities _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) @@ -256,23 +256,11 @@ async def _ack_session_update( async def _reject_session_update( client_ws: RealtimeClientWebSocket, parsed_client_message: Mapping[str, JsonValue], - rejection: BedrockSessionUpdateRejection, + rejection: RealtimeSessionUpdateRejection, ) -> bool: """Send an error event for a session.update Bedrock cannot apply; False means the client is gone.""" - client_event_id: Final = parsed_client_message.get("event_id") - error_event: Final = { - "type": "error", - "event_id": f"event_{uuid.uuid4()}", - "error": { - "type": "invalid_request_error", - "code": rejection.code, - "message": rejection.message, - "param": rejection.param, - "event_id": client_event_id if isinstance(client_event_id, str) else None, - }, - } try: - await client_ws.send_text(json.dumps(error_event)) + await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id")))) except Exception as e: # noqa: BLE001 # any send failure means the client is gone verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True) return False diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 952d8858f78..25498fef46f 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -11,7 +11,7 @@ import math import uuid as uuid_lib from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Final, Literal +from typing import Final import httpx from pydantic import ConfigDict, JsonValue, TypeAdapter @@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm from litellm.types.llms.base import LiteLLMBaseModel @@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[ return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed) -@dataclasses.dataclass(frozen=True, slots=True) -class BedrockSessionUpdateRejection: - code: Literal["invalid_type", "invalid_value", "session_update_not_applied"] - message: str - param: str - - @dataclasses.dataclass(frozen=True, slots=True) class BedrockLockedSessionSettings: """The session settings Bedrock Nova Sonic locks in once the prompt starts.""" @@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): max_tokens=max_tokens, ) - def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None: + def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None: """The error for a session.update this session cannot apply, or None when it applies.""" session_config: Final = json_message.get("session") if not isinstance(session_config, Mapping): @@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): malformed: Final = () if self.drop_params else _malformed_settings(session_config) if malformed: name, expected = malformed[0] - return BedrockSessionUpdateRejection( + return RealtimeSessionUpdateRejection( code="invalid_type", message=( f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. " @@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): ) unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config) if unplayable_voice is not None: - return BedrockSessionUpdateRejection( + return RealtimeSessionUpdateRejection( code="invalid_value", message=( f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. " @@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): if not self.prompt_started: return None if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings: - return BedrockSessionUpdateRejection( + return RealtimeSessionUpdateRejection( code="session_update_not_applied", message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied", param="session", diff --git a/litellm/llms/chatgpt/authenticator.py b/litellm/llms/chatgpt/authenticator.py index a5fb1c9c6b9..b237116c5f1 100644 --- a/litellm/llms/chatgpt/authenticator.py +++ b/litellm/llms/chatgpt/authenticator.py @@ -122,6 +122,12 @@ class Authenticator: return tokens["access_token"] def get_account_id(self, access_token: str | None = None) -> str | None: + return self._account_id(access_token, persist_derived=True) + + def read_account_id(self, access_token: str | None = None) -> str | None: + return self._account_id(access_token, persist_derived=False) + + def _account_id(self, access_token: str | None, persist_derived: bool) -> str | None: auth_data: Final = self._read_auth_file() if access_token and (not auth_data or auth_data.get("access_token") != access_token): return extract_chatgpt_account_id(access_token) @@ -133,7 +139,7 @@ class Authenticator: id_token: Final = auth_data.get("id_token") saved_access_token: Final = auth_data.get("access_token") derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token)) - if derived: + if derived and persist_derived: self._write_auth_file({**auth_data, "account_id": derived}) return derived diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 72b5cde21d1..26a5a3f150b 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo safe_convert_created_field, ) from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.llms.openai.common_utils import OpenAIError from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -295,3 +296,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def supports_native_websocket(self) -> bool: """ChatGPT does not support native WebSocket for Responses API""" return False + + def reasoning_field_values(self) -> ReasoningFieldValues: + return STANDARD_MODE_REASONING_FIELD_VALUES diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c92315e53b3..9a2b352dd86 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -249,7 +249,7 @@ if TYPE_CHECKING: FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig - from litellm.responses.streaming_iterator import ResponsesWsFrameGuard + from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission from litellm.types.google_genai.main import GenerateContentResponse from litellm.types.llms.openai_evals import ( CancelEvalResponse, @@ -6973,14 +6973,19 @@ class BaseLLMHTTPHandler: ), ) - # Auto-send session setup if the provider requires it (e.g. - # Gemini/Vertex AI Live needs a `setup` before any realtime_input). + # Auto-send session setup at connect unless the provider builds it from the + # client's first session.update (Gemini/Vertex AI Live accept exactly one + # setup, so RealTimeStreaming sends theirs on the first client message). # Build the streaming handler first so a transcription guardrail's # auto-response disable can be folded into this one setup: Gemini # rejects a second setup, so a follow-up disable would be dropped # and the guardrail bypassed. _session_config: str | None = None - if provider_config.requires_session_configuration(): + sends_setup_at_connect: Final = ( + provider_config.requires_session_configuration() + and not provider_config.builds_setup_from_session_update() + ) + if sends_setup_at_connect: _session_config = provider_config.session_configuration_request(model) if _session_config: _session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable( @@ -7264,7 +7269,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str | None = None, first_message: str | None = None, request_defaults: ResponsesWebSocketRequestDefaults | None = None, - turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_admission: "ResponsesWsTurnAdmission | None" = None, frame_guard: "ResponsesWsFrameGuard | None" = None, **kwargs: object, ) -> Exception | None: diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index ecaa0a2ef16..5b7c2a2c410 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -6,10 +6,12 @@ from collections.abc import Coroutine, Mapping, Sequence from typing import Final, Literal, cast, overload import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_search_results_text, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -18,6 +20,16 @@ from litellm.utils import supports_reasoning, supports_vision from ...openai.chat.gpt_transformation import OpenAIGPTConfig SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra") +_NO_BUDGET_REASON: Final = "DeepSeek takes thinking only as enabled or disabled, with no budget." + + +def _warn_on_dropped_budget(thinking: object, model: str) -> None: + fields: Final = as_str_mapping(thinking) + if fields is None or fields.get("type") != "enabled" or "budget_tokens" not in fields: + return + warn_dropped_reasoning( + "thinking.budget_tokens", fields["budget_tokens"], model, _NO_BUDGET_REASON, drop_params_gated=False + ) class DeepSeekChatConfig(OpenAIGPTConfig): @@ -53,6 +65,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig): # Then re-add only if valid for DeepSeek thinking_value: Final = optional_params.pop("thinking", None) reasoning_effort: Final = optional_params.pop("reasoning_effort", None) + _warn_on_dropped_budget(OPAQUE_VALUE.validate_python(thinking_value), model) # Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"): diff --git a/litellm/llms/fal_ai/chat/transformation.py b/litellm/llms/fal_ai/chat/transformation.py index 115b1f840de..848ae498cdd 100644 --- a/litellm/llms/fal_ai/chat/transformation.py +++ b/litellm/llms/fal_ai/chat/transformation.py @@ -12,7 +12,9 @@ from typing import TYPE_CHECKING, Final import httpx from pydantic import ConfigDict, TypeAdapter +import litellm from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel @@ -29,6 +31,10 @@ PROVIDER_PREFIX: Final[str] = "fal_ai/" PASSTHROUGH_PARAMS: Final[frozenset[str]] = frozenset(("reasoning", "temperature", "top_p")) REASONING_DISABLED_EFFORTS: Final[frozenset[str]] = frozenset(("none", "minimal")) REASONING_ENABLED_EFFORTS: Final[frozenset[str]] = frozenset(("low", "medium", "high")) +_UNSUPPORTED_EFFORT_REASON: Final = ( + f"fal only turns reasoning off for {sorted(REASONING_DISABLED_EFFORTS)} " + f"and on for {sorted(REASONING_ENABLED_EFFORTS)}." +) class _FalUsage(LiteLLMBaseModel): @@ -120,9 +126,10 @@ class FalAIChatConfig(BaseConfig): return False if isinstance(value, str) and value in REASONING_ENABLED_EFFORTS: return True - if drop_params: - return None - raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}") + if not (drop_params or litellm.drop_params is True): + raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}") + warn_dropped_reasoning("reasoning_effort", value, model, _UNSUPPORTED_EFFORT_REASON, drop_params_gated=True) + return None def _translate_param(self, param: str, value: object, model: str, drop_params: bool) -> tuple[str, object] | None: if param in ("temperature", "top_p"): diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index fa5794d71b3..92c6c8439b3 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_litellm_response_only_fields, unpack_legacy_defs, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -97,6 +98,7 @@ def _json_schema_response_format(schema: object, name: str) -> Mapping[str, obje EFFORT_KWARG_KEYS: Final = frozenset({"enable_thinking", "thinking", "reasoning_budget", "low_effort"}) +_NOT_A_REASONING_MODEL_REASON: Final = "litellm's model map does not list it as a reasoning model." def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bool | None: @@ -107,6 +109,13 @@ def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bo return None +def warn_dropped_effort_kwargs(chat_template_kwargs: object, effort: object, model: str) -> None: + if effort != "none": + warn_dropped_reasoning( + "chat_template_kwargs", chat_template_kwargs, model, _NOT_A_REASONING_MODEL_REASON, drop_params_gated=False + ) + + def effort_from_chat_template_kwargs(kwargs: Mapping[str, object]) -> object: enable_thinking: Final = _bool_from_kwargs(kwargs, ("enable_thinking", "thinking")) if enable_thinking is False: @@ -438,10 +447,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): if effort is None: return () if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): - verbose_logger.debug( - "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.", - model, - ) + warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model) return () return (("reasoning_effort", effort),) diff --git a/litellm/llms/fireworks_ai/completion/transformation.py b/litellm/llms/fireworks_ai/completion/transformation.py index 1f5d4f306a9..611e60cedd4 100644 --- a/litellm/llms/fireworks_ai/completion/transformation.py +++ b/litellm/llms/fireworks_ai/completion/transformation.py @@ -2,6 +2,7 @@ from collections.abc import Mapping from typing import Final from litellm._logging import verbose_logger +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage from litellm.utils import supports_reasoning @@ -12,6 +13,7 @@ from ..chat.transformation import ( NIM_VLLM_STRIP_PARAMS, FireworksAIConfig, effort_from_chat_template_kwargs, + warn_dropped_effort_kwargs, ) from ..common_utils import FireworksAIMixin, resolve_fireworks_resource_name @@ -129,10 +131,7 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig ) return result if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): - verbose_logger.debug( - "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.", - model, - ) + warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model) return result return {**result, "reasoning_effort": effort} diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index 86d9df9618a..463b9cbfc9a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -6,6 +6,10 @@ from urllib.parse import unquote import httpx from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.responses.transformation import ( + STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, + ReasoningFieldValues, +) from litellm.llms.fireworks_ai.common_utils import ( FIREWORKS_FORWARD_USER_ID_PARAM, get_fireworks_forwarded_user_id, @@ -129,3 +133,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def supports_native_file_search(self) -> bool: return False + + def reasoning_field_values(self) -> ReasoningFieldValues: + return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES + + def ignores_unsupported_reasoning_fields(self) -> bool: + return True diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 6f778ab2761..3134dfa1e4b 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -7,6 +7,7 @@ from collections import OrderedDict from collections.abc import Mapping, Sequence from typing import Final, cast +from pydantic import TypeAdapter from typing_extensions import ReadOnly, Required, TypedDict import litellm @@ -15,8 +16,9 @@ from litellm._uuid import uuid from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + OPENAI_TO_GEMINI_TTS_VOICES, VertexGeminiConfig, ) from litellm.responses.litellm_completion_transformation.transformation import ( @@ -76,27 +78,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res # Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped. _KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT} +_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"}) OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset( {"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"} ) +# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28) +GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset( + { + "achernar", + "achird", + "algenib", + "algieba", + "alnilam", + "aoede", + "autonoe", + "callirrhoe", + "charon", + "despina", + "enceladus", + "erinome", + "fenrir", + "gacrux", + "iapetus", + "kore", + "laomedeia", + "leda", + "orus", + "puck", + "pulcherrima", + "rasalgethi", + "sadachbia", + "sadaltager", + "schedar", + "sulafat", + "umbriel", + "vindemiatrix", + "zephyr", + "zubenelgenubi", + } +) -def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None: + +def _is_undocumented_live_voice(voice: object) -> bool: + return ( + isinstance(voice, str) + and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES + and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES + ) + + +def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None: """Build the Gemini Live speechConfig for a client-requested voice. - OpenAI stock voice names have no Gemini equivalent and Gemini Live closes - the session on an unknown voice, so they are dropped with a warning and - the model keeps its default voice. Every other name is forwarded verbatim. + OpenAI stock voice names have no Gemini equivalent, so they are always + dropped, and so is any name outside Google's documented Live voices when + ``drop_undocumented`` is set, keeping the model's default voice. Every + other name is forwarded verbatim. """ - if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES: + if not isinstance(voice, str): + return VertexGeminiConfig().map_audio_params({"voice": voice}) + if voice.lower() in OPENAI_STOCK_REALTIME_VOICES: verbose_logger.warning( "Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; " "dropping it so the session keeps the model's default voice.", voice, ) return None + if drop_undocumented and _is_undocumented_live_voice(voice): + verbose_logger.warning( + "Gemini Realtime: voice %s is not a documented Gemini Live voice; " + "drop_params is set, so it is dropped and the session keeps the model's default voice.", + voice, + ) + return None return VertexGeminiConfig().map_audio_params({"voice": voice}) @@ -142,16 +199,34 @@ def _base64_decoded_byte_count(data: str) -> int: return max(len(data) * 3 // 4 - padding, 0) +_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...]) + + +def _chat_completion_tool(tool: object) -> dict[str, object]: + fields: Final = as_str_mapping(tool) or {} + if fields.get("type") != "function" or "function" in fields: + return {**fields} + return {"type": "function", "function": {key: value for key, value in fields.items() if key != "type"}} + + +def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None: + parts: Final = system_instruction.get("parts", []) if system_instruction else [] + texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str)) + return "\n".join(texts) if texts else None + + class GeminiRealtimeConfig(BaseRealtimeConfig): _TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping - def __init__(self): + def __init__(self, drop_params: bool = False): super().__init__() + self.drop_params = drop_params self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict() # Gemini Live sometimes emits usageMetadata in a standalone frame between # turns; buffer it here so the next response.done carries the token counts. self._pending_usage_metadata: dict | None = None self._unbilled_input_audio_bytes: int = 0 + self._manual_activity_open: bool = False def is_setup_message(self, msg_obj: dict) -> bool: return "setup" in msg_obj @@ -167,6 +242,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): """Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``.""" return True + def _rejects_undocumented_voices(self, model: str) -> bool: + """Google AI Studio closes the socket (1007) on an unknown voice only for its native-audio Live models.""" + return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio")) + @staticmethod def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]: detail_mapping: Final = as_str_mapping(details) @@ -272,17 +351,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): except (json.JSONDecodeError, TypeError, AttributeError): return False + def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]: + if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request): + return () + self._manual_activity_open = True + activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}} + return (json.dumps({"realtimeInput": activity_start}),) + + def _end_manual_activity(self) -> tuple[str, ...]: + if not self._manual_activity_open: + return () + self._manual_activity_open = False + activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}} + return (json.dumps({"realtimeInput": activity_end}),) + def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]: """Map OpenAI buffer commit/end to Gemini Live turn-boundary signals.""" if self._manual_turn_detection_enabled(session_configuration_request): - realtime_input_dict: BidiGenerateContentRealtimeInput = { - "activityEnd": True, - } - verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend") - else: - realtime_input_dict = {"audioStreamEnd": True} - verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend") - return [json.dumps({"realtimeInput": realtime_input_dict})] + return list(self._end_manual_activity()) + audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True} + return [json.dumps({"realtimeInput": audio_stream_end})] def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection: """Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``. @@ -325,7 +413,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): "voice", ] - def map_openai_params(self, optional_params: dict, non_default_params: dict) -> dict: + def map_openai_params(self, optional_params: dict, non_default_params: dict, *, model: str) -> dict: if "generationConfig" not in optional_params: optional_params["generationConfig"] = {} for key, value in non_default_params.items(): @@ -333,12 +421,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}]) elif key == "temperature": optional_params["generationConfig"]["temperature"] = value - elif key == "max_response_output_tokens": + elif key == "max_response_output_tokens" and isinstance(value, int): optional_params["generationConfig"]["maxOutputTokens"] = value elif key == "modalities": - optional_params["generationConfig"]["responseModalities"] = [ - modality.upper() for modality in cast(list[str], value) - ] + optional_params["generationConfig"]["responseModalities"] = list( + GeminiRealtimeConfig._requested_response_modalities(cast(object, value)) + ) elif key == "tools": from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, @@ -347,34 +435,43 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): vertex_gemini_config = VertexGeminiConfig() # Tools should be at the top level of setup, not inside generationConfig optional_params["tools"] = vertex_gemini_config.map_function( - value=value, optional_params=optional_params + value=[_chat_completion_tool(tool) for tool in _JSON_ARRAY.validate_python(value)], + optional_params=optional_params, ) elif key == "input_audio_transcription" and value is not None: optional_params["inputAudioTranscription"] = {} - elif key == "turn_detection" and value is not None: - value_typed = cast(OpenAIRealtimeTurnDetection, value) - if ( - isinstance(value_typed, dict) - and value_typed.get("type") == "semantic_vad" - and "create_response" not in value_typed - ): - # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD. - # Only skip when there is no create_response override so that - # a guardrail-injected create_response:false is not dropped. - continue - transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed) - if transformed_audio_activity_config: - optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig( - automaticActivityDetection=transformed_audio_activity_config - ) + elif key == "turn_detection": + realtime_input_config = self._realtime_input_config(cast(object, value)) + if realtime_input_config is not None: + optional_params["realtimeInputConfig"] = realtime_input_config elif key == "voice": - speech_config = _gemini_live_speech_config(value) + speech_config = _gemini_live_speech_config( + value, self.drop_params and self._rejects_undocumented_voices(model) + ) if speech_config: optional_params["generationConfig"]["speechConfig"] = speech_config if len(optional_params["generationConfig"]) == 0: optional_params.pop("generationConfig") return optional_params + def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None: + if turn_detection is None: + return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True}) + value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection) + if ( + isinstance(value_typed, dict) + and value_typed.get("type") == "semantic_vad" + and "create_response" not in value_typed + ): + # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD. + # Only skip when there is no create_response override so that + # a guardrail-injected create_response:false is not dropped. + return None + transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed) + if not transformed_audio_activity_config: + return None + return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config) + @staticmethod def _extract_turn_detection(session: dict) -> dict | None: """Extract turn_detection from a session.update payload. @@ -423,6 +520,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if isinstance(input_cfg, dict): if "input_audio_transcription" not in normalized and "transcription" in input_cfg: normalized["input_audio_transcription"] = input_cfg["transcription"] + if "turn_detection" not in normalized and "turn_detection" in input_cfg: + normalized["turn_detection"] = input_cfg["turn_detection"] output_cfg: Final = audio.get("output") if isinstance(output_cfg, dict) and output_cfg.get("voice"): normalized["voice"] = output_cfg["voice"] @@ -434,7 +533,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return normalized @staticmethod - def _model_cost_entry(model: str) -> dict: + def _model_cost_entry(model: str) -> Mapping[str, object]: entry = litellm.model_cost.get(model) if entry is None: stripped: Final = model.split("/", 1)[-1] @@ -455,18 +554,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO" @staticmethod - def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: - """Swap responseModalities a Live model cannot produce: TEXT to AUDIO for - audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live).""" - normalized: Final = tuple( - modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities + def _requested_response_modalities(value: object) -> tuple[str, ...]: + if isinstance(value, str): + return (value.upper(),) + if not isinstance(value, (list, tuple)): + return () + return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value)) + + @staticmethod + def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]: + unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES) + if unknown: + return unknown + if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)): + return ("TEXT",) + if GeminiRealtimeConfig._is_text_only_live_model(model) and frozenset(requested) == frozenset(("AUDIO",)): + return ("AUDIO",) + return () + + def setup_session_update_rejection( + self, model: str, session: Mapping[str, object], *, setup_sent: bool + ) -> RealtimeSessionUpdateRejection | None: + if self.drop_params: + return None + if setup_sent: + return RealtimeSessionUpdateRejection( + code="session_update_not_applied", + message=( + "Gemini Live accepts one session configuration per connection, and this session already sent " + "its configuration, so this session.update was not applied. Put every setting in session.update " + "before the first conversation content, or set drop_params to drop it instead" + ), + param="session", + ) + return self._modality_rejection(model, session) or self._voice_rejection(model, session) + + @staticmethod + def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None: + param: Final = "output_modalities" if "output_modalities" in session else "modalities" + unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities( + model, GeminiRealtimeConfig._requested_response_modalities(session.get(param)) ) + if not unsupported: + return None + return RealtimeSessionUpdateRejection( + code="invalid_value", + message=( + f"Invalid value for 'session.{param}': {model} cannot produce " + f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead" + ), + param=f"session.{param}", + ) + + def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None: + audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {} + output_voice: Final = audio_output.get("voice") + voice: Final = output_voice or session.get("voice") + if not (self._rejects_undocumented_voices(model) and _is_undocumented_live_voice(voice)): + return None + param: Final = "session.audio.output.voice" if output_voice else "session.voice" + return RealtimeSessionUpdateRejection( + code="invalid_value", + message=( + f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. " + "Set drop_params to drop it instead" + ), + param=param, + ) + + @staticmethod + def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: + """Drop modalities Gemini Live cannot produce at all, swap the ones this + model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for + text-only ones such as transcribe-live), then keep a single modality + because Gemini Live rejects a setup naming two.""" + return GeminiRealtimeConfig._single_response_modality( + GeminiRealtimeConfig._swap_unsupported_response_modalities( + model, GeminiRealtimeConfig._known_response_modalities(model, modalities) + ) + ) + + @staticmethod + def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: + normalized: Final = tuple(str(modality).upper() for modality in modalities) + known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES) + if len(known) < len(normalized): + verbose_logger.warning( + "Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s", + model, + ", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES), + ) + return known or (GeminiRealtimeConfig._default_response_modality(model),) + + @staticmethod + def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]: if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized: + verbose_logger.warning( + "Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO", + model, + ) return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",) if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized: return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",) return normalized + @staticmethod + def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]: + if len(modalities) <= 1: + return modalities + chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0] + verbose_logger.warning( + "Gemini Live: at most one response modality is allowed per session; using %s for the requested %s", + chosen, + ", ".join(modalities), + ) + return (chosen,) + @staticmethod def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]: generation_config: Final = setup.get("generationConfig") @@ -504,7 +707,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # would be silently dropped because ``map_openai_params`` only # recognises the flat OpenAI-beta key names. session_payload = self._normalize_session_payload_for_mapping(session_payload) - new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload) + new_overrides: Final = self.map_openai_params( + optional_params={}, non_default_params=session_payload, model=model + ) if session_configuration_request is None: generation_config: Final = new_overrides.setdefault("generationConfig", {}) @@ -618,14 +823,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): message_str = str(message) raise ValueError(f"Invalid JSON message: {message_str}") - messages: Final[list[str]] = [] msg_type: Final = json_message.get("type") if msg_type == "session.update": return self._handle_session_update(json_message, model, session_configuration_request) if msg_type == "response.create": - return [] # Gemini responds automatically; nothing to forward + return list(self._end_manual_activity()) if msg_type == "conversation.item.create": return self._handle_conversation_item(json_message) @@ -643,8 +847,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict}) verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend") - messages.append(gemini_msg) - return messages + return [*self._start_manual_activity(session_configuration_request), gemini_msg] if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"): return self._handle_input_audio_buffer_commit_or_end(session_configuration_request) @@ -660,22 +863,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): logging_session_id: str, session_configuration_request: str | None = None, ) -> OpenAIRealtimeStreamSessionEvents: - if session_configuration_request: - session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request) - else: - session_configuration_request_dict = {} + session_configuration_request_dict: Final = _parse_setup( + session_configuration_request or self.session_configuration_request(model) + ) _model: Final = session_configuration_request_dict.get("model") or model generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {} gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"]) _modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)] - _system_instruction: Final = session_configuration_request_dict.get("systemInstruction") + instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction")) session: Final = OpenAIRealtimeStreamSession( id=logging_session_id, modalities=_modalities, ) - if _system_instruction is not None and isinstance(_system_instruction, str): - session["instructions"] = _system_instruction + if instructions is not None: + session["instructions"] = instructions if _model is not None and isinstance(_model, str): # Strip Vertex/AI Studio path prefixes to expose the bare model name. if "/models/" in _model: @@ -1662,6 +1864,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # Deferred setup opt-in: litellm.gemini_live_defer_setup = True return not litellm.gemini_live_defer_setup + def builds_setup_from_session_update(self) -> bool: + return True + def session_configuration_request(self, model: str) -> str: """ diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index b055d388dd5..b87a3d543e7 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -24,6 +24,7 @@ from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import ( ResponseInputParam, @@ -437,6 +438,12 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): return False + def reasoning_field_values(self) -> ReasoningFieldValues: + return STANDARD_MODE_REASONING_FIELD_VALUES + + def ignores_unsupported_reasoning_fields(self) -> bool: + return True + def supports_native_websocket(self) -> bool: """GitHub Copilot does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 63e6cb2b5da..9f9503f93b6 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -99,7 +99,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): thinking_effort: Final = ( None if "reasoning_effort" in non_default_params - else reasoning_effort_from_thinking(thinking, drop_params=drop_params) + else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) ) if thinking_effort is not None: non_default_params["reasoning_effort"] = thinking_effort diff --git a/litellm/llms/litellm_proxy/responses/transformation.py b/litellm/llms/litellm_proxy/responses/transformation.py index fc94eb30504..ed62e132c99 100644 --- a/litellm/llms/litellm_proxy/responses/transformation.py +++ b/litellm/llms/litellm_proxy/responses/transformation.py @@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.responses.transformation import ANY_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.utils import LlmProviders @@ -42,6 +43,9 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.LITELLM_PROXY + def reasoning_field_values(self) -> ReasoningFieldValues: + return ANY_REASONING_FIELD_VALUES + def get_complete_url( self, api_base: str | None, diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 2fb9402ed03..a402e2c4048 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -9,17 +9,23 @@ Docs - https://docs.mistral.ai/api/ from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from itertools import accumulate, count from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload +from urllib.parse import urlparse import httpx +from pydantic import TypeAdapter from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 +import litellm from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, strip_litellm_internal_message_fields, strip_none_values_from_message, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, @@ -30,7 +36,15 @@ from litellm.router_utils.reasoning_effort_capability import ( nearest_declared_reasoning_effort, ) from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.mistral import MistralTextBlock, MistralThinkingBlock, MistralToolCallMessage +from litellm.types.llms.mistral import ( + MistralCacheKeyParams, + MistralFilePart, + MistralReasoningEffortParams, + MistralRequestRouting, + MistralTextBlock, + MistralThinkingBlock, + MistralToolCallMessage, +) from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning @@ -57,6 +71,33 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: _NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) +_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting) +_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams) +_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams) +_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart) +_REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system prompt instead of the requested level." +_NON_LEVEL_EFFORT_REASON: Final = ( + "the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort." +) +_DOCUMENTED_MAGISTRAL_EFFORTS: Final = frozenset({"low", "medium", "high"}) +_REASONING_ON_THINKING_TYPES: Final = frozenset({"enabled", "adaptive"}) + + +def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]: + requested: Final = as_str_mapping(reasoning_effort) + if requested is None: + return reasoning_effort, None + return requested.get("effort"), requested.get("summary") + + +def _warn_unless_reasoning_prompt_honors(param: str, value: object, model: str) -> None: + if param == "reasoning_effort" and isinstance(value, str) and value in _DOCUMENTED_MAGISTRAL_EFFORTS: + return + thinking: Final = as_str_mapping(value) + thinking_type: Final = thinking.get("type") if thinking is not None else None + if param == "thinking" and isinstance(thinking_type, str) and thinking_type in _REASONING_ON_THINKING_TYPES: + return + warn_dropped_reasoning(param, value, model, _REASONING_PROMPT_REASON, drop_params_gated=False) def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list @@ -88,6 +129,8 @@ class MistralConfig(OpenAIGPTConfig): - `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message. """ + _is_base_class = False + temperature: int | None = None top_p: int | None = None max_tokens: int | None = None @@ -131,6 +174,7 @@ class MistralConfig(OpenAIGPTConfig): def get_supported_openai_params(self, model: str) -> list[str]: is_magistral: Final = "magistral" in model.lower() accepts_reasoning_effort: Final = self._accepts_reasoning(model) + accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral" return [ "stream", "temperature", @@ -145,6 +189,7 @@ class MistralConfig(OpenAIGPTConfig): "parallel_tool_calls", *(("thinking",) if is_magistral else ()), *(("reasoning_effort",) if accepts_reasoning_effort else ()), + *(("prompt_cache_key",) if accepts_prompt_cache_key else ()), ] def _map_tool_choice(self, tool_choice: str) -> str: @@ -211,15 +256,90 @@ class MistralConfig(OpenAIGPTConfig): optional_params["extra_body"] = {"random_seed": value} if param == "response_format": optional_params["response_format"] = value - if param == "reasoning_effort" and "magistral" not in model.lower(): - optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider) + if ( + param == "reasoning_effort" + and "magistral" not in model.lower() + and ( + reasoning_level := self._reasoning_level_to_send( + model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params + ) + ) + is not None + ): + optional_params["reasoning_effort"] = reasoning_level if param in ("reasoning_effort", "thinking") and "magistral" in model.lower(): # Flag that we need to add reasoning system prompt optional_params["_add_reasoning_prompt"] = True - if param == "parallel_tool_calls": - optional_params["parallel_tool_calls"] = value + _warn_unless_reasoning_prompt_honors(param, OPAQUE_VALUE.validate_python(value), model) + if param in ("parallel_tool_calls", "prompt_cache_key"): + optional_params[param] = value return optional_params + def _reasoning_level_to_send( + self, model: str, requested: MistralReasoningEffortParams, drop_params: bool + ) -> str | None: + requested_effort: Final = requested.get("reasoning_effort") + level, summary = _reasoning_level_and_summary(requested_effort) + sendable: Final = ( + _accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None + ) + if sendable is not None and summary is None: + return sendable + if drop_params or litellm.drop_params is True: + if sendable is None and level is not None: + warn_dropped_reasoning( + "reasoning_effort", requested_effort, model, _NON_LEVEL_EFFORT_REASON, drop_params_gated=True + ) + return sendable + raise UnsupportedParamsError( + message=( + f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, " + "since the Mistral API takes it only as a level such as 'high', with no reasoning summary. " + "To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=self.custom_llm_provider, + model=model, + ) + + def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool: + resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None) + hostname: Final = urlparse(resolved_api_base).hostname or "" + return hostname == "mistral.ai" or hostname.endswith(".mistral.ai") + + @staticmethod + def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool: + return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii() + + def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None: + if not self.accepts_prompt_cache_key_at(api_base): + return "since only the Mistral API (*.mistral.ai) takes it" + if not self.takes_prompt_cache_key_value(prompt_cache_key): + return "since the Mistral API takes it only as a string of ASCII characters" + return None + + def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool: + requested_api_base: Final = routing.get("api_base") + api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None + allowed_openai_params: Final = routing.get("allowed_openai_params") + if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params: + return False + rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base) + if rejection is None: + return False + if litellm.drop_params is True or routing.get("drop_params") is True: + return True + requested_provider: Final = routing.get("custom_llm_provider") + provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider + raise UnsupportedParamsError( + message=( + f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. " + "To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set " + "`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=provider, + model=model, + ) + def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai api_base = ( @@ -359,24 +479,31 @@ class MistralConfig(OpenAIGPTConfig): messages = self._handle_message_with_file(messages) return messages + @staticmethod + def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]: + file_part: Final = block.get("file") if block.get("type") == "file" else None + file_id: Final = ( + _MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None + ) + if not file_id: + return block + return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id} + + @classmethod + def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues: + content: Final = message.get("content") + if not isinstance(content, list): + return message + blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content) + return cast( # cast-ok: only file blocks change, to the file_id shape the Mistral API takes + AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]} + ) + def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: """ Mistral API supports only 'file_id' in message content with type 'file'. """ - for m in messages: - _content_block = m.get("content") - if _content_block and isinstance(_content_block, list): - if any(c.get("type") == "file" for c in _content_block): - # If file content is present, we get file_id from 'file' attribute of content block - # then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it. - file_contents = [c for c in _content_block if c.get("type") == "file"] - for file_content in file_contents: - file_id = file_content.get("file", {}).get("file_id") - if file_id: - # Replace 'file' with 'file_id' - file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape - file_content.pop("file", None) - return messages + return [self._with_mistral_file_ids(m) for m in messages] def _add_reasoning_system_prompt_if_needed( self, messages: list[AllMessageValues], optional_params: dict @@ -794,7 +921,16 @@ class MistralConfig(OpenAIGPTConfig): if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False): messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params) - upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"} + cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params) + withheld_params: Final = ( + ("client_metadata", "prompt_cache_key") + if "prompt_cache_key" in cache_key_params + and self._prompt_cache_key_withheld( + model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params) + ) + else ("client_metadata",) + ) + upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params} # Call parent transform_request which handles _transform_messages return super().transform_request( diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 8257a75b4a5..9c827822dc1 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -21,6 +21,7 @@ from litellm._logging import verbose_logger from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -428,6 +429,13 @@ class OCIChatConfig(BaseConfig): alias = param_map.get(key) if key == "reasoning_effort" and alias and not _model_supports_reasoning_effort(model): if drop_params or litellm.drop_params: + warn_dropped_reasoning( + key, + value, + model, + "the model map lists no reasoning_effort levels for it, so OCI would reject reasoningEffort.", + drop_params_gated=True, + ) continue raise OCIError( status_code=400, @@ -451,6 +459,14 @@ class OCIChatConfig(BaseConfig): if key == "n" and (value is None or value == 1): continue if drop_params or litellm.drop_params: + if key == "reasoning_effort": + warn_dropped_reasoning( + key, + value, + model, + "OCI's Cohere chat API has no reasoning effort field.", + drop_params_gated=True, + ) continue raise OCIError( status_code=400, diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index f9e7071db41..755d647da1d 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_images_from_message, extract_reasoning_content, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -29,7 +30,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ModelResponse, ModelResponseStream -from ..common_utils import OllamaError +from ..common_utils import OllamaError, ollama_think if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -172,10 +173,7 @@ class OllamaChatConfig(BaseConfig): if value.get("json_schema") and value["json_schema"].get("schema"): optional_params["format"] = value["json_schema"]["schema"] if param == "reasoning_effort" and value is not None: - if model.startswith("gpt-oss"): - optional_params["think"] = value - else: - optional_params["think"] = value in {"low", "medium", "high"} + optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value)) ### FUNCTION CALLING LOGIC ### # Ollama 0.4+ supports native tool calling - pass tools directly # and let Ollama handle model capability detection diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index f313c62f026..fcf6a396a13 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -8,11 +8,14 @@ from pydantic import BaseModel, TypeAdapter from litellm import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import ModelResponse, ModelResponseStream _PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object]) +_THINK_ON_EFFORTS: Final = frozenset({"low", "medium", "high"}) +_THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm maps this level to off." _DECODED_JSON: Final = TypeAdapter(object) @@ -296,3 +299,12 @@ class OllamaModelInfo(BaseLLMModelInfo): Return the base model name for Ollama (no-op). """ return model + + +def ollama_think(model: str, reasoning_effort: object) -> object: + if model.startswith("gpt-oss"): + return reasoning_effort + think: Final = reasoning_effort in _THINK_ON_EFFORTS + if isinstance(reasoning_effort, str) and not think and reasoning_effort != "none": + warn_dropped_reasoning("reasoning_effort", reasoning_effort, model, _THINK_OFF_REASON, drop_params_gated=False) + return think diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 239b4e32645..6dbcb050e11 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_inline_remote_media, inline_remote_image_urls, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -43,7 +44,7 @@ from litellm.types.utils import ( generate_id, ) -from ..common_utils import OllamaError, OllamaModelInfo, convert_image +from ..common_utils import OllamaError, OllamaModelInfo, convert_image, ollama_think if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -249,10 +250,7 @@ class OllamaConfig(BaseConfig): elif param == "stop": optional_params["stop"] = value elif param == "reasoning_effort" and value is not None: - if model.startswith("gpt-oss"): - optional_params["think"] = value - else: - optional_params["think"] = value in {"low", "medium", "high"} + optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value)) elif param == "response_format" and isinstance(value, dict): if value["type"] == "json_object": optional_params["format"] = "json" diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index aa14a985afe..c113cb351d8 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -32,6 +32,7 @@ import litellm from litellm.exceptions import error_response_without_credentials from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -165,17 +166,11 @@ _DECODED_JSON: Final = TypeAdapter(object) def drop_params_from_unprocessable_entity_error( e: openai.UnprocessableEntityError | httpx.HTTPStatusError, data: Mapping[str, object], + *, + model: str | None, + drop_params_gated: bool, ) -> dict[str, object]: - """ - Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message. - - Args: - e (UnprocessableEntityError): The UnprocessableEntityError exception - data (Dict[str, Any]): The original data dictionary containing all parameters - - Returns: - Dict[str, Any]: A new dictionary with invalid parameters removed - """ + """Return a copy of ``data`` without the params a 422 response names as invalid.""" invalid_params: Final[list[str]] = [] if isinstance(e, httpx.HTTPStatusError): error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json())) @@ -203,6 +198,13 @@ def drop_params_from_unprocessable_entity_error( invalid_params.append(error_dict["loc"][1]) new_data: Final = {k: v for k, v in data.items() if k not in invalid_params} + warn_dropped_reasoning_params( + invalid_params, + data, + model, + "the provider answered 422 naming it an invalid param.", + drop_params_gated=drop_params_gated, + ) return new_data diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index e0a23d6c99f..bdd9f285823 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -27,6 +27,7 @@ import litellm from litellm import LlmProviders from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RETRIES +from litellm.exceptions import UnsupportedParamsError from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -939,7 +940,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - inference_params = drop_params_from_unprocessable_entity_error(e, inference_params) + inference_params = drop_params_from_unprocessable_entity_error( + e, inference_params, model=model, drop_params_gated=True + ) else: raise e # e.message @@ -971,8 +974,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): litellm.remove_index_from_tool_calls(messages=messages) else: raise e - except OpenAIError as e: - raise e + except (OpenAIError, UnsupportedParamsError): + raise except Exception as e: status_code: Final = getattr(e, "status_code", 500) error_headers = getattr(e, "headers", None) @@ -1096,7 +1099,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - data = drop_params_from_unprocessable_entity_error(e, data) + data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True) else: raise e # e.message @@ -1251,7 +1254,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - data = drop_params_from_unprocessable_entity_error(e, data) + data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True) else: raise e except ( diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 9aaf552e79a..cca6815399f 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, cast import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.core_helpers import finite_cost from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -225,8 +226,11 @@ class OpenrouterConfig(OpenAIGPTConfig): additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata dict[str, object], hidden_params["additional_headers"] ) - if isinstance(response_cost, str | int | float): - additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost) + reported_cost: Final = ( + finite_cost(float(response_cost)) if isinstance(response_cost, str | int | float) else None + ) + if reported_cost is not None: + additional_headers["llm_provider-x-litellm-response-cost"] = reported_cost except Exception: # If we can't extract cost, continue without it - don't fail the response pass diff --git a/litellm/llms/pass_through/typesafe_logging_handler.py b/litellm/llms/pass_through/typesafe_logging_handler.py index 9eae57774c2..7a41d2a5d14 100644 --- a/litellm/llms/pass_through/typesafe_logging_handler.py +++ b/litellm/llms/pass_through/typesafe_logging_handler.py @@ -6,6 +6,7 @@ import httpx from pydantic import TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils.core_helpers import finite_reported_cost from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( @@ -100,9 +101,10 @@ class TypeSafePassthroughLoggingHandler: f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None ) pricing: Final = _pricing_for(candidate_model_keys) + reported_cost: Final = finite_reported_cost(usage.cost) if custom_llm_provider == "openrouter" else None response_cost: Final = ( - usage.cost - if custom_llm_provider == "openrouter" and usage.cost is not None + reported_cost + if reported_cost is not None else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token ) usage_object: Final = Usage( diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 2d762adf958..c3d178771a5 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -6,6 +6,7 @@ Helper util for handling perplexity-specific cost calculation from datetime import datetime from typing import Final +from litellm.litellm_core_utils.core_helpers import finite_reported_cost from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing from litellm.types.utils import Usage from litellm.utils import get_model_info @@ -34,10 +35,9 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non total_cost = cost_info.get("total_cost") elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool): total_cost = float(cost_info) - if total_cost is not None: - # Return total cost as completion_cost (prompt_cost=0) since Perplexity - # doesn't break down by input/output in their cost object - return (0.0, float(total_cost)) + reported_cost: Final = None if total_cost is None else finite_reported_cost(float(total_cost)) + if reported_cost is not None: + return (0.0, reported_cost) ## FALLBACK: Calculate cost manually if Perplexity doesn't provide it ## GET MODEL INFO diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index 58f71028451..afd27b0f696 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -9,6 +9,7 @@ The only provider quirks: Ref: https://docs.perplexity.ai/api-reference/responses-post """ +from types import MappingProxyType from typing import Any, Final import httpx @@ -17,6 +18,7 @@ from pydantic import TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse @@ -24,6 +26,9 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders _DECODED_JSON: Final = TypeAdapter(object) +PERPLEXITY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = MappingProxyType( + {**STANDARD_MODE_REASONING_FIELD_VALUES, "context": frozenset({"auto", "current_turn", "all_turns"})} +) class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): @@ -127,3 +132,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): def supports_native_websocket(self) -> bool: """Perplexity does not support native WebSocket for Responses API""" return False + + def reasoning_field_values(self) -> ReasoningFieldValues: + return PERPLEXITY_REASONING_FIELD_VALUES + + def ignores_unsupported_reasoning_fields(self) -> bool: + return True diff --git a/litellm/llms/sagemaker/chat/transformation.py b/litellm/llms/sagemaker/chat/transformation.py index 4a59d5f1802..76eafcd6bb6 100644 --- a/litellm/llms/sagemaker/chat/transformation.py +++ b/litellm/llms/sagemaker/chat/transformation.py @@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import LlmProviders from ...openai.chat.gpt_transformation import OpenAIGPTConfig -from ..common_utils import AWSEventStreamDecoder, SagemakerError +from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -59,7 +59,12 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM): inference_component_name: Final = optional_params.get("model_id") if not isinstance(inference_component_name, str): return headers - return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name} + other_headers: Final = { + name: value + for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items() + if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower() + } + return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name} def transform_request( self, @@ -76,9 +81,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM): litellm_params=litellm_params, headers=headers, ) - container_request: Final = { - key: value for key, value in request.items() if key not in self.aws_authentication_params - } + header_only_params: Final = frozenset((*self.aws_authentication_params, "model_id")) + container_request: Final = {key: value for key, value in request.items() if key not in header_only_params} served_model_name: Final = litellm_params.get("hf_model_name") if not isinstance(served_model_name, str): return container_request diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index eaa805e7769..bb7719a19a1 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -14,9 +14,10 @@ from typing_extensions import ReadOnly, Required, TypedDict from litellm import verbose_logger from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import GenericStreamingChunk as GChunk -from litellm.types.utils import StreamingChatCompletionChunk +from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True)) @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) @@ -167,6 +168,36 @@ def get_sagemaker_response_stream_shape(): return _load_sagemaker_response_stream_shape() +INFERENCE_COMPONENT_HEADER: Final = "X-Amzn-SageMaker-Inference-Component" + + +_SAGEMAKER_PROVIDERS: Final = frozenset( + provider.value for provider in (LlmProviders.SAGEMAKER, LlmProviders.SAGEMAKER_CHAT, LlmProviders.SAGEMAKER_NOVA) +) + + +def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None: + custom_llm_provider: Final = litellm_params.get("custom_llm_provider") + if isinstance(custom_llm_provider, str) and custom_llm_provider: + return custom_llm_provider + model: Final = litellm_params.get("model") + return model.split("/", 1)[0] if isinstance(model, str) else None + + +def deployment_inference_component(litellm_params: Mapping[str, object]) -> str | None: + headers: Final = litellm_params.get("extra_headers") + if _deployment_provider(litellm_params) not in _SAGEMAKER_PROVIDERS or not isinstance(headers, Mapping): + return None + return next( + ( + value + for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items() + if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str) + ), + None, + ) + + def with_inference_component_hint(error_message: str) -> str: if "Inference Component Name header is required" in error_message: return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`" diff --git a/litellm/llms/sagemaker/completion/transformation.py b/litellm/llms/sagemaker/completion/transformation.py index a69846c816f..5f6bef4246e 100644 --- a/litellm/llms/sagemaker/completion/transformation.py +++ b/litellm/llms/sagemaker/completion/transformation.py @@ -23,7 +23,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage from litellm.utils import token_counter -from ..common_utils import SagemakerError +from ..common_utils import STR_KEYED_MAPPING, SagemakerError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -153,7 +153,9 @@ class SagemakerConfig(BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - inference_params: Final = optional_params.copy() + inference_params: Final = { + key: value for key, value in STR_KEYED_MAPPING.validate_python(optional_params).items() if key != "model_id" + } stream: Final = inference_params.pop("stream", False) data: Final[dict] = {"parameters": inference_params} if stream is True: diff --git a/litellm/llms/subscription_accounts.py b/litellm/llms/subscription_accounts.py new file mode 100644 index 00000000000..79c186ca124 --- /dev/null +++ b/litellm/llms/subscription_accounts.py @@ -0,0 +1,37 @@ +""" +Providers whose requests are covered by a flat subscription, each with the way to read +the account id of the login a deployment is signed into. +""" + +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import Final + +from litellm.llms.chatgpt.authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file +from litellm.secret_managers.main import UnresolvableSecretReference, resolve_local_oidc_reference +from litellm.utils import with_named_credential + + +def _chatgpt_account_id(litellm_params: Mapping[str, object]) -> str | None: + params: Final = with_named_credential(litellm_params) + auth_file: Final = get_chatgpt_auth_file(params) + if auth_file: + return get_cached_authenticator(auth_file).read_account_id() + api_key: Final = params.get("api_key") + if not isinstance(api_key, str) or not api_key: + return Authenticator().read_account_id() + match resolve_local_oidc_reference(api_key): + case UnresolvableSecretReference(): + return None + case str() as access_token: + return Authenticator().read_account_id(access_token) + + +SUBSCRIPTION_ACCOUNT_ID_RESOLVERS: Final[Mapping[str, Callable[[Mapping[str, object]], str | None]]] = MappingProxyType( + {"chatgpt": _chatgpt_account_id} +) + + +def resolve_subscription_account_id(custom_llm_provider: str, litellm_params: Mapping[str, object]) -> str | None: + resolver: Final = SUBSCRIPTION_ACCOUNT_ID_RESOLVERS.get(custom_llm_provider) + return None if resolver is None else resolver(litellm_params) diff --git a/litellm/llms/tencent/chat/transformation.py b/litellm/llms/tencent/chat/transformation.py index 1f0bcd218d6..23a6ccd52c6 100644 --- a/litellm/llms/tencent/chat/transformation.py +++ b/litellm/llms/tencent/chat/transformation.py @@ -9,12 +9,15 @@ from typing import Final, TypedDict from typing_extensions import ReadOnly import litellm +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.secret_managers.main import get_secret_str from litellm.utils import supports_reasoning from ...openai.chat.gpt_transformation import OpenAIGPTConfig +_ADAPTIVE_BUDGET_REASON: Final = "it is not an integer, so litellm sends adaptive thinking with no budget." + class ThinkingPayload(TypedDict, total=False): """Tencent TokenHub `thinking` object. @@ -102,6 +105,10 @@ class TencentChatConfig(OpenAIGPTConfig): if isinstance(budget, int): coerced_with_budget: Final[ThinkingPayload] = {"type": "adaptive", "budget_tokens": budget} return coerced_with_budget + if budget is not None: + warn_dropped_reasoning( + "thinking.budget_tokens", budget, model, _ADAPTIVE_BUDGET_REASON, drop_params_gated=False + ) coerced: Final[ThinkingPayload] = {"type": "adaptive"} return coerced diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index c506192f253..1ef5267df14 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc from litellm.litellm_core_utils.prompt_templates.factory import ( encode_tool_call_id_with_signature, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -201,6 +202,13 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr _SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map" +_NON_INTEGER_BUDGET_REASON: Final = ( + "Gemini takes budget_tokens only as an integer, so it uses its default thinking budget." +) +_LEVEL_ONLY_BUDGET_REASON: Final = ( + "Gemini 3 and later set reasoning by thinking level, not budget, so litellm does not send the budget. " + "Send reasoning_effort to pick a level." +) _sampling_deprecation_lock: Final = threading.Lock() @@ -1466,6 +1474,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): params["includeThoughts"] = False else: params["includeThoughts"] = sends_thoughts + if thinking_budget is not None: + warn_dropped_reasoning( + "thinking.budget_tokens", + thinking_budget, + model, + _LEVEL_ONLY_BUDGET_REASON, + drop_params_gated=False, + ) # Follow provider defaults unless explicitly opted into legacy behavior. if litellm.enable_gemini_default_thinking_level_low is True: params["thinkingLevel"] = ( @@ -1478,8 +1494,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # For older Gemini models, use thinkingBudget if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget): params["includeThoughts"] = sends_thoughts - if thinking_budget is not None and isinstance(thinking_budget, int): + if isinstance(thinking_budget, int): params["thinkingBudget"] = thinking_budget + elif thinking_budget is not None: + warn_dropped_reasoning( + "thinking.budget_tokens", + thinking_budget, + model, + _NON_INTEGER_BUDGET_REASON, + drop_params_gated=False, + ) return params diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 13f547b3b92..7635cd32a06 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -39,8 +39,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): (they require async I/O) and injected at construction time. """ - def __init__(self, access_token: str, project: str, location: str) -> None: - super().__init__() + def __init__(self, access_token: str, project: str, location: str, drop_params: bool = False) -> None: + super().__init__(drop_params=drop_params) self._access_token = access_token self._project = project self._location = location @@ -51,6 +51,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): def _rejects_vad_tuning_with_detection_off(self) -> bool: return False + def _rejects_undocumented_voices(self, model: str) -> bool: + return True + # ------------------------------------------------------------------ # URL # ------------------------------------------------------------------ @@ -163,7 +166,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): # settings would be silently dropped because ``map_openai_params`` only # recognises the flat OpenAI-beta key names. session_params = self._normalize_session_payload_for_mapping(session_params) - setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params) + setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params, model=model) # Use full Vertex AI model path setup_config["model"] = self._vertex_model_path(model) @@ -177,15 +180,16 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): # ``silence_duration_ms``). ``map_automatic_turn_detection`` sets # ``disabled=True`` whenever ``create_response`` is absent or # ``False``. Force ``disabled=False`` only when the client did - # not explicitly request ``create_response: False`` — that path + # not explicitly request ``create_response: False`` or + # ``turn_detection: null`` — the first of those paths # is how transcription guardrails suppress automatic responses, # and overriding it here would silently bypass the guardrail. # Vertex Live has no "VAD on, no auto-response" mode, so callers # that need that behaviour must accept that VAD is off. client_turn_detection: Final = session_params.get("turn_detection") client_disabled_auto_response: Final = ( - isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False - ) + "turn_detection" in session_params and client_turn_detection is None + ) or (isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False) realtime_input_config: Final = setup_config.setdefault("realtimeInputConfig", {}) automatic_detection: Final = realtime_input_config.setdefault("automaticActivityDetection", {}) if not client_disabled_auto_response: @@ -261,4 +265,5 @@ def vertex_realtime_config( access_token=access_token, project=project, location=VertexBase.get_vertex_region(vertex_region=location, model=model), + drop_params=drop_params, ) diff --git a/litellm/llms/xai/responses/transformation.py b/litellm/llms/xai/responses/transformation.py index b6ea516fb79..c907fde9926 100644 --- a/litellm/llms/xai/responses/transformation.py +++ b/litellm/llms/xai/responses/transformation.py @@ -10,6 +10,10 @@ from litellm._logging import verbose_logger from litellm.constants import XAI_API_BASE from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.responses.transformation import ( + STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, + ReasoningFieldValues, +) from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd from litellm.secret_managers.main import get_secret_str @@ -290,3 +294,9 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def supports_native_websocket(self) -> bool: """XAI does not support native WebSocket for Responses API""" return False + + def reasoning_field_values(self) -> ReasoningFieldValues: + return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES + + def ignores_unsupported_reasoning_fields(self) -> bool: + return True diff --git a/litellm/main.py b/litellm/main.py index d469aa629fe..5346741e4d1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -681,7 +681,7 @@ async def acompletion( - The `completion` function is called using `run_in_executor` to execute synchronously in the event loop. - If `stream` is True, the function returns an async generator that yields completion lines. """ - fallbacks = kwargs.get("fallbacks", None) + fallbacks: Final = kwargs.get("fallbacks", None) or litellm.model_fallbacks mock_timeout = kwargs.get("mock_timeout", None) if mock_timeout is True: @@ -697,6 +697,7 @@ async def acompletion( loop: Final = asyncio.get_event_loop() custom_llm_provider = kwargs.get("custom_llm_provider", None) _ = _resolve_control_options(kwargs, model) + caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs}) # cast-ok: acompletion takes untyped kwargs ## PROMPT MANAGEMENT HOOKS ## ######################################################### @@ -715,15 +716,34 @@ async def acompletion( custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs - api_base=kwargs.get("api_base") or base_url, + api_base=base_url or kwargs.get("api_base"), ) - if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( - litellm_logging_obj.should_run_prompt_management_hooks( - prompt_id=kwargs.get("prompt_id", None), + prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None)) # cast-ok: read from untyped kwargs + prompt_management_logger: Final = ( + litellm_logging_obj.get_custom_logger_for_prompt_management( + model=model, + non_default_params=kwargs, + tools=tools, + prompt_id=prompt_id, + dynamic_callback_params=litellm_logging_obj.standard_callback_dynamic_params, + ) + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) + and litellm_logging_obj.should_run_prompt_management_hooks( + prompt_id=prompt_id, non_default_params=kwargs, tools=tools, ) + else None + ) + each_fallback_writes_its_own_cache_dialect: Final = fallbacks is not None and isinstance( + prompt_management_logger, AnthropicCacheControlHook + ) + + if ( + isinstance(litellm_logging_obj, LiteLLMLoggingObj) + and prompt_management_logger is not None + and not each_fallback_writes_its_own_cache_dialect ): ( model, @@ -733,8 +753,9 @@ async def acompletion( model=model, messages=messages, non_default_params=kwargs, - prompt_id=kwargs.get("prompt_id", None), + prompt_id=prompt_id, prompt_variables=kwargs.get("prompt_variables", None), + prompt_management_logger=prompt_management_logger, tools=tools, prompt_label=kwargs.get("prompt_label", None), prompt_version=kwargs.get("prompt_version", None), @@ -826,9 +847,10 @@ async def acompletion( litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None), ) - fallbacks = fallbacks or litellm.model_fallbacks if fallbacks is not None: - response = await async_completion_with_fallbacks(**completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs}) + response = await async_completion_with_fallbacks( + **completion_kwargs, kwargs={"fallbacks": fallbacks, **caller_kwargs} + ) if response is None: raise Exception( "No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details." @@ -5765,7 +5787,7 @@ def completion( custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs - api_base=kwargs.get("api_base") or base_url, + api_base=base_url or kwargs.get("api_base"), ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -6199,6 +6221,7 @@ def completion( PROVIDER_AFFINITY_HEADER_KWARG_KEY, "fireworks_forward_user_id", "responses_api_version", + "allowed_openai_params", ) if key in kwargs }, @@ -6266,6 +6289,9 @@ def completion( # reasoningSummary/reasoning_summary without tools (AI SDK) that the first # (early) check doesn't cover. _reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params) + typed_deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) if responses_api_model_info.get("mode") != "responses": responses_api_model_info, model = responses_api_bridge_check( model=model, @@ -6275,9 +6301,7 @@ def completion( reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")), reasoning_summary=_reasoning_summary_for_bridge, api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), + deployment_model_info=typed_deployment_model_info, ) # Use base_model (the true underlying model) for Azure model-type @@ -6289,19 +6313,28 @@ def completion( optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) - if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict): - from litellm.responses.utils import openai_bridge_reasoning + if isinstance(resolved_reasoning_effort, dict): + from litellm.responses.utils import ( + RefusedReasoningFields, + bridge_reasoning, + raise_refused_reasoning_fields, + ) - optional_params["reasoning_effort"] = openai_bridge_reasoning( + bridged_reasoning: Final = bridge_reasoning( requested=resolved_reasoning_effort, effort_kept="reasoning_effort" in optional_params, summary_alias=rs_val, model=model, + base_model=base_model if isinstance(base_model, str) else None, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + deployment_model_info=typed_deployment_model_info, drop_params=drop_unsupported_reasoning, additional_drop_params=routing_kwargs.get("additional_drop_params"), ) - elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort: - optional_params["reasoning_effort"] = resolved_reasoning_effort + if isinstance(bridged_reasoning, RefusedReasoningFields): + raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider) + optional_params["reasoning_effort"] = bridged_reasoning elif rs_val is not None: eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort) if isinstance(eff, dict): diff --git a/litellm/models/__init__.py b/litellm/models/__init__.py index 50eb6f8af4f..bb47bd52c2d 100644 --- a/litellm/models/__init__.py +++ b/litellm/models/__init__.py @@ -30,6 +30,7 @@ from litellm.models.organization_membership import LiteLLM_OrganizationMembershi from litellm.models.project import LiteLLM_ProjectTable from litellm.models.skills import LiteLLM_SkillsTable from litellm.models.spend_logs import LiteLLM_ErrorLogs, LiteLLM_SpendLogs +from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable from litellm.models.tag import LiteLLM_TagTable from litellm.models.team import LiteLLM_TeamTable from litellm.models.team_membership import LiteLLM_TeamMembership @@ -59,6 +60,7 @@ __all__ = [ "LiteLLM_ProxyModelTable", "LiteLLM_SkillsTable", "LiteLLM_SpendLogs", + "LiteLLM_SubscriptionAccountTable", "LiteLLM_TagTable", "LiteLLM_TeamMemberTable", "LiteLLM_TeamMembership", diff --git a/litellm/models/subscription_account.py b/litellm/models/subscription_account.py new file mode 100644 index 00000000000..36eaa0ffc37 --- /dev/null +++ b/litellm/models/subscription_account.py @@ -0,0 +1,28 @@ +""" +Subscription account table model. + +Canonical definition for ``litellm_subscriptionaccounttable``: the operator-entered +recurring fee of a provider account whose requests are covered by a flat subscription. +""" + +from datetime import datetime + +from pydantic import ConfigDict + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + + +class LiteLLM_SubscriptionAccountTable(LiteLLMPydanticObjectBase): + model_config = ConfigDict(protected_namespaces=(), frozen=True) + + subscription_account_id: str + custom_llm_provider: str + account_id: str + label: str | None = None + monthly_fee: float + currency: str + billing_period_start: str + created_at: datetime + created_by: str + updated_at: datetime + updated_by: str diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index bc64a4c7ef0..c76b9be7c33 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -1648,7 +1648,7 @@ }, "/access_group/{access_group}/update": { "put": { - "description": "Update an access group's model names.\n\nThis will:\n1. Remove the access group from all current deployments\n2. Add the access group to all deployments for the new model_names list\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found", + "description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit holds the group or one of its models for 30 s", "operationId": "update_access_group_access_group__access_group__update_put", "parameters": [ { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0a09fe4a855..f165355da5a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -667,6 +667,10 @@ class LiteLLMRoutes(enum.Enum): "/team/spend/by_user", # gateway request counts (SGR); deployment-wide, admin-only "/gateway/daily/activity", + "/subscription_accounts/usage", + "/subscription_accounts/new", + "/subscription_accounts/update", + "/subscription_accounts/delete", # model "/model/new", "/model/update", diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index b8d9530d2eb..f80296f4bd7 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -84,6 +84,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.agent_endpoints.auth.agent_access_groups import ( + AgentAccessGroupCeiling, CeilingResolver, resolve_agent_access_group_ceiling, ) @@ -1684,6 +1685,7 @@ def get_actual_routes(allowed_routes: list) -> list: KEY_END_USER_BUDGET_ID_METADATA_FIELD: Final = "end_user_budget_id" _KEY_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +_OBJECTS_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) def get_key_end_user_budget_id(key_metadata: Mapping[str, object] | None) -> str | None: @@ -5085,6 +5087,36 @@ async def can_agent_call_model( ) +async def _agent_access_group_ceilings( + valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver +) -> tuple[AgentAccessGroupCeiling, ...]: + if not valid_token.agent_id: + return () + + from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings + + managed: Final = managed_agent_policy(valid_token) + if managed is not None: + return await resolve_managed_agent_ceilings(managed) + unmanaged: Final = await resolve_ceiling(valid_token.agent_id) + return (unmanaged,) if unmanaged is not None else () + + +def _managed_agent_model_grants(valid_token: UserAPIKeyAuth) -> frozenset[str]: + managed: Final = managed_agent_policy(valid_token) + granted: Final = (managed.object_permission or {}).get("models") if managed is not None else None + if not isinstance(granted, (list, tuple)): + return frozenset() + return frozenset(name for name in _OBJECTS_ADAPTER.validate_python(granted) if isinstance(name, str)) + + +async def agent_model_grants( + valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling +) -> frozenset[str]: + ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling) + return _managed_agent_model_grants(valid_token).union(*(ceiling.models for ceiling in ceilings)) + + async def _check_agent_access_group_model_access( model: str | list[str] | None, # mutable-ok: _can_object_call_model and the client message helper take list[str] valid_token: UserAPIKeyAuth | None, @@ -5092,20 +5124,9 @@ async def _check_agent_access_group_model_access( resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling, ) -> Literal[True]: """Attached groups naming no model deny every model; the empty allowlist in ``_can_object_call_model`` allows.""" - if not model or valid_token is None or not valid_token.agent_id: + if not model or valid_token is None: return True - - from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings - - managed: Final = managed_agent_policy(valid_token) - unmanaged: Final = await resolve_ceiling(valid_token.agent_id) if managed is None else None - ceilings: Final = ( - await resolve_managed_agent_ceilings(managed) - if managed is not None - else (unmanaged,) - if unmanaged is not None - else () - ) + ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling) dispatched: Final = _resolve_team_alias( model, team_model_aliases_for_auth_check(valid_token), valid_token.team_id, llm_router ) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 7bd7b1226b1..9e11179c686 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -2199,15 +2199,19 @@ class JWTAuthManager: # not mistaken for claimless and left with stale DB memberships the fallback # could later attribute. With the flag off, keep the upstream plural-only # reconciliation so existing deployments are unchanged. - jwt_team_ids: Final = set( - jwt_handler.get_all_jwt_team_ids(jwt_valid_token) - if jwt_handler.litellm_jwtauth.fallback_to_db_teams - else jwt_handler.get_team_ids_from_jwt(jwt_valid_token) + claim_team_ids: Final = tuple( + dict.fromkeys( + jwt_handler.get_all_jwt_team_ids(jwt_valid_token) + if jwt_handler.litellm_jwtauth.fallback_to_db_teams + else jwt_handler.get_team_ids_from_jwt(jwt_valid_token) + ) ) - existing_teams: Final = set(user_object.teams or []) - teams_to_add: Final = jwt_team_ids - existing_teams + jwt_team_ids: Final = frozenset(claim_team_ids) + existing_team_order: Final = tuple(dict.fromkeys(user_object.teams or [])) + existing_teams: Final = frozenset(existing_team_order) + teams_to_add: Final = tuple(team_id for team_id in claim_team_ids if team_id not in existing_teams) preserve_db_teams_without_claims: Final = jwt_handler.litellm_jwtauth.fallback_to_db_teams and not jwt_team_ids - teams_to_remove: Final = set() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids + teams_to_remove: Final = frozenset() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids if teams_to_add or teams_to_remove: from litellm.proxy.management_endpoints.scim.scim_v2 import ( patch_team_membership, @@ -2218,7 +2222,10 @@ class JWTAuthManager: teams_ids_to_add_user_to=list(teams_to_add), teams_ids_to_remove_user_from=list(teams_to_remove), ) - user_object.teams = list(jwt_team_ids) + user_object.teams = [ + *(team_id for team_id in existing_team_order if team_id not in teams_to_remove), + *teams_to_add, + ] if user_api_key_cache is not None: await user_api_key_cache.async_set_cache( key=user_object.user_id, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index a79d8fe29db..762f552dadc 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -60,6 +60,7 @@ from litellm.litellm_core_utils.bug_report import ( strip_bug_report_notice, ) from litellm.litellm_core_utils.core_helpers import ( + finite_cost, get_or_create_metadata_bucket, independent_snapshot, is_expected_client_error, @@ -140,6 +141,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports _check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export check_and_merge_model_level_guardrails, ) +from litellm.responses.mcp.parked_stream_dispatch import in_flight_round_biller, parked_success from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task from litellm.router import Router from litellm.router_api import RoutingContext @@ -535,6 +537,16 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons return await _dispatch_disconnect_billing_event(logging_obj, partial_response) +async def _bill_in_flight_round_on_disconnect(request_data: Mapping[str, object]) -> None: + bill_in_flight_round: Final = in_flight_round_biller(request_data.get("litellm_logging_obj")) + if bill_in_flight_round is None: + return + try: + await bill_in_flight_round() + except Exception as e: # noqa: BLE001 # disconnect billing is best-effort; never break stream teardown + verbose_proxy_logger.debug("Failed to bill the in-flight round after client disconnect: %s", e) + + async def _dispatch_disconnect_billing_event(logging_obj: LiteLLMLoggingObj, partial_response: object) -> bool: try: await logging_obj.dispatch_success_handlers( @@ -1801,14 +1813,16 @@ def _get_cost_breakdown_from_logging_obj( def _header_value(value: float | None) -> str | None: - return str(value) if value is not None else None + return str(value) if finite_cost(value) is not None else None def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float | str: if hidden_params.get("cache_hit") is True: return 0.0 response_cost: Final = hidden_params.get("response_cost") - return response_cost if isinstance(response_cost, (int, float, str)) and response_cost else "" + if isinstance(response_cost, str): + return response_cost + return response_cost if isinstance(response_cost, int | float) and finite_cost(response_cost) else "" def cost_headers( @@ -2083,6 +2097,13 @@ class _ClientStreamFailureReport: ) +def _metadata_logged_on(logged_on: LiteLLMLoggingObj) -> Mapping[str, object]: + logged_params: Final = object_mapping(logged_on.model_call_details.get("litellm_params")) or MappingProxyType({}) + return MappingProxyType( + {key: logged_params[key] for key in ("metadata", "litellm_metadata") if key in logged_params} + ) + + class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict, served_model: str | None = None): self.data = data @@ -4151,11 +4172,21 @@ class ProxyBaseLLMRequestProcessing: _captured_user_api_key_dict: Final = user_api_key_dict _captured_logging_obj: Final = logging_obj - async def _on_deferred_stream_complete(assembled_response: object, cache_hit: object) -> None: + async def _on_deferred_stream_complete( + assembled_response: object, cache_hit: object, logged_on: LiteLLMLoggingObj | None = None + ) -> None: await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( - captured_data=_captured_data, + captured_data=_captured_data + if logged_on is None + else { + **_captured_data, + **{ + key: value for key, value in _metadata_logged_on(logged_on).items() if key in _captured_data + }, + "litellm_logging_obj": logged_on, + }, captured_user_api_key_dict=_captured_user_api_key_dict, - captured_logging_obj=_captured_logging_obj, + captured_logging_obj=_captured_logging_obj if logged_on is None else logged_on, assembled_response=assembled_response, cache_hit=cache_hit, ) @@ -4180,14 +4211,8 @@ class ProxyBaseLLMRequestProcessing: GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine) case (logging_coroutine,) if asyncio.iscoroutine(logging_coroutine): GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine) - case (assembled_response, cache_hit): - await _as_success_dispatcher(_captured_native_logging_obj).dispatch_success_handlers( - assembled_response, - cache_hit=cache_hit, - start_time=None, - end_time=None, - prefer_async_handlers=True, - ) + case _ if (success := parked_success(args)) is not None: + await success.dispatch(_captured_native_logging_obj) case _: verbose_proxy_logger.error( "Deferred stream logging dropped: unexpected stored args shape %s", @@ -4487,7 +4512,7 @@ class ProxyBaseLLMRequestProcessing: """ if isinstance(chunk, dict): # Use safe_dumps for proper JSON serialization with circular reference detection - chunk_str: Final = safe_dumps(chunk) + chunk_str: Final = safe_dumps(chunk, non_finite_as_null=True) return f"{STREAM_SSE_DATA_PREFIX}{chunk_str}\n\n" else: return chunk @@ -4524,6 +4549,7 @@ class ProxyBaseLLMRequestProcessing: client_disconnected, ) if recorded_client_disconnect: + await _bill_in_flight_round_on_disconnect(_REQUEST_FIELDS.validate_python(request_data)) deferred_stream_logging_armed: Final = _deferred_stream_logging_is_armed(request_data) ProxyLogging.fire_deferred_stream_logging(request_data) # A disconnect-time success event (the deferred-guardrail flush @@ -4862,7 +4888,11 @@ class ProxyBaseLLMRequestProcessing: obj, model_name, litellm_logging_obj ) if maybe_modified is not None: - lines[idx] = "data: " + safe_dumps(maybe_modified) + ("\r" if ln.endswith("\r") else "") + lines[idx] = ( + "data: " + + safe_dumps(maybe_modified, non_finite_as_null=True) + + ("\r" if ln.endswith("\r") else "") + ) return "\n".join(lines) return None except Exception: diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py index da74851b2d8..e9aaa0e81e4 100644 --- a/litellm/proxy/common_utils/config_sync_pubsub.py +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -2,9 +2,12 @@ import asyncio import json import random import time -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # untyped prisma/redis boundary needs cast +from itertools import chain +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast # noqa: TID251 # untyped prisma/redis boundary + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.repositories.prisma_protocols import RowT_co, TableActions @@ -106,24 +109,69 @@ _pubsub_capable_client: Final = pubsub_capable_client @dataclass(frozen=True, slots=True) class _ConfigChangeMessage: object_type: str + model_ids: tuple[str, ...] = () -def _config_change_message_json(object_type: str) -> str: - return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type))) +@dataclass(frozen=True, slots=True) +class _PubSubEnvelope: + data: str | bytes -async def publish_config_change(redis_cache: "RedisCache | None", object_type: str) -> None: +class _WrittenModelRow(BaseModel): + model_config = ConfigDict(from_attributes=True, frozen=True) + + model_id: str + + +_CONFIG_CHANGE_MESSAGE: Final = TypeAdapter(_ConfigChangeMessage) +_PUBSUB_ENVELOPE: Final = TypeAdapter(_PubSubEnvelope) +_MODEL_TABLE_NAME: Final = "litellm_proxymodeltable" +_TARGETED_WRITE_ACTION_NAMES: Final[frozenset[str]] = frozenset({"create", "update", "upsert"}) + +PublishConfigChange: TypeAlias = Callable[[str, tuple[str, ...]], Awaitable[None]] + + +def _config_change_message_json(object_type: str, model_ids: tuple[str, ...]) -> str: + return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type, model_ids=model_ids))) + + +def parse_config_change(message: object) -> _ConfigChangeMessage | None: + try: + payload: Final = ( + message if isinstance(message, (str, bytes)) else _PUBSUB_ENVELOPE.validate_python(message).data + ) + return _CONFIG_CHANGE_MESSAGE.validate_json(payload) + except ValidationError: + return None + + +def written_model_ids(object_type: str, action_name: str, result: object) -> tuple[str, ...]: + if object_type != _MODEL_TABLE_NAME or action_name not in _TARGETED_WRITE_ACTION_NAMES: + return () + try: + return (_WrittenModelRow.model_validate(result).model_id,) + except ValidationError: + return () + + +async def publish_config_change( + redis_cache: "RedisCache | None", object_type: str, model_ids: tuple[str, ...] = () +) -> None: if redis_cache is None: return try: client: Final = pubsub_capable_client(redis_cache) - await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type)) + await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type, model_ids)) except Exception as e: # noqa: BLE001 # best-effort publish; writes must never fail on redis errors verbose_proxy_logger.warning("config sync publish for %s failed: %s", object_type, e) -async def publish_config_change_for_object_type(object_type: str) -> None: - await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type) +async def publish_config_change_for_object_type(object_type: str, model_ids: tuple[str, ...] = ()) -> None: + await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type, model_ids=model_ids) + + +async def publish_model_rows_written(model_ids: tuple[str, ...]) -> None: + await publish_config_change_for_object_type(_MODEL_TABLE_NAME, model_ids) async def publish_config_param_change(param_name: str) -> None: @@ -139,7 +187,7 @@ async def publish_config_param_change(param_name: str) -> None: class _PublishOnWriteActions: __slots__ = ("_actions", "_object_type", "_publish") - def __init__(self, actions: object, object_type: str, publish: Callable[[str], Awaitable[None]]) -> None: + def __init__(self, actions: object, object_type: str, publish: PublishConfigChange) -> None: self._actions = actions self._object_type = object_type self._publish = publish @@ -157,7 +205,7 @@ class _PublishOnWriteActions: **kwargs: object, # kwargs-ok: transparent passthrough to untyped prisma action ) -> object: result: Final = await write_action(*args, **kwargs) - await publish(object_type) + await publish(object_type, written_model_ids(object_type, name, result)) return result return _write_then_publish @@ -166,7 +214,7 @@ class _PublishOnWriteActions: def wrap_table_actions_for_config_sync( actions: "TableActions[RowT_co]", table_name: str, - publish: Callable[[str], Awaitable[None]] = publish_config_change_for_object_type, + publish: PublishConfigChange = publish_config_change_for_object_type, ) -> "TableActions[RowT_co]": if table_name not in _CONFIG_SYNCED_TABLE_NAMES: return actions @@ -174,8 +222,14 @@ def wrap_table_actions_for_config_sync( return cast("TableActions[RowT_co]", wrapped) # cast-ok: dynamic write-through proxy keeps the wrapped row type +def _published_model_ids(messages: tuple[object, ...]) -> tuple[str, ...]: + changes: Final = (parse_config_change(message) for message in messages) + return tuple(dict.fromkeys(chain.from_iterable(change.model_ids for change in changes if change is not None))) + + class ConfigSyncSubscriber: __slots__ = ( + "_apply_model_ids", "_backoff_initial_seconds", "_backoff_max_seconds", "_debounce_seconds", @@ -195,6 +249,7 @@ class ConfigSyncSubscriber: self, redis_cache: "RedisCache", resync_callbacks: tuple[Callable[[], Awaitable[None]], ...], + apply_model_ids: Callable[[tuple[str, ...]], Awaitable[None]] | None = None, debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS, jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS, min_resync_interval_seconds: float = CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS, @@ -206,6 +261,7 @@ class ConfigSyncSubscriber: ) -> None: self._redis_cache = redis_cache self._resync_callbacks = resync_callbacks + self._apply_model_ids = apply_model_ids self._debounce_seconds = debounce_seconds self._jitter_max_seconds = jitter_max_seconds self._min_resync_interval_seconds = min_resync_interval_seconds @@ -266,32 +322,64 @@ class ConfigSyncSubscriber: message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=_POLL_TIMEOUT_SECONDS) if message is None: continue + await self._apply(pubsub, message) await self._debounce_and_resync(pubsub) - async def _debounce_and_resync(self, pubsub: _ConfigSyncPubSub) -> None: - await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)) - await self._wait_for_min_resync_interval() + async def _debounce_and_resync(self, pubsub: ConfigSyncPubSub) -> None: + await self._apply_until( + pubsub, self._monotonic() + self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds) + ) + await self._apply_until(pubsub, self._next_resync_at()) await self._drain_pending(pubsub) await self._run_resync_callbacks() self._last_resync_at = self._monotonic() - async def _wait_for_min_resync_interval(self) -> None: + def _next_resync_at(self) -> float: if self._last_resync_at is None: - return - seconds_until_next_resync = self._min_resync_interval_seconds - (self._monotonic() - self._last_resync_at) - if seconds_until_next_resync <= 0: - return - verbose_proxy_logger.debug( - "config sync resync throttled for %.1fs to cap fleet-wide reload rate", - seconds_until_next_resync, - ) - await self._sleep(seconds_until_next_resync) + return self._monotonic() + next_resync_at: Final = self._last_resync_at + self._min_resync_interval_seconds + throttled_for: Final = next_resync_at - self._monotonic() + if throttled_for > 0: + verbose_proxy_logger.debug( + "config sync resync throttled for %.1fs to cap fleet-wide reload rate", + throttled_for, + ) + return next_resync_at + + async def _apply_until(self, pubsub: ConfigSyncPubSub, deadline: float) -> None: + while True: + remaining = deadline - self._monotonic() + if remaining <= 0: + return + message = await pubsub.get_message( + ignore_subscribe_messages=True, timeout=min(remaining, _POLL_TIMEOUT_SECONDS) + ) + if message is not None: + await self._apply(pubsub, message) @staticmethod async def _drain_pending(pubsub: ConfigSyncPubSub) -> None: while await pubsub.get_message(ignore_subscribe_messages=True, timeout=0) is not None: pass + @staticmethod + async def _ready_messages(pubsub: ConfigSyncPubSub) -> AsyncIterator[object]: + while (message := await pubsub.get_message(ignore_subscribe_messages=True, timeout=0)) is not None: + yield message + + async def _apply(self, pubsub: ConfigSyncPubSub, message: object) -> None: + apply: Final = self._apply_model_ids + if apply is None: + return + ready: Final = (message, *[ready_message async for ready_message in self._ready_messages(pubsub)]) + model_ids: Final = _published_model_ids(ready) + if not model_ids: + return + try: + await apply(model_ids) + except Exception as e: # noqa: BLE001 # the debounced full resync retries what a targeted apply could not + verbose_proxy_logger.warning("config sync targeted apply of %s failed: %s", model_ids, e) + async def _run_resync_callbacks(self) -> None: for callback in self._resync_callbacks: try: diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index ad4c3bd8770..a9424018d9a 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -219,7 +219,7 @@ def caller_alias_maps( def alias_map(aliases: object) -> Mapping[str, str]: try: - entries: Final = _ALIAS_ENTRIES.validate_python(aliases, strict=True) + entries: Final = _ALIAS_ENTRIES.validate_python(aliases) except ValidationError: return _NO_ALIASES return MappingProxyType( @@ -287,6 +287,15 @@ def _dispatch(model_id: str, layers: Sequence[_AliasLayer]) -> tuple[str, bool]: return reduce(_through_layer, layers, (model_id, False)) +def dispatched_model(model_id: str, aliases: CallerAliases) -> str: + target, _ = _dispatch(model_id, _alias_layers(aliases)) + return target + + +def requestable_alias_names(aliases: CallerAliases) -> frozenset[str]: + return frozenset(chain.from_iterable(layer.aliases for layer in _alias_layers(aliases))) + + def _lookup_id(name: str, lookup_by_response: Mapping[str, str], lookup_ids: Container[str]) -> str | None: if name in lookup_by_response: return lookup_by_response[name] diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index 53b2a05d40f..a948c390375 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -28,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( _AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN}) +_HTTP_ERROR_STATUSES: Final = range(status.HTTP_400_BAD_REQUEST, 600) + _ERROR_BODY: Final = TypeAdapter(dict[str, object]) _FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"}) @@ -47,13 +49,26 @@ def attribute_of(value: object, name: str, default: object = None) -> object: def error_status_code(exc: object, default: int) -> int: - """The HTTP status an exception carries as ``status_code`` or, the way ``ProxyException`` - stores it, as a stringified ``code``; ``default`` when it carries neither.""" - carried: Final = attribute_of(exc, "status_code") - if isinstance(carried, int) and not isinstance(carried, bool): + """The 4xx or 5xx HTTP status an exception carries as ``status_code`` (an int or a decimal string) or, + the way ``ProxyException`` stores it, as a stringified ``code``; ``default`` when it carries neither.""" + carried: Final = _http_error_status(attribute_of(exc, "status_code")) + if carried is not None: return carried stringified: Final = attribute_of(exc, "code") - return int(stringified) if isinstance(stringified, str) and stringified.isdecimal() else default + from_code: Final = _http_error_status(stringified) if isinstance(stringified, str) else None + return default if from_code is None else from_code + + +def _http_error_status(value: object) -> int | None: + match value: + case bool(): + return None + case int() if value in _HTTP_ERROR_STATUSES: + return value + case str() if len(value) == 3 and value.isascii() and value.isdigit() and int(value) in _HTTP_ERROR_STATUSES: + return int(value) + case _: + return None def openai_error_type(exc: object, status_code: int) -> str: @@ -105,9 +120,9 @@ def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyExcept code=status.HTTP_503_SERVICE_UNAVAILABLE, ) case HTTPException(status_code=status_code, detail=detail): - return _openai_shaped_proxy_exception(exc, str(detail), status_code) + return openai_shaped_proxy_exception(exc, str(detail), status_code) case _: - return _openai_shaped_proxy_exception( + return openai_shaped_proxy_exception( exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code) ) @@ -176,14 +191,19 @@ def _route_failure_error_type(exc: Exception, raised_status: int, answered_statu return openai_error_type(exc, answered_status) -def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException: +def openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException: answered_status: Final = answered_status_code(message, status_code) - return ProxyException( - message=message, - type=_catch_all_error_type(exc, answered_status), - param=openai_error_param(exc), - code=answered_status, - ) + return _proxy_exception(exc, message, answered_status, _catch_all_error_type(exc, answered_status)) + + +def status_typed_proxy_exception(exc: HTTPException) -> ProxyException: + message: Final = str(exc.detail) + answered_status: Final = answered_status_code(message, exc.status_code) + return _proxy_exception(exc, message, answered_status, openai_error_type(exc, answered_status)) + + +def _proxy_exception(exc: Exception, message: str, answered_status: int, error_type: str) -> ProxyException: + return ProxyException(message=message, type=error_type, param=openai_error_param(exc), code=answered_status) def _catch_all_error_type(exc: Exception, status_code: int) -> str: diff --git a/litellm/proxy/common_utils/registry_read_through.py b/litellm/proxy/common_utils/registry_read_through.py index 22df11429c0..487c6feb68d 100644 --- a/litellm/proxy/common_utils/registry_read_through.py +++ b/litellm/proxy/common_utils/registry_read_through.py @@ -15,12 +15,13 @@ or the first routed request on this replica would pick a tier it has not loaded. import asyncio import time from collections.abc import Awaitable, Callable, Mapping, Sequence -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper if TYPE_CHECKING: @@ -33,8 +34,14 @@ if TYPE_CHECKING: ) from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.repositories.prisma_protocols import TableActions from litellm.types.agents import AgentResponse + _ModelTable: TypeAlias = TableActions[prisma_models.LiteLLM_ProxyModelTable] + _ModelRows: TypeAlias = Sequence[prisma_models.LiteLLM_ProxyModelTable] + +ModelLoadOutcome: TypeAlias = Literal["loaded", "missing", "no_router", "auto_router"] + READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0 READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0 READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20 @@ -123,47 +130,114 @@ async def _resync_model_deployments(model_name: str) -> bool: return False prisma_client: Final = proxy_server.prisma_client assert prisma_client is not None - table: Final = ModelRepository(prisma_client).table - name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name} - id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name} - rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter) + rows: Final = await _find_rows_by_name_or_id(model_name, ModelRepository(prisma_client).table) if not rows: return False - router: Final = proxy_server.llm_router - if router is None: - await proxy_server.proxy_config.add_deployment( - prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj - ) - return proxy_server.llm_router is not None - if _holds_an_auto_router(rows): - await proxy_server.proxy_config.add_deployment( - prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj - ) - return _model_is_loaded(model_name) async with proxy_server.MODEL_RECONCILE_LOCK: + outcome: Final = await _install_model_rows(rows, _always) + if outcome in ("no_router", "auto_router"): + await _reload_every_model() + return outcome == "loaded" or _model_is_loaded(model_name) + + +async def resync_model_deployments_by_ids(model_ids: tuple[str, ...]) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient + from litellm.repositories.model_repository import ModelRepository + + if not _db_backed_registries_enabled("models"): + return + prisma_client: Final = proxy_server.prisma_client + assert prisma_client is not None + writer_table: Final = ModelRepository(WriterPinnedClient(prisma_client.db)).table + async with proxy_server.MODEL_RECONCILE_LOCK: + rows: Final = await _find_rows_by_ids(model_ids, writer_table) + _ = await _install_model_rows(_without_auto_routers(rows), _names_an_unloaded_credential) + + +async def _find_rows_by_name_or_id(model_name: str, table: "_ModelTable") -> "_ModelRows": + name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name} + id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name} + return await table.find_many(where=name_filter) or await table.find_many(where=id_filter) + + +async def _find_rows_by_ids(model_ids: tuple[str, ...], table: "_ModelTable") -> "_ModelRows": + from litellm.repositories.chunked_in import find_many_in + + return list(await find_many_in(table, "model_id", model_ids)) + + +async def _install_model_rows( + rows: "_ModelRows", + needs_credentials: Callable[["_ModelRows"], bool], +) -> ModelLoadOutcome: + from litellm.proxy import proxy_server + + router: Final = proxy_server.llm_router + if not rows: + return "missing" + if router is None: + return "no_router" + if _holds_an_auto_router(rows): + return "auto_router" + if needs_credentials(rows): + prisma_client: Final = proxy_server.prisma_client + assert prisma_client is not None await proxy_server.proxy_config.get_credentials(prisma_client=prisma_client) - proxy_server.proxy_config._add_deployment(db_models=rows) - proxy_server.llm_model_list = router.registry().deployments() + proxy_server.proxy_config._add_deployment(db_models=rows) + proxy_server.llm_model_list = router.registry().deployments() + return "loaded" + + +async def _reload_every_model() -> None: + from litellm.proxy import proxy_server + + prisma_client: Final = proxy_server.prisma_client + assert prisma_client is not None + await proxy_server.proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj + ) + + +def _always(rows: "_ModelRows") -> bool: return True -def _holds_an_auto_router(rows: Sequence["prisma_models.LiteLLM_ProxyModelTable"]) -> bool: +def _holds_an_auto_router(rows: "_ModelRows") -> bool: return any(_names_an_auto_router(row.litellm_params) for row in rows) +def _without_auto_routers(rows: "_ModelRows") -> "_ModelRows": + return list(row for row in rows if not _names_an_auto_router(row.litellm_params)) + + +def _names_an_unloaded_credential(rows: "_ModelRows") -> bool: + return any(_names_an_unloaded_credential_param(row.litellm_params) for row in rows) + + _STORED_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) -def _names_an_auto_router(stored_litellm_params: object) -> bool: +def _decrypted_param(stored_litellm_params: object, key: str) -> str | None: try: params: Final = _STORED_LITELLM_PARAMS.validate_python(stored_litellm_params) except ValidationError: - return False - stored_model: Final = params.get("model") - if not isinstance(stored_model, str): - return False - model: Final = decrypt_value_helper(value=stored_model, key="model", return_original_value=True) - return isinstance(model, str) and model.startswith("auto_router/") + return None + stored_value: Final = params.get(key) + if not isinstance(stored_value, str): + return None + value: Final = decrypt_value_helper(value=stored_value, key=key, return_original_value=True) + return value if isinstance(value, str) else None + + +def _names_an_auto_router(stored_litellm_params: object) -> bool: + model: Final = _decrypted_param(stored_litellm_params, "model") + return model is not None and model.startswith("auto_router/") + + +def _names_an_unloaded_credential_param(stored_litellm_params: object) -> bool: + credential_name: Final = _decrypted_param(stored_litellm_params, "litellm_credential_name") + return credential_name is not None and CredentialAccessor.find_credential(credential_name) is None async def _resync_guardrails(guardrail_name: str) -> bool: diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index fe804a82e96..d9fb236632b 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -34,6 +34,7 @@ from litellm.constants import ( DB_SPEND_UPDATE_JOB_NAME, INTERNAL_CALL_ORIGIN_METADATA_KEY, ) +from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.proxy._types import ( @@ -652,7 +653,14 @@ class DBSpendUpdateWriter: end_time=end_time, llm_router=llm_router, ) - payload["spend"] = response_cost or 0.0 + billable_cost: Final = finite_cost(response_cost) + if response_cost is not None and billable_cost is None: + verbose_proxy_logger.warning( + "Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s", + response_cost, + single_log_line(payload["request_id"]), + ) + payload["spend"] = billable_cost or 0.0 if isinstance(payload["startTime"], datetime): payload["startTime"] = payload["startTime"].isoformat() if isinstance(payload["endTime"], datetime): @@ -702,7 +710,7 @@ class DBSpendUpdateWriter: spawn_logging_task( asyncio.get_running_loop(), self._batch_database_updates( - response_cost=response_cost, + response_cost=billable_cost, user_id=user_id, hashed_token=hashed_token, team_id=team_id, diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 3a4375ac54a..459f95972ce 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -8,7 +8,7 @@ import asyncio import json from collections.abc import Mapping, Sequence from datetime import datetime -from functools import reduce +from functools import partial, reduce from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast @@ -109,6 +109,16 @@ _ValueT = TypeVar("_ValueT") _DECODED_JSON: Final = TypeAdapter(object) +def _zero_for_non_finite_constant(constant: str) -> float: + verbose_proxy_logger.warning( + "Redis spend buffer - reading a parked %s spend as 0, the spend it held is lost", constant + ) + return 0.0 + + +_load_buffered_transaction: Final = partial(json.loads, parse_constant=_zero_for_non_finite_constant) + + def _spend_log_json_default(value: object) -> str: return value.isoformat() if isinstance(value, datetime) else str(value) @@ -118,7 +128,7 @@ def _encode_spend_log_row(row: SpendLogRow) -> str: def _decode_spend_log_row(encoded: str) -> dict[str, object] | None: - decoded: Final = _DECODED_JSON.validate_python(json.loads(encoded)) + decoded: Final = _DECODED_JSON.validate_python(_load_buffered_transaction(encoded)) return decoded if isinstance(decoded, dict) else None @@ -825,13 +835,13 @@ class RedisUpdateBuffer: if slot_result is None: daily_results.append(None) else: - list_of_daily = [json.loads(t) for t in slot_result] + list_of_daily = [_load_buffered_transaction(t) for t in slot_result] aggregated = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(list_of_daily) daily_results.append(aggregated) window_spend: Final = ( WindowSpendUpdateQueue.get_aggregated_window_spend_transactions( - tuple(json.loads(transaction) for transaction in raw_results[6]) + tuple(_load_buffered_transaction(transaction) for transaction in raw_results[6]) ) if raw_results[6] is not None else None @@ -880,7 +890,7 @@ class RedisUpdateBuffer: ) if list_of_transactions is None: return None - return [json.loads(transaction) for transaction in list_of_transactions] + return [_load_buffered_transaction(transaction) for transaction in list_of_transactions] async def get_all_daily_spend_update_transactions_from_redis_buffer( self, @@ -1010,9 +1020,9 @@ class RedisUpdateBuffer: Parses the list of transactions from Redis """ if isinstance(list_of_transactions, list): - return [json.loads(transaction) for transaction in list_of_transactions] + return [_load_buffered_transaction(transaction) for transaction in list_of_transactions] else: - return [json.loads(list_of_transactions)] + return [_load_buffered_transaction(list_of_transactions)] @staticmethod def _combine_list_of_transactions( diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index a5822bfb0e3..363b9162e00 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -22,6 +22,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX +from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception from litellm.proxy.common_utils.path_utils import is_within, safe_join from litellm.proxy.guardrails.content_filter_data import CATEGORIES_DIR, DATA_ROOTS, category_dirs, find_category_file from litellm.proxy.guardrails.guardrail_hooks.custom_code.bounded_execution import ( @@ -2533,7 +2534,7 @@ async def apply_guardrail( e = transformed_exception except Exception: verbose_proxy_logger.exception("apply_guardrail: post_call_failure_hook failed") - raise handle_exception_on_proxy(e) + raise handle_exception_on_proxy(status_typed_proxy_exception(e) if isinstance(e, HTTPException) else e) # Success logging outside except so a hook error never triggers failure handlers. response = await _emit_guardrail_success_logs( diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 73c2fc35492..20d920f1104 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -32,7 +32,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import ( classify_strategy_router_model, strategy_router_dependencies, ) -from litellm.router_utils.common_utils import is_model_capability_field +from litellm.router_utils.common_utils import deployment_model_id_pin, is_model_capability_field from litellm.types.utils import secret_bearing_wif_litellm_params, server_owned_wif_litellm_params # Provider routing and server-owned federation or OAuth token-exchange fields. Allowed for proxy @@ -855,6 +855,8 @@ def update_litellm_params_for_health_check( `deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model` in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept, and `modalities` set there without "audio" turns the audio fields off + - sends a SageMaker deployment's `X-Amzn-SageMaker-Inference-Component` extra header as `model_id` when the + deployment sets none, so legacy `sagemaker/` probes reach that component the way routed requests do """ incoming_model: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model") base_model_owner: Final = deployment_model or (incoming_model if isinstance(incoming_model, str) else None) @@ -935,9 +937,11 @@ def update_litellm_params_for_health_check( audio_output_params: Final = _health_check_audio_output_params( _MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner ) + model_id_pin: Final = deployment_model_id_pin(probe_params) + model_id_params: Final = {} if model_id_pin is None else {"model_id": model_id_pin} return { key: value - for key, value in {**audio_output_params, **probe_params}.items() + for key, value in {**audio_output_params, **model_id_params, **probe_params}.items() if not is_model_capability_field(key) } diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 59c63a18f0f..8545e0609db 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -7,13 +7,15 @@ import secrets import time import traceback from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from itertools import chain, groupby from types import MappingProxyType from typing import Final, Literal, TypedDict import fastapi from fastapi import APIRouter, Depends, HTTPException, Request, Response, status -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from typing_extensions import ReadOnly import litellm @@ -31,7 +33,9 @@ from litellm.proxy._types import ( CallInfo, EnterpriseLicenseData, Litellm_EntityType, + LiteLLM_UserTable, LitellmUserRoles, + ModelAccessDeniedProxyException, ProxyException, SpecialModelNames, UserAPIKeyAuth, @@ -39,13 +43,28 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_checks import ( _resolve_key_models_for_auth_check, # pyright: ignore[reportPrivateUsage] # the auth layer's sentinel resolution, reused so /health scopes exactly like a request + agent_model_grants, + can_agent_call_model, + can_key_call_resolved_model, + can_user_call_model, + get_user_object, + key_model_aliases_for_auth_check, + model_matches_any_wildcard_pattern_in_list, + team_model_aliases_for_auth_check, +) +from litellm.proxy.auth.auth_exception_handler import ( + _as_proxy_exception, # pyright: ignore[reportPrivateUsage] # one auth-failure mapping, shared with user_api_key_auth ) from litellm.proxy.auth.auth_utils import ( _BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check ) -from litellm.proxy.auth.model_checks import get_key_models from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils import http_parsing_utils +from litellm.proxy.common_utils.model_listing_utils import ( + CallerAliases, + dispatched_model, + requestable_alias_names, +) from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler @@ -79,6 +98,7 @@ from litellm.router_utils.clientside_credential_handler import ( _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path clientside_credential_keys, ) +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.secret_managers.main import get_secret_bool from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields from litellm.types.router import reject_server_owned_wif_fields @@ -253,6 +273,28 @@ def get_callback_identifier(callback): router: Final = APIRouter() _OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_OBJECTS: Final = TypeAdapter(tuple[object, ...]) +_DEPLOYMENTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +class _ModelGrants(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + models: tuple[object, ...] = () + + +class _ModelInfoAccessGroups(BaseModel): + model_config = ConfigDict(frozen=True) + + access_groups: tuple[object, ...] + + +class _DeploymentAccessGroups(BaseModel): + model_config = ConfigDict(frozen=True) + + model_info: _ModelInfoAccessGroups + + services = ( Literal[ "slack_budget_alerts", @@ -1045,42 +1087,339 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict: return out -def _health_accessible_model_names( - user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None -) -> frozenset[str] | None: - """Model names the caller may health-check, or None when the key is unrestricted.""" +def _key_models_unrestricted(user_api_key_dict: UserAPIKeyAuth) -> bool: granted_models: Final = _resolve_key_models_for_auth_check(user_api_key_dict) - if not granted_models or SpecialModelNames.all_proxy_models.value in granted_models: + return not granted_models or SpecialModelNames.all_proxy_models.value in granted_models + + +def _own_team_names(deployment: object, team_id: str | None) -> tuple[str, str] | None: + match deployment: + case { + "model_name": str(internal_name), + "model_info": {"team_id": str(owner_team_id), "team_public_model_name": str(public_name)}, + } if owner_team_id == team_id: + return (internal_name, public_name) + case _: + return None + + +def _names_caller_may_request(deployment: object, team_id: str | None) -> frozenset[str]: + team_names: Final = _own_team_names(deployment, team_id) + if team_names is not None: + return frozenset(team_names) + match deployment: + case {"model_name": str(name)}: + return frozenset((name,)) + case _: + return frozenset() + + +def _public_names_by_internal_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, str]: + team_names: Final = (_own_team_names(deployment, team_id) for deployment in llm_model_list) + return MappingProxyType(dict(names for names in team_names if names is not None)) + + +async def _key_owner(user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_UserTable | None: + if user_api_key_dict.team_id is not None or user_api_key_dict.user_id is None: return None + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + try: + return await get_user_object( + user_id=user_api_key_dict.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=user_api_key_dict.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception: # noqa: BLE001 # auth reads an unloadable user as no user limit, so /health does too + return None + + +async def _key_may_call_model( + model_name: str, + user_api_key_dict: UserAPIKeyAuth, + owner: LiteLLM_UserTable | None, + llm_router: Router | None, +) -> bool: + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + await asyncio.sleep(0) + try: + await can_key_call_resolved_model( + model=model_name, llm_model_list=None, valid_token=user_api_key_dict, llm_router=llm_router + ) + await can_agent_call_model( + model=model_name, + valid_token=user_api_key_dict, + llm_router=llm_router, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + await can_user_call_model( + model=model_name, + llm_router=llm_router, + user_object=owner, + key_model_aliases=key_model_aliases_for_auth_check(user_api_key_dict), + ) + except ModelAccessDeniedProxyException: + return False + except ProxyException: + raise + except Exception as failure: + if isinstance(failure, HTTPException) and failure.status_code == status.HTTP_403_FORBIDDEN: + return False + raise _as_proxy_exception(failure) from failure + return True + + +async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth) -> CallerAliases: + from litellm.proxy.proxy_server import prisma_client, proxy_config, proxy_logging_obj + + router_settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict, prisma_client, proxy_logging_obj + ) + return CallerAliases( + team_model_aliases_for_auth_check(user_api_key_dict), + key_model_aliases_for_auth_check(user_api_key_dict), + router_settings.get("model_group_alias") if router_settings is not None else None, + ) + + +def _model_auth_checks(model_name: str, aliases: CallerAliases) -> str: + return resolve_model_group_alias(aliases.router, model_name) or model_name + + +def _deployment_id(deployment: object) -> str | None: + match deployment: + case {"model_info": {"id": str(model_id)}}: + return model_id + case _: + return None + + +def _granted_names(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]: + grants: Final = ( + *_ModelGrants.model_validate(user_api_key_dict).models, + *_OBJECTS.validate_python(user_api_key_dict.team_models), + ) + return frozenset(name for name in grants if isinstance(name, str)) + + +async def _names_any_limit_grants(user_api_key_dict: UserAPIKeyAuth, owner: LiteLLM_UserTable | None) -> frozenset[str]: + owner_grants: Final = _ModelGrants.model_validate(owner).models if owner is not None else () + return ( + _granted_names(user_api_key_dict) + | frozenset(name for name in owner_grants if isinstance(name, str)) + | await agent_model_grants(user_api_key_dict) + ) + + +def _alias_names_caller_may_request(aliases: CallerAliases, llm_router: Router | None) -> frozenset[str]: + router_aliases: Final = frozenset(llm_router.registry().aliases()) if llm_router is not None else frozenset[str]() + return requestable_alias_names(aliases) | router_aliases + + +def _served_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str: + dispatched: Final = dispatched_model(model_name, aliases) + return llm_router.registry().resolve(dispatched) if llm_router is not None else dispatched + + +def _name_id_pairs(deployment: object, team_id: str | None) -> tuple[tuple[str, str], ...]: + model_id: Final = _deployment_id(deployment) + if model_id is None: + return () + return tuple((name, model_id) for name in _names_caller_may_request(deployment, team_id) | {model_id}) + + +def _ids_by_requested_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, frozenset[str]]: + pairs: Final = sorted(chain.from_iterable(_name_id_pairs(deployment, team_id) for deployment in llm_model_list)) + return MappingProxyType( + {name: frozenset(model_id for _, model_id in group) for name, group in groupby(pairs, key=lambda pair: pair[0])} + ) + + +def _ids_of(deployments: Iterable[object]) -> frozenset[str]: + return frozenset(model_id for model_id in map(_deployment_id, deployments) if model_id is not None) + + +def _access_groups_of(deployment: object) -> frozenset[str]: + match deployment: + case {"model_info": {"access_groups": list()}}: + access_groups: Final = _DeploymentAccessGroups.model_validate(deployment).model_info.access_groups + return frozenset(access_group for access_group in access_groups if isinstance(access_group, str)) + case _: + return frozenset() + + +@dataclass(frozen=True, slots=True) +class _AccessGroupGrants: + granted_names: frozenset[str] + ids_in_granted_groups: frozenset[str] + + def requested_name(self, name: str, public_names: Mapping[str, str], admitted_names: frozenset[str]) -> str: + public_name: Final = public_names.get(name) + if public_name is None or public_name not in admitted_names or self._grants(name): + return name + return public_name + + def _grants(self, name: str) -> bool: + return name in self.granted_names or model_matches_any_wildcard_pattern_in_list(name, tuple(self.granted_names)) + + def narrow(self, serving_ids: frozenset[str], served_names: frozenset[str]) -> frozenset[str]: + in_granted_groups: Final = serving_ids & self.ids_in_granted_groups + granted_directly: Final = not self.granted_names.isdisjoint(served_names) + return serving_ids if granted_directly or not in_granted_groups else in_granted_groups + + +def _access_group_grants(llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth) -> _AccessGroupGrants: + granted: Final = _granted_names(user_api_key_dict) + in_granted_groups: Final = ( + deployment for deployment in llm_model_list if not granted.isdisjoint(_access_groups_of(deployment)) + ) + return _AccessGroupGrants(granted, _ids_of(in_granted_groups)) + + +def _ids_serving_group( + group: str, + ids_by_name: Mapping[str, frozenset[str]], + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router | None, + access_group_grants: _AccessGroupGrants, +) -> frozenset[str]: + named: Final = ids_by_name.get(group) + if named is not None: + return access_group_grants.narrow(named, frozenset((group,))) if llm_router is None: - return frozenset(granted_models) - registry: Final = llm_router.registry() + return frozenset() + team_id: Final = user_api_key_dict.team_id + wildcard_served: Final = llm_router.registry().deployments(group, source="wildcard", team_id=team_id) + served_names: Final = frozenset((group,)).union( + *(_names_caller_may_request(deployment, team_id) for deployment in wildcard_served) + ) + return access_group_grants.narrow(_ids_of(wildcard_served), served_names) + + +async def _names_key_may_request( + names: Iterable[str], + aliases: CallerAliases, + user_api_key_dict: UserAPIKeyAuth, + owner: LiteLLM_UserTable | None, + llm_router: Router | None, +) -> frozenset[str]: return frozenset( - get_key_models( - user_api_key_dict=user_api_key_dict, - proxy_model_list=registry.names(team_id=user_api_key_dict.team_id), - model_access_groups=registry.access_groups(), + [ + name + for name in names + if await _key_may_call_model(_model_auth_checks(name, aliases), user_api_key_dict, owner, llm_router) + ] + ) + + +def _ids_serving( + groups: Iterable[str], + ids_by_name: Mapping[str, frozenset[str]], + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router | None, + access_group_grants: _AccessGroupGrants, +) -> frozenset[str]: + return frozenset[str]().union( + *( + _ids_serving_group(group, ids_by_name, user_api_key_dict, llm_router, access_group_grants) + for group in groups ) ) -def _caller_may_probe_deployment( - deployment: Mapping[str, object], - allowed_models: frozenset[str] | None, +def _shadowed_ids( + llm_model_list: Sequence[object], admitted_names: frozenset[str], reached_ids: frozenset[str], team_id: str | None +) -> frozenset[str]: + ids_of_admitted_names: Final = ( + _deployment_id(deployment) + for deployment in llm_model_list + if not admitted_names.isdisjoint(_names_caller_may_request(deployment, team_id)) + ) + return frozenset(model_id for model_id in ids_of_admitted_names if model_id is not None) - reached_ids + + +@dataclass(frozen=True, slots=True) +class _ReachableDeployments: + model_ids: frozenset[str] + groups: frozenset[str] + + def include(self, deployment: object, team_id: str | None) -> bool: + model_id: Final = _deployment_id(deployment) + if model_id is None: + return not self.groups.isdisjoint(_names_caller_may_request(deployment, team_id)) + return model_id in self.model_ids + + +_NOTHING_REACHABLE: Final = _ReachableDeployments(frozenset(), frozenset()) + + +async def _deployments_key_may_reach( + llm_model_list: Sequence[object], + aliases: CallerAliases, + user_api_key_dict: UserAPIKeyAuth, + owner: LiteLLM_UserTable | None, llm_router: Router | None, +) -> _ReachableDeployments: + candidate_names: Final = (await _names_any_limit_grants(user_api_key_dict, owner)).union( + _alias_names_caller_may_request(aliases, llm_router), + *(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list), + ) + admitted_names: Final = await _names_key_may_request(candidate_names, aliases, user_api_key_dict, owner, llm_router) + access_group_grants: Final = _access_group_grants(llm_model_list, user_api_key_dict) + public_names: Final = _public_names_by_internal_name(llm_model_list, user_api_key_dict.team_id) + groups: Final = frozenset( + _served_group(access_group_grants.requested_name(name, public_names, admitted_names), aliases, llm_router) + for name in admitted_names + ) + ids_by_name: Final = _ids_by_requested_name(llm_model_list, user_api_key_dict.team_id) + reached_ids: Final = _ids_serving(groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants) + admitted_ids: Final = await _names_key_may_request( + _shadowed_ids(llm_model_list, admitted_names, reached_ids, user_api_key_dict.team_id), + aliases, + user_api_key_dict, + owner, + llm_router, + ) + id_groups: Final = frozenset(_served_group(model_id, aliases, llm_router) for model_id in admitted_ids) + return _ReachableDeployments( + reached_ids | _ids_serving(id_groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants), groups + ) + + +def _model_param_target( + model: str, + aliases: CallerAliases, + llm_model_list: Sequence[Mapping[str, object]], team_id: str | None, - caller_is_admin: bool, + llm_router: Router | None, +) -> str: + group: Final = _served_group(model, aliases, llm_router) + if llm_router is None or deployments_targeted_by_name(llm_model_list, group, team_id): + return group + patterns: Final = frozenset( + deployment.get("model_name") + for deployment in llm_router.registry().deployments(group, source="wildcard", team_id=team_id) + ) + match tuple(patterns): + case (str(pattern),): + return pattern + case _: + return group + + +def _caller_may_probe_deployment( + deployment: Mapping[str, object], reachable: _ReachableDeployments, team_id: str | None, caller_is_admin: bool ) -> bool: """Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included.""" if not caller_is_admin and not deployment_usable_by_team(deployment, team_id): return False - if allowed_models is None: - return True - if llm_router is None: - return deployment.get("model_name") in allowed_models - model: Final = dict(deployment) - registry: Final = llm_router.registry() - return any(registry.includes(model, model_name=name, team_id=team_id) for name in allowed_models) + return reachable.include(deployment, team_id) def _resolve_targeted_model_ids( @@ -1137,6 +1476,7 @@ def _filter_health_check_results_by_model_ids(results: dict, allowed_model_ids: async def _perform_health_check_and_save( model_list, target_model, + recorded_model_name: str | None, cli_model, details, prisma_client, @@ -1159,8 +1499,7 @@ async def _perform_health_check_and_save( # Optionally save health check result to database (non-blocking) if prisma_client is not None: - # For CLI model, use cli_model name; for router models, use target_model - model_name_for_db: Final = cli_model if cli_model is not None else target_model + model_name_for_db: Final = cli_model if cli_model is not None else recorded_model_name if model_name_for_db is not None: asyncio.create_task( _save_health_check_to_db( @@ -1290,6 +1629,7 @@ async def health_endpoint( cli_result: Final = await _perform_health_check_and_save( model_list=[], target_model=None, + recorded_model_name=None, cli_model=user_model, details=health_check_details, prisma_client=prisma_client, @@ -1304,15 +1644,39 @@ async def health_endpoint( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": "Model list not initialized"}, ) - allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router) - restrict_to_allowed_models: Final = not is_admin or allowed_models is not None + restrict_to_allowed_models: Final = not is_admin or not _key_models_unrestricted(user_api_key_dict) + aliases: Final = await _caller_aliases(user_api_key_dict) if restrict_to_allowed_models else None + reachable: Final = ( + await _deployments_key_may_reach( + _OBJECTS.validate_python(llm_model_list), + aliases, + user_api_key_dict, + await _key_owner(user_api_key_dict), + llm_router, + ) + if aliases is not None + else _NOTHING_REACHABLE + ) + requested_model: Final = ( + _model_param_target( + model, + aliases, + _DEPLOYMENTS.validate_python(llm_model_list), + user_api_key_dict.team_id, + llm_router, + ) + if model is not None and aliases is not None + else model + ) _llm_model_list: Final = [ m for m in copy.deepcopy(llm_model_list) if not restrict_to_allowed_models - or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin) + or _caller_may_probe_deployment(m, reachable, user_api_key_dict.team_id, is_admin) ] - targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id) + targeted_ids: Final = _resolve_targeted_model_ids( + _llm_model_list, requested_model, model_id, user_api_key_dict.team_id + ) if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, @@ -1370,7 +1734,8 @@ async def health_endpoint( else: router_result: Final = await _perform_health_check_and_save( model_list=_llm_model_list, - target_model=target_model, + target_model=target_model if model_id else requested_model, + recorded_model_name=target_model, cli_model=None, details=health_check_details, prisma_client=prisma_client, diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 9c4a4bc775b..0d333d4fd08 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -16,6 +16,7 @@ from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLE from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( budget_reservation_from_metadata, + finite_cost, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, get_parent_otel_span_from_kwargs, @@ -703,7 +704,7 @@ class _ProxyDBLogger(CustomLogger): raw_cost: Final = _str_keyed_fields( sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload ).get("response_cost") - priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None + priced_cost: Final = finite_cost(raw_cost) response_cost: Final = ( await apply_search_free_tier( kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index c1e541d20a9..9b52f2f8a8d 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -3188,9 +3188,9 @@ def _fallback_lists( ) -> tuple[Sequence[object], ...]: """Every fallback list the router can route this request through: the three router-level lists, the same three keys from the request body (which replace - the router's when present, so the union is the safe superset), and the - key/team ``router_settings.fallbacks`` the proxy prefers over router-level - fallbacks in ``_configured_fallbacks``.""" + the router's when present, so the union is the safe superset), and the same + three keys from key/team ``router_settings``, which the proxy forwards into + the request after this discovery runs.""" lists: Final[list[Sequence[object]]] = [] # mutable-ok: accumulates each fallback list found configured: Final = llm_router.settings() for router_list in ( @@ -3205,11 +3205,12 @@ def _fallback_lists( if isinstance(entries, list): lists.append(cast("Sequence[object]", entries)) # cast-ok: request fallback lists hold mixed entry shapes for settings in (router_settings, team_router_settings): - key_fallbacks: object = settings.get("fallbacks") if settings is not None else None - if isinstance(key_fallbacks, list): - lists.append( - cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes - ) + for key in _REQUEST_FALLBACK_KEYS: + key_fallbacks: object = settings.get(key) if settings is not None else None + if isinstance(key_fallbacks, list): + lists.append( + cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes + ) return tuple(lists) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 98372b64015..6e4ea0bd0cb 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -12,7 +12,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never from litellm import constants from litellm._logging import verbose_proxy_logger -from litellm.constants import PTU_SENTINEL_API_KEY +from litellm.constants import PTU_SENTINEL_API_KEY, SUBSCRIPTION_BACKED_PROVIDERS from litellm.proxy._types import CommonProxyErrors from litellm.proxy.spend_tracking.key_metadata_recovery import ( attach_user_details, @@ -295,6 +295,12 @@ def _entity_metadata( return stored if stored is not None else {} +def provider_bucket_metadata(provider: str, base: Mapping[str, object]) -> Mapping[str, object]: + if provider in SUBSCRIPTION_BACKED_PROVIDERS: + return {**base, "subscription_covered": True} + return dict(base) + + def update_breakdown_metrics( breakdown: BreakdownMetrics, record: DailySpendRecord, @@ -395,7 +401,7 @@ def update_breakdown_metrics( if provider not in breakdown.providers: breakdown.providers[provider] = MetricWithMetadata( metrics=SpendMetrics(), - metadata=provider_metadata.get(provider, {}), # Add any provider-specific metadata here + metadata=provider_bucket_metadata(provider, provider_metadata.get(provider, {})), ) breakdown.providers[provider].metrics = update_metrics(breakdown.providers[provider].metrics, record) @@ -795,12 +801,23 @@ def _aggregate_grouping_sets_records_sync( grouped_data[date_str] = bucket return bucket - def assign_metric_with_metadata(target: dict[str, MetricWithMetadata], key: str, metrics: SpendMetrics) -> None: + def assign_metric_with_metadata( + target: dict[str, MetricWithMetadata], + key: str, + metrics: SpendMetrics, + metadata: Mapping[str, object] | None = None, + ) -> None: existing: Final = target.get(key) - if existing is None: - target[key] = MetricWithMetadata(metrics=metrics, metadata={}) - else: - existing.metrics = sum_spend_metrics((existing.metrics, metrics)) + target[key] = ( + MetricWithMetadata(metrics=metrics, metadata=dict(metadata or {})) + if existing is None + else existing.model_copy( + update={ + "metrics": sum_spend_metrics((existing.metrics, metrics)), + "metadata": {**existing.metadata, **(metadata or {})}, + } + ) + ) def assign_api_key_breakdown( target: dict[str, MetricWithMetadata], @@ -864,7 +881,9 @@ def _aggregate_grouping_sets_records_sync( # provider the base build reported. provider_metrics = metrics.model_copy(update={"flat_cost": 0.0}) provider = record.custom_llm_provider or "unknown" - assign_metric_with_metadata(breakdown.providers, provider, provider_metrics) + assign_metric_with_metadata( + breakdown.providers, provider, provider_metrics, provider_bucket_metadata(provider, {}) + ) elif level == _GROUP_DATE_PROVIDER_API_KEY: if not is_ptu_sentinel: provider = record.custom_llm_provider or "unknown" diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index d688aac0ed8..477cf44073f 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -14,10 +14,12 @@ import json from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Annotated, Final, Literal -from pydantic import Field, JsonValue, TypeAdapter +from pydantic import Field, JsonValue, TypeAdapter, ValidationError +import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.auth_utils import fallback_target_model_name from litellm.proxy.auth.model_checks import get_all_fallbacks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.utils import PrismaClient, invalidate_config_param @@ -26,6 +28,7 @@ if TYPE_CHECKING: from fastapi import APIRouter, Depends, HTTPException, status from litellm.proxy.proxy_server import ProxyConfig + from litellm.router import Router else: try: from fastapi import APIRouter, Depends, HTTPException, status @@ -41,6 +44,7 @@ from litellm.types.management_endpoints.router_settings_endpoints import ( FallbackGetResponse, FallbackResponse, ) +from litellm.types.router import ContentPolicyFallbackRule, ContentPolicyFallbackRules, FallbackTargets router: Final = APIRouter() @@ -50,12 +54,83 @@ StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_m _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback]) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) _STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue]) +_OPTIONAL_JSON_RULES: Final[TypeAdapter[Sequence[JsonValue] | None]] = TypeAdapter(Sequence[JsonValue] | None) +_CONTENT_POLICY_FALLBACK_RULE: Final[TypeAdapter[ContentPolicyFallbackRule]] = TypeAdapter(ContentPolicyFallbackRule) +_FALLBACK_TARGETS: Final[TypeAdapter[FallbackTargets]] = TypeAdapter(FallbackTargets) def _rule_covers(entry: StoredFallback, model: str) -> bool: return isinstance(entry, dict) and model in entry +def _layered_over_config_rules( + runtime_rules: ContentPolicyFallbackRules, config_rules: Sequence[Mapping[str, JsonValue]] +) -> Sequence[Mapping[str, object]]: + covered: Final = frozenset[str]().union(*runtime_rules) + return [*runtime_rules, *(rule for rule in config_rules if covered.isdisjoint(rule))] + + +def _config_file_rules(rules: object) -> Sequence[Mapping[str, JsonValue]]: + try: + candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or () + except ValidationError: + return () + return tuple(rule for rule in candidates if isinstance(rule, Mapping)) + + +def _log_ignored_content_policy_rules(rules: object, error: ValidationError) -> None: + verbose_proxy_logger.error( + "Ignoring invalid content_policy_fallbacks %r from the DB, every other rule still applies. " + "Fix it through POST /fallback or /config/update: %s", + rules, + error, + ) + + +def _checked_content_policy_rule(rule: JsonValue) -> ContentPolicyFallbackRule | ValidationError: + try: + return _CONTENT_POLICY_FALLBACK_RULE.validate_python(rule) + except ValidationError as invalid_rule: + return invalid_rule + + +def _checked_content_policy_rules( + rules: object, +) -> Sequence[tuple[object, ContentPolicyFallbackRule | ValidationError]]: + try: + candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or () + except ValidationError as not_a_rule_list: + return ((rules, not_a_rule_list),) + return tuple((candidate, _checked_content_policy_rule(candidate)) for candidate in candidates) + + +def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None: + checked_rules: Final = _checked_content_policy_rules(rules) + runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError)) + layered_rules: Final = ( + _layered_over_config_rules( + runtime_rules, + _config_file_rules( + litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list + ), + ) + or None + ) + for rule, result in checked_rules: + if log_skipped_rules and isinstance(result, ValidationError): + _log_ignored_content_policy_rules(rule, result) + if layered_rules == llm_router.settings().content_policy_fallbacks: # pyright: ignore[reportUnknownMemberType] # untyped RouterSettings list + return + llm_router.apply(ChangeSet(settings={"content_policy_fallbacks": layered_rules})) + + +def _apply_fallback_rules(llm_router: "Router", fallback_key: str, rules: Sequence[StoredFallback]) -> None: + if fallback_key == "content_policy_fallbacks": + apply_content_policy_fallbacks(llm_router, rules, log_skipped_rules=True) + return + llm_router.apply(ChangeSet(assign={fallback_key: rules})) + + async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None: await proxy_config.get_config() @@ -83,6 +158,7 @@ def _editable_rules( async def _write_rules( prisma_client: PrismaClient, proxy_config: "ProxyConfig", + llm_router: "Router", stored_settings: Mapping[str, JsonValue], fallback_key: str, rules: Sequence[StoredFallback], @@ -91,8 +167,10 @@ async def _write_rules( section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)} ) if proxy_config.router_settings.owned_by_config(fallback_key): + llm_router.apply(ChangeSet(assign={fallback_key: rules})) return await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules}) + _apply_fallback_rules(llm_router, fallback_key, rules) async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None: @@ -218,10 +296,7 @@ async def create_fallback( # Add new fallback existing_fallbacks.append({data.model: data.fallback_models}) - await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, existing_fallbacks) - - # Update the in-memory router configuration - llm_router.apply(ChangeSet(assign={fallback_key: existing_fallbacks})) + await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, existing_fallbacks) verbose_proxy_logger.info( "Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type @@ -277,7 +352,10 @@ async def get_fallback( ) # Get fallbacks using the existing utility function - fallback_models: Final = get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type) + fallback_targets: Final = _FALLBACK_TARGETS.validate_python( + get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type) + ) + fallback_models: Final = [fallback_target_model_name(target) or model for target in fallback_targets] if not fallback_models: raise HTTPException( @@ -367,10 +445,7 @@ async def delete_fallback( detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"}, ) - await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, updated_fallbacks) - - # Update the in-memory router configuration - llm_router.apply(ChangeSet(assign={fallback_key: updated_fallbacks})) + await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, updated_fallbacks) verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 6ceab27befa..6950cf08a25 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -13,6 +13,7 @@ These are members of a Team on LiteLLM """ import asyncio +import contextlib import json import traceback from collections.abc import Awaitable, Mapping, Sequence @@ -443,30 +444,25 @@ async def add_new_user_to_default_team( teams: Sequence[str | NewUserRequestTeam], prisma_client: "PrismaClient", ): - tasks: Final[list[Awaitable[object]]] = [] - for team in teams: - user_role: Literal["user", "admin"] = "user" - max_budget_in_team: float | None = None - if isinstance(team, str): - team_id = team - elif isinstance(team, NewUserRequestTeam): - team_id = team.team_id - user_role = team.user_role - max_budget_in_team = team.max_budget_in_team - else: - raise ValueError(f"Invalid team type: {type(team)}") - - tasks.append( - _add_user_to_team( + memberships: Final = tuple(_default_team_membership(team) for team in teams) + for membership in memberships: + with contextlib.suppress(Exception): + await _add_user_to_team( user_id=user_id, - team_id=team_id, + team_id=membership.team_id, user_email=user_email, user_api_key_dict=user_api_key_dict, - max_budget_in_team=max_budget_in_team, - user_role=user_role, + max_budget_in_team=membership.max_budget_in_team, + user_role=membership.user_role, ) - ) - await asyncio.gather(*tasks, return_exceptions=True) + + +def _default_team_membership(team: str | NewUserRequestTeam) -> NewUserRequestTeam: + if isinstance(team, str): + return NewUserRequestTeam(team_id=team) + if isinstance(team, NewUserRequestTeam): + return team + raise ValueError(f"Invalid team type: {type(team)}") async def _fetch_user_team_ids(user_id: str, prisma_client: "PrismaClient") -> tuple[str, ...]: diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index 2d88882a2ac..3a2de12d108 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -12,23 +12,30 @@ Endpoints here: - DELETE /access_group/{access_group}/budget - Clear an access group's shared budget """ +import asyncio +import contextlib +import itertools import json -from collections.abc import Mapping, Sequence -from datetime import datetime +import time +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from datetime import datetime, timedelta from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Any, Final, Protocol from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.config_sync_pubsub import publish_model_rows_written from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, model_access_group_cache_key, model_access_group_registry_cache_key, ) +from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper from litellm.proxy.management_endpoints.common_utils import validate_budget_duration # Clear cache and reload models to pick up the access group changes @@ -116,6 +123,28 @@ class _ModelTableClient(Protocol): async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... +class _ModelTableBatchActions(Protocol): + def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... + + +class _ModelTableBatch(Protocol): + @property + def litellm_proxymodeltable(self) -> _ModelTableBatchActions: ... + + +class _ModelTableTxActions(Protocol): + async def find_many(self) -> Sequence[_DeploymentRow]: ... + + +class _ModelTableTx(Protocol): + @property + def litellm_proxymodeltable(self) -> _ModelTableTxActions: ... + + async def query_raw(self, query: str, *args: object) -> object: ... + + def batch_(self) -> contextlib.AbstractAsyncContextManager[_ModelTableBatch]: ... + + class _BudgetRow(Protocol): @property def budget_id(self) -> str: ... @@ -168,7 +197,42 @@ class _ModelAccessGroupBudgetTableClient(Protocol): def _model_table(prisma_client: PrismaClient) -> _ModelTableClient: - return ModelRepository(prisma_client).table + return ModelRepository(WriterPinnedClient(prisma_client.db)).table + + +_TRY_LOCK_ACCESS_GROUP: Final = ( + "SELECT pg_try_advisory_xact_lock(hashtext('model_access_group'), hashtext($1)) AS locked" +) +_LOCK_MODEL_ROWS: Final = ( + 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = ANY($1::text[]) ' + "ORDER BY model_id FOR UPDATE SKIP LOCKED" +) +_MEMBERSHIP_TX_TIMEOUT: Final = timedelta(seconds=30) +_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds() +_LOCK_RETRY_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8) +_LOCK_RETRY_LONGEST_DELAY_SECONDS: Final = 1.0 + + +class _LockedModelRow(BaseModel): + model_config = ConfigDict(frozen=True) + + model_id: str + + +_LOCKED_MODEL_ROWS: Final = TypeAdapter(tuple[_LockedModelRow, ...]) + + +class _AccessGroupLock(BaseModel): + model_config = ConfigDict(frozen=True) + + locked: bool + + +_ACCESS_GROUP_LOCK: Final = TypeAdapter(tuple[_AccessGroupLock]) + + +def _model_table_tx(prisma_client: PrismaClient) -> contextlib.AbstractAsyncContextManager[_ModelTableTx]: + return writer_wrapper(prisma_client.db).tx(timeout=_MEMBERSHIP_TX_TIMEOUT) def _model_access_group_budget_table(prisma_client: PrismaClient) -> _ModelAccessGroupBudgetTableClient: @@ -379,11 +443,10 @@ async def _tag_deployment_with_access_group( """Write `access_group` into one deployment's model_info; returns the (model_id, updated model_info) pair when a write happened, None when the deployment already carried the group.""" - updated_model_info, was_modified = add_access_group_to_deployment( - model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)), - access_group=access_group, + updated_model_info: Final = _model_info_with_access_group( + model_id=model_id, model_info=model_info, access_group=access_group ) - if not was_modified: + if updated_model_info is None: return None await _model_table(prisma_client).update( where={"model_id": model_id}, @@ -393,6 +456,141 @@ async def _tag_deployment_with_access_group( return (model_id, updated_model_info) +def _model_info_with_access_group(model_id: str, model_info: object, access_group: str) -> Mapping[str, object] | None: + updated_model_info, was_modified = add_access_group_to_deployment( + model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)), + access_group=access_group, + ) + return updated_model_info if was_modified else None + + +def _model_info_without_access_group( + model_id: str, model_info: object, access_group: str +) -> Mapping[str, object] | None: + updated_model_info, was_modified = remove_access_group_from_deployment( + model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)), + access_group=access_group, + ) + return updated_model_info if was_modified else None + + +def _membership_rewrite(deployment: _DeploymentRow, selected: bool, access_group: str) -> Mapping[str, object] | None: + if selected: + return _model_info_with_access_group( + model_id=deployment.model_id, model_info=deployment.model_info, access_group=access_group + ) + return _model_info_without_access_group( + model_id=deployment.model_id, model_info=deployment.model_info, access_group=access_group + ) + + +def _membership_rewrites( + deployments: Sequence[_DeploymentRow], selected_model_ids: frozenset[str], access_group: str +) -> tuple[tuple[str, Mapping[str, object]], ...]: + rewrites: Final = ( + ( + deployment.model_id, + _membership_rewrite( + deployment=deployment, + selected=deployment.model_id in selected_model_ids, + access_group=access_group, + ), + ) + for deployment in deployments + ) + return tuple((model_id, model_info) for model_id, model_info in rewrites if model_info is not None) + + +def _carries_access_group(model_info: object, access_group: str) -> bool: + access_groups: Final = (model_info_as_mapping(model_info) or {}).get("access_groups") + return isinstance(access_groups, list) and access_group in access_groups + + +def _model_ids_to_lock( + deployments: Sequence[_DeploymentRow], access_group: str, data: UpdateModelGroupRequest +) -> frozenset[str]: + selected_model_ids: Final = frozenset(data.model_ids or ()) + selected_model_names: Final = frozenset(() if data.model_ids else data.model_names or ()) + return frozenset( + deployment.model_id + for deployment in deployments + if deployment.model_id in selected_model_ids + or deployment.model_name in selected_model_names + or _carries_access_group(deployment.model_info, access_group) + ) + + +async def _raise_404_unless_the_primary_carries(access_group: str, prisma_client: PrismaClient) -> None: + if access_group not in _access_groups_of_deployments(await _model_table(prisma_client).find_many()): + raise _http_error(404, f"Access group '{access_group}' not found") + + +async def _rewrite_membership_unless_locked( + prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest +) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None: + async with _model_table_tx(prisma_client) as tx: + (group_lock,) = _ACCESS_GROUP_LOCK.validate_python(await tx.query_raw(_TRY_LOCK_ACCESS_GROUP, access_group)) + if not group_lock.locked: + return None + model_ids_to_lock: Final = _model_ids_to_lock( + deployments=await tx.litellm_proxymodeltable.find_many(), access_group=access_group, data=data + ) + locked_rows: Final = _LOCKED_MODEL_ROWS.validate_python( + await tx.query_raw(_LOCK_MODEL_ROWS, sorted(model_ids_to_lock)) + ) + if frozenset(row.model_id for row in locked_rows) != model_ids_to_lock: + return None + deployments: Final = tuple( + deployment + for deployment in await tx.litellm_proxymodeltable.find_many() + if deployment.model_id in model_ids_to_lock + ) + if access_group not in _access_groups_of_deployments(deployments): + raise HTTPException( + status_code=404, + detail={"error": f"Access group '{access_group}' not found"}, + ) + selected_deployments: Final = _selected_deployments_or_400(deployments=deployments, data=data) + rewrites: Final = _membership_rewrites( + deployments=deployments, + selected_model_ids=frozenset(deployment.model_id for deployment in selected_deployments), + access_group=access_group, + ) + if rewrites: + async with tx.batch_() as batch: + for model_id, model_info in rewrites: + batch.litellm_proxymodeltable.update( + where={"model_id": model_id}, data={"model_info": json.dumps(model_info)} + ) + return selected_deployments, rewrites + + +def _lock_retry_delays() -> Iterator[float]: + return itertools.chain(_LOCK_RETRY_BACKOFF_SECONDS, itertools.repeat(_LOCK_RETRY_LONGEST_DELAY_SECONDS)) + + +async def _rewrite_membership_once_unlocked( + prisma_client: PrismaClient, + access_group: str, + data: UpdateModelGroupRequest, + retry_budget_seconds: float = _LOCK_RETRY_BUDGET_SECONDS, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + clock: Callable[[], float] = time.monotonic, +) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]: + deadline: Final = clock() + retry_budget_seconds + for delay in _lock_retry_delays(): + if ( + rewritten := await _rewrite_membership_unless_locked( + prisma_client=prisma_client, access_group=access_group, data=data + ) + ) is not None: + return rewritten + if clock() + delay > deadline: + break + await sleep(delay) + raise _http_error(409, f"Another edit holds access group '{access_group}' or one of its models, try again") + + def _readable_model_info_or_raise(model_id: str, model_info: object) -> Mapping[str, object]: """These helpers rewrite the model_info column wholesale, so a present-but-unreadable value must refuse loudly rather than be silently replaced with a fresh object; an @@ -412,11 +610,10 @@ async def _strip_access_group_from_deployment( """Remove `access_group` from one deployment's model_info; returns the (model_id, updated model_info) pair when a write happened, None when the deployment did not carry the group.""" - updated_model_info, was_modified = remove_access_group_from_deployment( - model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)), - access_group=access_group, + updated_model_info: Final = _model_info_without_access_group( + model_id=model_id, model_info=model_info, access_group=access_group ) - if not was_modified: + if updated_model_info is None: return None await _model_table(prisma_client).update( where={"model_id": model_id}, @@ -493,6 +690,35 @@ async def update_specific_deployments_with_access_group( return tuple(pair for pair in tagged if pair is not None) +def _selected_deployments_or_400( + deployments: Sequence[_DeploymentRow], data: UpdateModelGroupRequest +) -> tuple[_DeploymentRow, ...]: + if data.model_ids: + known_model_ids: Final = frozenset(deployment.model_id for deployment in deployments) + unknown_model_id: Final = next( + (model_id for model_id in data.model_ids if model_id not in known_model_ids), None + ) + if unknown_model_id is not None: + raise HTTPException( + status_code=400, + detail={"error": f"Deployment with model_id '{unknown_model_id}' not found in Database."}, + ) + selected_model_ids: Final = frozenset(data.model_ids) + return tuple(deployment for deployment in deployments if deployment.model_id in selected_model_ids) + model_names: Final = data.model_names or () + known_model_names: Final = frozenset(deployment.model_name for deployment in deployments) + unknown_model_name: Final = next((name for name in model_names if name not in known_model_names), None) + if unknown_model_name is not None: + raise HTTPException( + status_code=400, + detail={ + "error": f"Can't find model '{unknown_model_name}' in Database. Access group management is only supported for database models." + }, + ) + selected_model_names: Final = frozenset(model_names) + return tuple(deployment for deployment in deployments if deployment.model_name in selected_model_names) + + async def _find_deployment_or_400(model_id: str, prisma_client: PrismaClient) -> object: deployment: Final = await _model_table(prisma_client).find_unique(where={"model_id": model_id}) if deployment is None: @@ -536,10 +762,10 @@ async def get_all_access_groups_from_db( Returns: Dict[str, AccessGroupInfo]: Dictionary mapping access_group name to info """ - # Get all deployments - deployments: Final = await ModelRepository(prisma_client).table.find_many() + return dict(_access_groups_of_deployments(await ModelRepository(prisma_client).table.find_many())) - # Build access group map + +def _access_groups_of_deployments(deployments: Sequence[_DeploymentRow]) -> Mapping[str, AccessGroupInfo]: model_names_by_group: Final[dict[str, list[str]]] = {} for deployment in deployments: @@ -835,8 +1061,8 @@ async def update_access_group( Update an access group's model names. This will: - 1. Remove the access group from all current deployments - 2. Add the access group to all deployments for the new model_names list + 1. Add the access group to all deployments for the new model_names list + 2. Remove the access group from every other deployment Example: ```bash @@ -858,6 +1084,7 @@ async def update_access_group( Raises: - HTTPException 400: If any model names don't exist - HTTPException 404: If access group not found + - HTTPException 409: If another edit holds the group or one of its models for 30 s """ from litellm.proxy.proxy_server import llm_router, prisma_client @@ -879,26 +1106,14 @@ async def update_access_group( detail={"error": "Either model_names or model_ids must be provided and non-empty"}, ) - use_model_ids: Final = has_model_ids - - # Validation: Check if access group exists try: - access_groups_map: Final = await get_all_access_groups_from_db(prisma_client=prisma_client) - if access_group not in access_groups_map: - raise HTTPException( - status_code=404, - detail={"error": f"Access group '{access_group}' not found"}, - ) + await _raise_404_unless_the_primary_carries(access_group=access_group, prisma_client=prisma_client) except HTTPException: raise except Exception as e: - raise HTTPException( - status_code=500, - detail={"error": f"Failed to check access group existence: {e}"}, - ) + raise _http_error(500, f"Failed to check access group existence: {e}") - # Validation: Check if all new models exist (only if using model_names path) - if not use_model_ids and has_model_names: + if not has_model_ids and has_model_names: assert data.model_names is not None missing_models: Final = await _missing_models_after_read_through( model_names=data.model_names, @@ -912,43 +1127,19 @@ async def update_access_group( ) try: - # Step 1: Remove access group from ALL DB deployments (skip config models) - all_deployments: Final = await _model_table(prisma_client).find_many() - - stripped: Final = [ - await _strip_access_group_from_deployment( - model_id=deployment.model_id, - model_info=deployment.model_info, - access_group=access_group, - prisma_client=prisma_client, - ) - for deployment in all_deployments - ] - stripped_pairs: Final = tuple(pair for pair in stripped if pair is not None) - - # Step 2: Add access group using the appropriate method - if use_model_ids: - assert data.model_ids is not None - updated_pairs = await update_specific_deployments_with_access_group( - model_ids=data.model_ids, - access_group=access_group, - prisma_client=prisma_client, - ) - else: - assert data.model_names is not None - updated_pairs = await update_deployments_with_access_group( - model_names=data.model_names, - access_group=access_group, - prisma_client=prisma_client, - ) - models_updated: Final = len(updated_pairs) + selected_deployments, rewrites = await _rewrite_membership_once_unlocked( + prisma_client=prisma_client, access_group=access_group, data=data + ) + if rewrites: + await publish_model_rows_written(tuple(model_id for model_id, _ in rewrites)) + models_updated: Final = len(selected_deployments) # Clear cache and reload models to pick up the access group changes live_before_reload: Final = live_model_ids_snapshot() reload_outcome: Final = await clear_cache() _raise_http_if_reload_degraded_serving( before=live_before_reload, - written_models=list({**dict(stripped_pairs), **dict(updated_pairs)}.items()), + written_models=rewrites, access_group=access_group, still_desired=reload_outcome.still_desired, live_after=reload_outcome.live_after, diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index cdfe3c90182..74fa8e3b3c1 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -779,7 +779,11 @@ async def _auto_router_capability_slot( limit=limit, ) yield tables.litellm_proxymodeltable - await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable") + await publish_config_change( + redis_cache=coordination_redis_cache(), + object_type="litellm_proxymodeltable", + model_ids=() if model_id is None else (model_id,), + ) ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add" diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index 668bd64227b..e48f68a2d0a 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -229,7 +229,11 @@ class UserProvisionerHelpers: return None requested_teams: Final = list(dict.fromkeys(new_user_request.teams or [])) - new_teams: Final = requested_teams if requested_teams else list(existing_user.teams or []) + new_teams: Final = ( + list(_keep_record_team_order(existing_user.teams or [], requested_teams)) + if requested_teams + else list(existing_user.teams or []) + ) if new_user_request.user_id != existing_user.user_id: verbose_proxy_logger.info( @@ -1165,7 +1169,7 @@ async def _handle_team_membership_changes( existing_teams_set: Final = set(existing_teams) new_teams_set: Final = set(new_teams) - teams_to_add: Final = new_teams_set - existing_teams_set + teams_to_add: Final = tuple(team for team in dict.fromkeys(new_teams) if team not in existing_teams_set) teams_to_remove: Final = existing_teams_set - new_teams_set if teams_to_add or teams_to_remove: @@ -1948,7 +1952,11 @@ async def update_user( # SCIM User.groups is readOnly (RFC 7643 4.1.2): IdPs sync membership via /Groups and send # no groups or `groups: []` on profile PUTs, so empty means unspecified, not "remove from every team" - target_teams: Final = user_data["teams"] or existing_user.teams + target_teams: Final = ( + list(_keep_record_team_order(existing_user.teams, user_data["teams"])) + if user_data["teams"] + else existing_user.teams + ) await _handle_team_membership_changes( user_id=user_id, existing_teams=existing_user.teams, @@ -2241,19 +2249,29 @@ def _handle_name_update(path: str, op_type: str, value: object, scim_metadata: d scim_metadata["familyName"] = str(value) -def _handle_group_operations(op_type: str, value: object, teams_set: set[str], path: str | None) -> set[str] | None: - """Handle group/team membership operations.""" - group_values = _extract_group_values(value) - if not group_values and value is None: - group_values = _extract_ids_from_path_filter(path, "groups") - if op_type == "replace": - return set(group_values) - elif op_type == "add": - teams_set.update(group_values) - elif op_type == "remove": - for gid in group_values: - teams_set.discard(gid) - return None +def _is_group_path(path: str | None) -> bool: + return isinstance(path, str) and path.lower().startswith("groups") + + +def _apply_group_operation(teams: tuple[str, ...], op: SCIMPatchOperation) -> tuple[str, ...]: + listed: Final = _extract_group_values(op.value) + group_values: Final = listed if listed or op.value is not None else _extract_ids_from_path_filter(op.path, "groups") + if op.op == "replace": + return tuple(dict.fromkeys(group_values)) + if op.op == "add": + return tuple(dict.fromkeys((*teams, *group_values))) + removed: Final = frozenset(group_values) + return tuple(team for team in teams if team not in removed) + + +def _keep_record_team_order(record_teams: Sequence[str], target_teams: Sequence[str]) -> tuple[str, ...]: + target: Final = frozenset(target_teams) + record: Final = tuple(dict.fromkeys(record_teams)) + known: Final = frozenset(record) + return ( + *(team for team in record if team in target), + *(team for team in dict.fromkeys(target_teams) if team not in known), + ) def _multi_valued_attribute_base(path: str) -> str: @@ -2520,8 +2538,8 @@ def _handle_generic_metadata(path: str, op_type: str, value: object, metadata: d def _apply_patch_ops( existing_user: LiteLLM_UserTable, patch_ops: SCIMPatchOp, -) -> tuple[dict[str, object], set[str]]: - """Apply patch operations and return update data and final team set.""" +) -> tuple[dict[str, object], tuple[str, ...]]: + """Apply patch operations and return update data and the final teams in order.""" update_data: Final[dict[str, object]] = {} stored_metadata: Final[object] = TypeAdapter(object).validate_python(existing_user.model_dump().get("metadata")) metadata: Final = _JSON_OBJECT_ADAPTER.validate_python(stored_metadata) if isinstance(stored_metadata, dict) else {} @@ -2541,9 +2559,6 @@ def _apply_patch_ops( if isinstance(enterprise, _InvalidEnterpriseValue): _raise_invalid_enterprise_value(enterprise) - teams_set: Final[set[str]] = set(existing_user.teams or []) - replace_team_set: set[str] | None = None - for op in patch_ops.Operations: path = op.path.lower() if isinstance(op.path, str) else "" value = op.value @@ -2585,16 +2600,20 @@ def _apply_patch_ops( _handle_name_update(path, op_type, value, scim_metadata) elif _multi_valued_attribute_base(path) in SCIM_MULTI_VALUED_ATTRIBUTE_METADATA_KEYS: _handle_multi_valued_attribute_update(path, op_type, value, metadata) - elif path.startswith("groups"): - new_replace_set = _handle_group_operations(op_type, value, teams_set, op.path) - if new_replace_set is not None: - replace_team_set = new_replace_set - elif _enterprise_extension_sub_path(path) is not None: + elif _is_group_path(path) or _enterprise_extension_sub_path(path) is not None: continue else: _handle_generic_metadata(path, op_type, value, metadata) - final_team_set: Final = replace_team_set if replace_team_set is not None else teams_set + record_teams: Final = tuple(existing_user.teams or []) + group_ops: Final = tuple(op for op in patch_ops.Operations if _is_group_path(op.path)) + last_replace: Final = next((op for op in reversed(group_ops) if op.op == "replace"), None) + final_teams: Final = _keep_record_team_order( + record_teams, + _apply_group_operation(record_teams, last_replace) + if last_replace is not None + else reduce(_apply_group_operation, group_ops, record_teams), + ) final_metadata: Final = ( { **{key: metadata_value for key, metadata_value in metadata.items() if key != SCIM_ENTERPRISE_METADATA_KEY}, @@ -2605,7 +2624,7 @@ def _apply_patch_ops( else {**metadata, "scim_metadata": scim_metadata} ) update_data["metadata"] = final_metadata - return update_data, final_team_set + return update_data, final_teams def _is_user_not_in_team_error(exc: HTTPException) -> bool: @@ -2758,7 +2777,7 @@ async def patch_user( prev_active: Final = _user_scim_active(existing_user) - update_data, final_team_set = _apply_patch_ops( + update_data, final_teams = _apply_patch_ops( existing_user=existing_user, patch_ops=patch_ops, ) @@ -2770,15 +2789,15 @@ async def patch_user( await _handle_team_membership_changes( user_id=user_id, existing_teams=existing_user.teams or [], - new_teams=list(final_team_set), + new_teams=list(final_teams), ) - update_data["teams"] = list(final_team_set) + update_data["teams"] = list(final_teams) admin_group: Final = await _get_scim_admin_group() if admin_group is not None: update_data["user_role"] = _resolve_scim_user_role( - await _scim_groups_from_team_ids(prisma_client, list(final_team_set)), + await _scim_groups_from_team_ids(prisma_client, list(final_teams)), admin_group, _default_scim_user_role(), ) diff --git a/litellm/proxy/management_endpoints/subscription_accounts.py b/litellm/proxy/management_endpoints/subscription_accounts.py new file mode 100644 index 00000000000..363d6cfba5d --- /dev/null +++ b/litellm/proxy/management_endpoints/subscription_accounts.py @@ -0,0 +1,335 @@ +""" +Subscription accounts: the operator-entered recurring fee of a provider account whose +requests are covered by a flat subscription, shown beside metered spend on the Usage page. +""" + +import re +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from datetime import date +from itertools import chain +from typing import TYPE_CHECKING, Annotated, Final, TypeAlias + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from pydantic import TypeAdapter + +from litellm.constants import LITELLM_PROXY_ADMIN_NAME, SUBSCRIPTION_BACKED_PROVIDERS +from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider +from litellm.llms.subscription_accounts import resolve_subscription_account_id +from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable +from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth, user_api_key_has_admin_view +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.spend_tracking.subscription_billing_periods import billing_periods_starting_within +from litellm.proxy.utils import get_prisma_client_or_throw +from litellm.repositories.subscription_account_repository import ( + DuplicateSubscriptionAccount, + KeepLabel, + SetLabel, + SubscriptionAccountFeeChanges, + SubscriptionAccountFeeFields, + SubscriptionAccountRepository, + SubscriptionAccountStore, +) +from litellm.types.proxy.management_endpoints.subscription_accounts import ( + BillingPeriod, + DeleteSubscriptionAccountRequest, + SubscriptionAccountFeeRequest, + SubscriptionAccountUsage, + SubscriptionUsageResponse, + UpdateSubscriptionAccountFeeRequest, +) +from litellm.types.router import DeploymentTypedDict + +if TYPE_CHECKING: + from litellm.router import Router + +router: Final = APIRouter(prefix="/subscription_accounts", tags=["subscription accounts"]) +Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] + + +@dataclass(frozen=True, slots=True) +class SubscriptionDeployment: + model_name: str + custom_llm_provider: str + account_id: str | None + + +AccountIdResolver: TypeAlias = Callable[[str, Mapping[str, object]], str | None] +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +def _declared_provider(litellm_params: Mapping[str, object]) -> str | None: + model: Final = litellm_params.get("model") + configured_provider: Final = litellm_params.get("custom_llm_provider") + return declared_authenticating_provider( + model if isinstance(model, str) else None, + configured_provider if isinstance(configured_provider, str) else None, + ) + + +def _model_info(deployment: Mapping[str, object]) -> Mapping[str, object]: + return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) + + +def _public_model_name(entry: DeploymentTypedDict) -> str: + team_public_model_name: Final = _model_info(entry).get("team_public_model_name") + return team_public_model_name if isinstance(team_public_model_name, str) else entry["model_name"] + + +def _subscription_deployment( + entry: DeploymentTypedDict, account_id_resolver: AccountIdResolver +) -> SubscriptionDeployment | None: + litellm_params: Final = entry["litellm_params"] + provider: Final = _declared_provider(litellm_params) + if provider is None or provider not in SUBSCRIPTION_BACKED_PROVIDERS: + return None + return SubscriptionDeployment( + model_name=_public_model_name(entry), + custom_llm_provider=provider, + account_id=account_id_resolver(provider, litellm_params), + ) + + +def subscription_deployments_from_model_list( + model_list: Iterable[DeploymentTypedDict], account_id_resolver: AccountIdResolver +) -> tuple[SubscriptionDeployment, ...]: + candidates: Final = (_subscription_deployment(entry, account_id_resolver) for entry in model_list) + return tuple(candidate for candidate in candidates if candidate is not None) + + +def subscription_deployments_of_router( + llm_router: "Router | None", account_id_resolver: AccountIdResolver +) -> tuple[SubscriptionDeployment, ...]: + if llm_router is None: + return () + registry: Final = llm_router.registry() + indexed_names: Final = tuple(registry.index("model_name")) + deployments: Final = chain.from_iterable(registry.deployments(name, source="named") or () for name in indexed_names) + return subscription_deployments_from_model_list(deployments, account_id_resolver) + + +def get_subscription_deployments() -> tuple[SubscriptionDeployment, ...]: + from litellm.proxy.proxy_server import llm_router + + return subscription_deployments_of_router(llm_router, resolve_subscription_account_id) + + +def get_subscription_account_repository() -> SubscriptionAccountStore: + return SubscriptionAccountRepository(get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value)) + + +Store: TypeAlias = Annotated[SubscriptionAccountStore, Depends(get_subscription_account_repository)] +Deployments: TypeAlias = Annotated[tuple[SubscriptionDeployment, ...], Depends(get_subscription_deployments)] + + +def _forbidden(user_api_key_dict: UserAPIKeyAuth) -> HTTPException: + return HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={"error": f"{CommonProxyErrors.not_allowed_access.value}, your role={user_api_key_dict.user_role}"}, + ) + + +def _account_usage( + custom_llm_provider: str, + account_id: str | None, + deployments: Iterable[SubscriptionDeployment], + fee_row: LiteLLM_SubscriptionAccountTable | None, + start_date: date, + end_date: date, +) -> SubscriptionAccountUsage: + model_names: Final = tuple(sorted({deployment.model_name for deployment in deployments})) + if fee_row is None: + return SubscriptionAccountUsage( + custom_llm_provider=custom_llm_provider, + account_id=account_id, + deployments=model_names, + fee=None, + billing_periods=(), + fixed_cost=None, + ) + periods: Final = billing_periods_starting_within( + date.fromisoformat(fee_row.billing_period_start), start_date, end_date + ) + return SubscriptionAccountUsage( + custom_llm_provider=custom_llm_provider, + account_id=account_id, + deployments=model_names, + fee=fee_row, + billing_periods=tuple(BillingPeriod(start=period.start, end=period.end) for period in periods), + fixed_cost=len(periods) * fee_row.monthly_fee, + ) + + +def build_subscription_usage( + deployments: Iterable[SubscriptionDeployment], + fee_rows: Iterable[LiteLLM_SubscriptionAccountTable], + start_date: date, + end_date: date, +) -> SubscriptionUsageResponse: + deployment_tuple: Final = tuple(deployments) + fees_by_account: Final = {(row.custom_llm_provider, row.account_id): row for row in fee_rows} + deployment_keys: Final = {(d.custom_llm_provider, d.account_id) for d in deployment_tuple} + account_keys: Final = sorted(deployment_keys | set(fees_by_account), key=lambda key: (key[0], key[1] or "~")) + accounts: Final = tuple( + _account_usage( + provider, + account_id, + (d for d in deployment_tuple if (d.custom_llm_provider, d.account_id) == (provider, account_id)), + fees_by_account.get((provider, account_id)) if account_id is not None else None, + start_date, + end_date, + ) + for provider, account_id in account_keys + ) + return SubscriptionUsageResponse( + start_date=start_date, + end_date=end_date, + subscription_providers=tuple(sorted(SUBSCRIPTION_BACKED_PROVIDERS)), + accounts=accounts, + ) + + +_DAY_PATTERN: Final = re.compile(r"\d{4}-\d{2}-\d{2}") + + +def _parse_day(value: str) -> date | None: + if _DAY_PATTERN.fullmatch(value) is None: + return None + try: + return date.fromisoformat(value) + except ValueError: + return None + + +def _parse_date(value: str, name: str) -> date: + parsed: Final = _parse_day(value) + if parsed is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail={"error": f"{name} must be YYYY-MM-DD, got {value!r}"} + ) + return parsed + + +@router.get("/usage", response_model=SubscriptionUsageResponse, dependencies=[Depends(user_api_key_auth)]) +async def get_subscription_usage( + start_date: Annotated[str, Query(description="First day of the Usage page filter, YYYY-MM-DD")], + end_date: Annotated[str, Query(description="Last day of the Usage page filter, YYYY-MM-DD")], + user_api_key_dict: Auth, + repository: Store, + deployments: Deployments, +) -> SubscriptionUsageResponse: + """ + Subscription-backed accounts behind the proxy's deployments or carrying a stored fee, each with + its configured monthly fee and the fixed cost attributed to the date range: the fee once per billing + period whose start date falls inside the range, however many deployments share the account. + """ + if not user_api_key_has_admin_view(user_api_key_dict): + raise _forbidden(user_api_key_dict) + start: Final = _parse_date(start_date, "start_date") + end: Final = _parse_date(end_date, "end_date") + if end < start: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail={"error": "end_date must not be before start_date"} + ) + return build_subscription_usage(deployments, await repository.list_all(), start, end) + + +def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise _forbidden(user_api_key_dict) + + +def _actor(user_api_key_dict: UserAPIKeyAuth) -> str: + return user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME + + +@router.post("/new", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)]) +async def new_subscription_account( + request: SubscriptionAccountFeeRequest, + user_api_key_dict: Auth, + repository: Store, +) -> LiteLLM_SubscriptionAccountTable: + """Record the monthly fee of one subscription-backed provider account.""" + _require_proxy_admin(user_api_key_dict) + if request.custom_llm_provider not in SUBSCRIPTION_BACKED_PROVIDERS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": f"{request.custom_llm_provider!r} is not a subscription-backed provider; " + f"expected one of {sorted(SUBSCRIPTION_BACKED_PROVIDERS)}" + }, + ) + created: Final = await repository.create_account( + request.custom_llm_provider, + request.account_id, + SubscriptionAccountFeeFields( + monthly_fee=request.monthly_fee, + currency=request.currency, + billing_period_start=request.billing_period_start.isoformat(), + label=request.label, + ), + _actor(user_api_key_dict), + ) + if isinstance(created, DuplicateSubscriptionAccount): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": f"A fee is already configured for {created.custom_llm_provider} account " + f"{created.account_id}; use /subscription_accounts/update to change it" + }, + ) + return created + + +def _fee_changes(request: UpdateSubscriptionAccountFeeRequest) -> SubscriptionAccountFeeChanges: + return SubscriptionAccountFeeChanges( + monthly_fee=request.monthly_fee, + currency=request.currency, + billing_period_start=( + request.billing_period_start.isoformat() if request.billing_period_start is not None else None + ), + label=SetLabel(request.label) if "label" in request.model_fields_set else KeepLabel(), + ) + + +@router.post("/update", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)]) +async def update_subscription_account( + request: UpdateSubscriptionAccountFeeRequest, + user_api_key_dict: Auth, + repository: Store, +) -> LiteLLM_SubscriptionAccountTable: + """ + Change the fee, currency, billing period start, or label of a subscription account. + Only the fields named in the request are written, so two admins changing different + fields at the same time both land. + """ + _require_proxy_admin(user_api_key_dict) + updated: Final = await repository.update_fee( + request.subscription_account_id, _fee_changes(request), _actor(user_api_key_dict) + ) + if updated is None: + raise _not_found(request.subscription_account_id) + return updated + + +@router.post("/delete", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)]) +async def delete_subscription_account( + request: DeleteSubscriptionAccountRequest, + user_api_key_dict: Auth, + repository: Store, +) -> LiteLLM_SubscriptionAccountTable: + """Stop tracking a subscription account's fee; its requests stay marked as subscription-covered.""" + _require_proxy_admin(user_api_key_dict) + if await repository.find_by_id(request.subscription_account_id) is None: + raise _not_found(request.subscription_account_id) + deleted: Final = await repository.delete_account(request.subscription_account_id) + if deleted is None: + raise _not_found(request.subscription_account_id) + return deleted + + +def _not_found(subscription_account_id: str) -> HTTPException: + return HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"error": f"No subscription account with id {subscription_account_id}"}, + ) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index a980b9c33ff..59c2df91d77 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -8,7 +8,6 @@ Has all /sso/* routes /sso/debug/callback - returns the OpenID object returned by the SSO provider """ -import asyncio import base64 import hashlib import inspect @@ -151,6 +150,7 @@ from litellm.proxy.utils import ( get_custom_url, get_server_root_path, ) +from litellm.repositories.chunked_in import find_many_in from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import SSOConfigRepository from litellm.repositories.team_repository import TeamRepository @@ -1855,7 +1855,7 @@ async def warn_if_id_jag_capture_gap(*, retention_enabled: RetentionCheck | None verbose_proxy_logger.warning("SSO debug callback ran with an oauth2_id_jag capture gap: %s", gap) -async def create_team_member_add_task(team_id, user_info): +async def create_team_member_add_task(team_id: str, user_info: NewUserResponse | LiteLLM_UserTable): """Create a task for adding a member to a team.""" try: member: Final = Member(user_id=user_info.user_id, role="user") @@ -1871,22 +1871,43 @@ async def create_team_member_add_task(team_id, user_info): verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e) -async def add_missing_team_member(user_info: NewUserResponse | LiteLLM_UserTable, sso_teams: list[str]): +async def find_existing_team_ids(team_ids: Sequence[str]) -> frozenset[str] | None: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return None + try: + rows: Final = await find_many_in(TeamRepository(prisma_client, use_writer=True).table, "team_id", team_ids) + except Exception as e: + verbose_proxy_logger.debug("[Non-Blocking] Error looking up SSO teams: %s", e) + return None + return frozenset(row.team_id for row in rows) + + +async def add_missing_team_member( + user_info: NewUserResponse | LiteLLM_UserTable, + sso_teams: Sequence[str], + *, + add_team_member: Callable[ + [str, NewUserResponse | LiteLLM_UserTable], Awaitable[object] + ] = create_team_member_add_task, + existing_team_ids: Callable[[Sequence[str]], Awaitable[frozenset[str] | None]] = find_existing_team_ids, +) -> None: """ - Get missing teams (diff b/w user_info.team_ids and sso_teams) - - Add missing user to missing teams + - Drop the IdP groups that are not teams with one lookup, since group claims often carry many of them + - Add missing user to missing teams, one at a time in sso_teams order, so user.teams keeps the claim's order """ - # Handle None as empty list for new users - user_teams: Final = user_info.teams if user_info.teams is not None else [] - missing_teams: Final = set(sso_teams) - set(user_teams) - missing_teams_list: Final = list(missing_teams) - tasks = [] - tasks = [create_team_member_add_task(team_id, user_info) for team_id in missing_teams_list] - - try: - await asyncio.gather(*tasks) - except Exception as e: - verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e) + held_teams: Final = frozenset(user_info.teams or ()) + missing_teams: Final = tuple(team_id for team_id in dict.fromkeys(sso_teams) if team_id not in held_teams) + if not missing_teams: + return + existing: Final = await existing_team_ids(missing_teams) + teams_to_add: Final = ( + missing_teams if existing is None else tuple(team_id for team_id in missing_teams if team_id in existing) + ) + for team_id in teams_to_add: + await add_team_member(team_id, user_info) def get_disabled_non_admin_personal_key_creation(): @@ -2451,7 +2472,9 @@ async def fetch_cli_sso_team_details( except Exception as e: verbose_proxy_logger.error("Error fetching team details for CLI SSO session: %s", e) return None - return tuple(_cli_sso_team_detail(team_row.model_dump()) for team_row in prisma_teams) + fetched_details: Final = tuple(_cli_sso_team_detail(team_row.model_dump()) for team_row in prisma_teams) + details_by_team_id: Final = MappingProxyType({detail.team_id: detail for detail in fetched_details}) + return tuple(details_by_team_id[team_id] for team_id in dict.fromkeys(teams) if team_id in details_by_team_id) def _cli_sso_session_teams(team_details: Sequence[CliSsoTeamDetail]) -> list[str]: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b0ccf74c63b..f078267c6b2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -549,6 +549,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import ( write_reload_interval, ) from litellm.proxy.common_utils.proxy_state import ProxyState +from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState from litellm.proxy.common_utils.scheduled_job_stagger import ( @@ -717,6 +718,9 @@ from litellm.proxy.management_endpoints.customer_endpoints import ( router as customer_router, ) from litellm.proxy.management_endpoints.daily_activity_routes import router as daily_activity_router +from litellm.proxy.management_endpoints.fallback_management_endpoints import ( + apply_content_policy_fallbacks, +) from litellm.proxy.management_endpoints.fallback_management_endpoints import ( router as fallback_management_router, ) @@ -776,6 +780,9 @@ from litellm.proxy.management_endpoints.session_endpoints import ( router as session_management_router, ) from litellm.proxy.management_endpoints.sso.saml_sso import SAML_AUTHN_REQUEST_TTL_SECONDS +from litellm.proxy.management_endpoints.subscription_accounts import ( + router as subscription_accounts_router, +) from litellm.proxy.management_endpoints.tag_management_endpoints import ( router as tag_management_router, ) @@ -8159,14 +8166,38 @@ class ProxyConfig: if db_router_settings is not None and db_router_settings.param_value is not None else _EMPTY_SETTINGS_MAPPING ) + previous_content_policy_rules: Final = self.router_settings.resolved().get("content_policy_fallbacks") self.router_settings.apply_db_row("router_settings", db_values) combined_router_settings: Final = self.router_settings.resolved() if combined_router_settings: - self._apply_router_settings(llm_router, combined_router_settings) + self._apply_router_settings( + llm_router, + combined_router_settings, + content_policy_owned_by_config=self.router_settings.owned_by_config("content_policy_fallbacks"), + content_policy_rules_changed=( + combined_router_settings.get("content_policy_fallbacks") != previous_content_policy_rules + ), + ) @staticmethod - def _apply_router_settings(llm_router: Router, router_settings: Mapping[str, object]) -> None: - llm_router.apply(ChangeSet(settings={k: v for k, v in router_settings.items() if k != "routing_groups"})) + def _apply_router_settings( + llm_router: Router, + router_settings: Mapping[str, object], + *, + content_policy_owned_by_config: bool, + content_policy_rules_changed: bool, + ) -> None: + llm_router.apply( + ChangeSet( + settings={ + k: v for k, v in router_settings.items() if k not in ("routing_groups", "content_policy_fallbacks") + } + ) + ) + if "content_policy_fallbacks" in router_settings and not content_policy_owned_by_config: + apply_content_policy_fallbacks( + llm_router, router_settings["content_policy_fallbacks"], log_skipped_rules=content_policy_rules_changed + ) if "routing_groups" not in router_settings: return try: @@ -8750,6 +8781,7 @@ class ProxyConfig: subscriber: Final = ConfigSyncSubscriber( redis_cache=redis_cache, resync_callbacks=(_resync_config_from_db,), + apply_model_ids=resync_model_deployments_by_ids, ) self.config_sync_subscriber = subscriber subscriber.start() @@ -21767,6 +21799,7 @@ app.include_router(memory_router) app.include_router(lens_router) app.include_router(plugin_router) app.include_router(cost_tracking_settings_router) +app.include_router(subscription_accounts_router) app.include_router(prompt_caching_requests_router) app.include_router(router_settings_router) app.include_router(fallback_management_router) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 2965174114b..179b741fe7c 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery { @@id([event_id, target, version]) @@index([target, state, next_attempt_at]) } + +model LiteLLM_SubscriptionAccountTable { + subscription_account_id String @id @default(uuid()) + custom_llm_provider String + account_id String + label String? + monthly_fee Float + currency String + billing_period_start String + created_at DateTime @default(now()) @map("created_at") + created_by String + updated_at DateTime @default(now()) @updatedAt @map("updated_at") + updated_by String + + @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") +} diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 28602fbbefc..8e79ea5255f 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -689,9 +689,13 @@ def compute_savings_spend( cost_breakdown=cost_breakdown, ) ) - return SavingsSpend( + savings: Final = SavingsSpend( compression=compression, prompt_caching=prompt_caching, autorouter=0.0 if autorouter is None else autorouter, gateway_injected_caching=gateway_injected_caching, ) + if all(isfinite(value) for value in savings): + return savings + verbose_proxy_logger.warning("Spend tracking - recording non-finite savings %s as 0 for model=%s", savings, model) + return SavingsSpend(*(value if isfinite(value) else 0.0 for value in savings)) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 09b7ce7ee9a..f8d5cba4229 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -912,7 +912,7 @@ def get_logging_payload( user=metadata.get("user_api_key_user_id", "") or "", team_id=metadata.get("user_api_key_team_id", "") or "", organization_id=metadata.get("user_api_key_org_id") or "", - metadata=safe_dumps(clean_metadata), + metadata=safe_dumps(clean_metadata, non_finite_as_null=True), cache_key=cache_key, spend=kwargs.get("response_cost", 0), total_tokens=usage.get("total_tokens", standard_logging_total_tokens), @@ -1224,7 +1224,7 @@ def _get_messages_for_spend_logs_payload( messages: Final = standard_logging_payload.get("messages") if messages is not None: try: - return safe_dumps(messages) + return safe_dumps(messages, non_finite_as_null=True) except Exception: return "{}" return "{}" @@ -1919,7 +1919,8 @@ def _get_proxy_server_request_for_spend_logs_payload( _request_body_json_str: Final = safe_dumps( _stored_request_without_requester_ip_address(stored_request_body) if drop_requester_ip_address - else stored_request_body + else stored_request_body, + non_finite_as_null=True, ) if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str: verbose_proxy_logger.info( @@ -2000,7 +2001,7 @@ def _get_response_for_spend_logs_payload( if isinstance(sanitized_response, str): result_str = strip_null_bytes(sanitized_response) else: - result_str = safe_dumps(sanitized_response) + result_str = safe_dumps(sanitized_response, non_finite_as_null=True) if LITELLM_TRUNCATED_PAYLOAD_FIELD in result_str: verbose_proxy_logger.info( "Spend Log: response was truncated before storing in DB. %s", @@ -2071,7 +2072,7 @@ def spend_log_row_with_retained_metadata( return row metadata: Final = _SPEND_LOGS_METADATA_ADAPTER.validate_json(metadata_json) retained: Final = {name: value for name, value in metadata.items() if fields.keeps(name)} - return {**row, "metadata": safe_dumps(retained)} + return {**row, "metadata": safe_dumps(retained, non_finite_as_null=True)} def _get_status_for_spend_log( diff --git a/litellm/proxy/spend_tracking/subscription_billing_periods.py b/litellm/proxy/spend_tracking/subscription_billing_periods.py new file mode 100644 index 00000000000..a7282cc942c --- /dev/null +++ b/litellm/proxy/spend_tracking/subscription_billing_periods.py @@ -0,0 +1,54 @@ +from calendar import monthrange +from dataclasses import dataclass +from datetime import date, timedelta +from typing import Final + + +@dataclass(frozen=True, slots=True) +class BillingPeriodRange: + start: date + end: date + + +def _month_index(anchor: date, months_after_anchor: int) -> int: + return anchor.year * 12 + (anchor.month - 1) + months_after_anchor + + +_LAST_MONTH_INDEX: Final = _month_index(date.max, 0) + + +def _month_start(anchor: date, months_after_anchor: int) -> date: + month_index: Final = _month_index(anchor, months_after_anchor) + year: Final = month_index // 12 + month: Final = month_index % 12 + 1 + return date(year, month, min(anchor.day, monthrange(year, month)[1])) + + +def _period_end(anchor: date, months_after_anchor: int) -> date: + if _month_index(anchor, months_after_anchor + 1) > _LAST_MONTH_INDEX: + return date.max + return _month_start(anchor, months_after_anchor + 1) - timedelta(days=1) + + +def _first_candidate_index(anchor: date, start_date: date) -> int: + months_apart: Final = (start_date.year - anchor.year) * 12 + (start_date.month - anchor.month) + return max(0, months_apart - 1) + + +def billing_periods_starting_within(anchor: date, start_date: date, end_date: date) -> tuple[BillingPeriodRange, ...]: + """Monthly periods anchored on `anchor`'s day of month whose start lies inside [start_date, end_date].""" + if end_date < start_date or end_date < anchor: + return () + first_index: Final = _first_candidate_index(anchor, start_date) + last_index: Final = min( + first_index + _month_span(start_date, end_date) + 1, _LAST_MONTH_INDEX - _month_index(anchor, 0) + ) + candidates: Final = ( + BillingPeriodRange(start=_month_start(anchor, index), end=_period_end(anchor, index)) + for index in range(first_index, last_index + 1) + ) + return tuple(period for period in candidates if start_date <= period.start <= end_date) + + +def _month_span(start_date: date, end_date: date) -> int: + return (end_date.year - start_date.year) * 12 + (end_date.month - start_date.month) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b1055bb9124..ddaa8d2e69d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -85,12 +85,13 @@ from litellm.proxy._types import ( from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.openai_error_payload import ( attribute_of, - litellm_call_id_headers, - openai_error_param, + error_status_code, + openai_shaped_proxy_exception, with_litellm_call_id, ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction +from litellm.responses.mcp.parked_stream_dispatch import held_round_spend from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage @@ -4943,6 +4944,8 @@ class ProxyLogging: if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS): return False match _args: + case (ModelResponse(), *_) if (held := held_round_spend(logging_obj)) is not None: + logging_obj.record_billed_rounds_for_failure(held) case (ModelResponse() as assembled, *_): logging_obj.record_assembled_response_for_failure(assembled) case (BilledRoundsDispatch() as billed,): @@ -9186,39 +9189,30 @@ def _recreate_writer_on_read_only_transaction(prisma_client: "PrismaClient | Non asyncio.create_task(prisma_client.recreate_read_only_writer(reason="postgres_read_only_transaction")) +def _client_message(e: Exception) -> str: + status_is_none: Final = hasattr(e, "status_code") and attribute_of(e, "status_code") is None + return "Internal server error" if status_is_none else strip_bug_report_notice(str(e)) + + def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) -> ProxyException: """ Returns an Exception as ProxyException, this ensures all exceptions are OpenAI API compatible """ - from fastapi import status - verbose_proxy_logger.exception("Exception: %s", e) if PrismaDBExceptionHandler.is_read_only_transaction_error(e): from litellm.proxy.proxy_server import prisma_client _recreate_writer_on_read_only_transaction(prisma_client) - headers: Final = litellm_call_id_headers(litellm_call_id) - if isinstance(e, HTTPException): - return ProxyException( - message=getattr(e, "detail", f"error({e})"), - type=ProxyErrorTypes.internal_server_error, - param=openai_error_param(e), - headers=headers, - code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), - ) - elif isinstance(e, ProxyException): + if isinstance(e, ProxyException): return with_litellm_call_id(e, litellm_call_id) - _status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + if isinstance(e, HTTPException): + return with_litellm_call_id(openai_shaped_proxy_exception(e, str(e.detail), e.status_code), litellm_call_id) if should_report_bug(e): verbose_proxy_logger.error(bug_report_notice(build_proxy_bug_report(e))) - return ProxyException( - message=strip_bug_report_notice(str(e)), - type=ProxyErrorTypes.internal_server_error, - param=openai_error_param(e), - headers=headers, - code=_status_code, - ) + message: Final = _client_message(e) + status_code: Final = error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR) + return with_litellm_call_id(openai_shaped_proxy_exception(e, message, status_code), litellm_call_id) def require_enterprise_license(feature: str | None = None) -> None: diff --git a/litellm/repositories/subscription_account_repository.py b/litellm/repositories/subscription_account_repository.py new file mode 100644 index 00000000000..ddf441a9191 --- /dev/null +++ b/litellm/repositories/subscription_account_repository.py @@ -0,0 +1,134 @@ +""" +Subscription account repository for database operations on LiteLLM_SubscriptionAccountTable. +""" + +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias + +from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable +from litellm.repositories.base_repository import BaseRepository, is_unique_violation +from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.table_repositories import PrismaTableRepository + +if TYPE_CHECKING: + from prisma import models as prisma_models + + +@dataclass(frozen=True, slots=True) +class DuplicateSubscriptionAccount: + custom_llm_provider: str + account_id: str + + +@dataclass(frozen=True, slots=True) +class SubscriptionAccountFeeFields: + monthly_fee: float + currency: str + billing_period_start: str + label: str | None + + +@dataclass(frozen=True, slots=True) +class KeepLabel: + pass + + +@dataclass(frozen=True, slots=True) +class SetLabel: + value: str | None + + +LabelChange: TypeAlias = KeepLabel | SetLabel + + +@dataclass(frozen=True, slots=True) +class SubscriptionAccountFeeChanges: + monthly_fee: float | None = None + currency: str | None = None + billing_period_start: str | None = None + label: LabelChange = KeepLabel() + + def columns(self) -> Mapping[str, float | str | None]: + named: Final = { + "monthly_fee": self.monthly_fee, + "currency": self.currency, + "billing_period_start": self.billing_period_start, + } + present: Final = {column: value for column, value in named.items() if value is not None} + label_column: Final = {"label": self.label.value} if isinstance(self.label, SetLabel) else {} + return MappingProxyType({**present, **label_column}) + + +class SubscriptionAccountStore(Protocol): + async def list_all(self) -> tuple[LiteLLM_SubscriptionAccountTable, ...]: ... + + async def find_by_id(self, id_value: str) -> LiteLLM_SubscriptionAccountTable | None: ... + + async def create_account( + self, custom_llm_provider: str, account_id: str, fee: SubscriptionAccountFeeFields, created_by: str + ) -> LiteLLM_SubscriptionAccountTable | DuplicateSubscriptionAccount: ... + + async def update_fee( + self, subscription_account_id: str, changes: SubscriptionAccountFeeChanges, updated_by: str + ) -> LiteLLM_SubscriptionAccountTable | None: ... + + async def delete_account(self, subscription_account_id: str) -> LiteLLM_SubscriptionAccountTable | None: ... + + +class _SubscriptionAccountTable(PrismaTableRepository["prisma_models.LiteLLM_SubscriptionAccountTable"]): + table_name = "litellm_subscriptionaccounttable" + + +class SubscriptionAccountRepository(BaseRepository[LiteLLM_SubscriptionAccountTable]): + def __init__(self, prisma_client: object) -> None: + super().__init__(prisma_client) + self._table: Final = _SubscriptionAccountTable(prisma_client) + + @property + def table(self) -> TableActions["prisma_models.LiteLLM_SubscriptionAccountTable"]: + return self._table.table + + @property + def model_class(self) -> type[LiteLLM_SubscriptionAccountTable]: + return LiteLLM_SubscriptionAccountTable + + async def list_all(self) -> tuple[LiteLLM_SubscriptionAccountTable, ...]: + return tuple(await self.find_many(order={"created_at": "asc"})) + + async def find_by_id( + self, id_value: str, id_field: str = "subscription_account_id" + ) -> LiteLLM_SubscriptionAccountTable | None: + return await super().find_by_id(id_value, id_field) + + async def create_account( + self, custom_llm_provider: str, account_id: str, fee: SubscriptionAccountFeeFields, created_by: str + ) -> LiteLLM_SubscriptionAccountTable | DuplicateSubscriptionAccount: + try: + return await self.create( + { + "custom_llm_provider": custom_llm_provider, + "account_id": account_id, + "monthly_fee": fee.monthly_fee, + "currency": fee.currency, + "billing_period_start": fee.billing_period_start, + "label": fee.label, + "created_by": created_by, + "updated_by": created_by, + } + ) + except Exception as exc: + if is_unique_violation(exc): + return DuplicateSubscriptionAccount(custom_llm_provider=custom_llm_provider, account_id=account_id) + raise + + async def update_fee( + self, subscription_account_id: str, changes: SubscriptionAccountFeeChanges, updated_by: str + ) -> LiteLLM_SubscriptionAccountTable | None: + return await self.update( + subscription_account_id, {**changes.columns(), "updated_by": updated_by}, id_field="subscription_account_id" + ) + + async def delete_account(self, subscription_account_id: str) -> LiteLLM_SubscriptionAccountTable | None: + return await self.delete(subscription_account_id, id_field="subscription_account_id") diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 063ef9d91ea..9259103cb5c 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars import json -from collections.abc import Awaitable, Callable, Coroutine, Generator, Iterable, Mapping, Sequence +from collections.abc import Coroutine, Generator, Iterable, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass from functools import partial @@ -31,7 +31,10 @@ from litellm.integrations.anthropic_cache_control_hook import ( from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params -from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header +from litellm.litellm_core_utils.get_llm_provider_logic import ( + declared_authenticating_provider, + get_provider_affinity_header, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -78,7 +81,7 @@ from litellm.types.llms.openai import ( # Handle ResponseText import with fallback if TYPE_CHECKING: - from litellm.responses.streaming_iterator import ResponsesWsFrameGuard + from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission from litellm.types.llms.openai import ResponseText else: ResponseText = str # Fallback for ResponseText import @@ -805,6 +808,7 @@ def provider_stores_the_conversation(model: str, custom_llm_provider: str | None return not _will_bridge_to_chat_completions( model, custom_llm_provider + or declared_authenticating_provider(model) or litellm.get_llm_provider(model=model, api_base=api_base, litellm_params=litellm_params)[1], bool(kwargs.get("use_chat_completions_api")), kwargs.get("model_info"), @@ -2863,7 +2867,7 @@ async def _aresponses_websocket( api_base: str | None = None, api_key: str | None = None, timeout: float | None = None, - turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_admission: "ResponsesWsTurnAdmission | None" = None, responses_ws_frame_guard: "ResponsesWsFrameGuard | None" = None, **kwargs, ) -> Exception | None: diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index cd072825ef5..1e07b768f62 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -1,15 +1,18 @@ """Helpers for handling MCP-aware `/chat/completions` requests.""" +import asyncio import logging from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast +import anyio from pydantic import TypeAdapter from typing_extensions import TypedDict, Unpack from litellm._internal_context import internal_sub_call from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.responses.mcp.chat_stream_round_billing import ChatStreamRoundBilling, engaged_round_billing from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) @@ -19,7 +22,7 @@ from litellm.responses.mcp.round_billing import ( first_round_billed_on_failure, summed_cost_breakdown, ) -from litellm.types.utils import Message, ModelResponse +from litellm.types.utils import Message, ModelResponse, Usage from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: @@ -258,10 +261,20 @@ async def acompletion_with_mcp( self.follow_up_stream = None self.follow_up_iterator = None self.follow_up_exhausted = False + self._round_billing: ChatStreamRoundBilling | None = engaged_round_billing(initial_stream.logging_obj) + self._unsent_usage_chunk: ModelResponseStream | None = None def __aiter__(self): return self + def __del__(self) -> None: + self.release_abandoned() + + def release_abandoned(self) -> None: + round_billing: Final[object] = getattr(self, "_round_billing", None) + if isinstance(round_billing, ChatStreamRoundBilling): + round_billing.release_abandoned() + def _add_mcp_list_tools_to_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: """Add mcp_list_tools to the first chunk.""" from litellm.types.utils import ( @@ -321,18 +334,54 @@ async def acompletion_with_mcp( return chunk - async def _drain_inner_stream(self): + async def _drain_inner_stream(self) -> tuple[ModelResponseStream, ...]: try: - while True: - await self._stream_iterator.__anext__() - except StopAsyncIteration: - pass + return tuple([chunk async for chunk in initial_stream]) except Exception: logging.getLogger("LiteLLM").exception( "Error draining inner MCP stream after final chunk; spend logging may be incomplete" ) + return () - async def __anext__(self): + async def _complete_first_round(self) -> ModelResponseStream | None: + drained: Final = await self._drain_inner_stream() + if self._round_billing is not None: + self._round_billing.complete_first_round() + usage_chunks: Final = tuple( + chunk for chunk in drained if isinstance(getattr(chunk, "usage", None), Usage) + ) + return usage_chunks[-1] if usage_chunks else None + + async def _follow_up_or_release_first_round(self, usage_chunk: ModelResponseStream | None) -> None: + if self.tool_results and self.complete_response: + await self._prepare_follow_up_call() + if self.follow_up_stream is not None: + return + self._unsent_usage_chunk = usage_chunk + if self._round_billing is not None: + self._round_billing.finish() + + async def __anext__(self) -> ModelResponseStream: + round_billing: Final = self._round_billing + if round_billing is None: + return await self._next_chunk() + with round_billing.armed_while_reading(): + try: + return await self._next_chunk() + except StopAsyncIteration: + usage_chunk: Final = round_billing.finish() + if usage_chunk is None: + raise + return usage_chunk + except asyncio.CancelledError: + with anyio.CancelScope(shield=True): + await round_billing.bill_in_flight_round() + raise + except Exception: + round_billing.fail() + raise + + async def _next_chunk(self) -> ModelResponseStream: # Phase 1: Collect and yield initial stream chunks if not self.stream_exhausted: # Get the iterator from the stream wrapper @@ -362,29 +411,22 @@ async def acompletion_with_mcp( if is_final: self.stream_exhausted = True + usage_chunk: Final = await self._complete_first_round() await self._process_tool_calls() chunk = self._add_mcp_tool_metadata_to_final_chunk(chunk) - if self.tool_results and self.complete_response: - await self._prepare_follow_up_call() - # Drain inner stream so CustomStreamWrapper fires its - # end-of-stream handler (dispatch_success_handlers → - # _ProxyDBLogger → LiteLLM_SpendLogs). The CSW may - # yield one usage chunk before raising StopAsyncIteration. - await self._drain_inner_stream() + await self._follow_up_or_release_first_round(usage_chunk) return chunk except StopAsyncIteration: self.stream_exhausted = True + drained_usage_chunk: Final = await self._complete_first_round() # Process tool calls after stream is exhausted await self._process_tool_calls() # If we have chunks, yield the final one with metadata if self.collected_chunks: final_chunk = self.collected_chunks[-1] final_chunk = self._add_mcp_tool_metadata_to_final_chunk(final_chunk) - # If we have tool results, prepare follow-up call - if self.tool_results and self.complete_response: - await self._prepare_follow_up_call() - await self._drain_inner_stream() + await self._follow_up_or_release_first_round(drained_usage_chunk) return final_chunk # Phase 2: Yield follow-up stream chunks if available @@ -400,7 +442,6 @@ async def acompletion_with_mcp( from litellm._logging import verbose_logger verbose_logger.debug("Follow-up chunk yielded: %s", chunk) - return chunk except StopAsyncIteration: self.follow_up_exhausted = True from litellm._logging import verbose_logger @@ -408,6 +449,10 @@ async def acompletion_with_mcp( verbose_logger.debug("Follow-up stream exhausted") # After follow-up stream is exhausted, check if we need to raise StopAsyncIteration raise StopAsyncIteration + client_chunk: Final = ( + chunk if self._round_billing is None else self._round_billing.client_chunk(chunk) + ) + return await self._next_chunk() if client_chunk is None else client_chunk # If we're here and follow_up_stream is None but we expected it, log a warning if ( @@ -420,7 +465,11 @@ async def acompletion_with_mcp( verbose_logger.warning("Follow-up stream was not created despite having tool results") - raise StopAsyncIteration + unsent_usage_chunk: Final = self._unsent_usage_chunk + if unsent_usage_chunk is None: + raise StopAsyncIteration + self._unsent_usage_chunk = None + return unsent_usage_chunk async def _process_tool_calls(self): """Process tool calls after streaming completes.""" @@ -488,10 +537,16 @@ async def acompletion_with_mcp( # This ensures the patch works correctly in tests import litellm + earlier_rounds: Final = None if self._round_billing is None else self._round_billing.bank_first_round() follow_up_response: Final = await litellm.acompletion(**follow_up_call_args) # Ensure follow-up response is a CustomStreamWrapper if isinstance(follow_up_response, CustomStreamWrapper): + if earlier_rounds is not None: + follow_up_response.bill_failure_after_earlier_rounds(earlier_rounds) + follow_up_response.messages = follow_up_messages + if self._round_billing is not None: + self._round_billing.follow_up_started(follow_up_response.partial_response) self.follow_up_stream = follow_up_response from litellm._logging import verbose_logger @@ -548,11 +603,13 @@ async def acompletion_with_mcp( def __aiter__(self): return self._custom_iterator + async def aclose(self) -> None: + self._custom_iterator.release_abandoned() + await super().aclose() + def __iter__(self): # For synchronous iteration, create a sync wrapper if self._sync_iterator is None: - import asyncio - try: self._sync_loop = asyncio.get_event_loop() except RuntimeError: diff --git a/litellm/responses/mcp/chat_stream_round_billing.py b/litellm/responses/mcp/chat_stream_round_billing.py new file mode 100644 index 00000000000..370de663606 --- /dev/null +++ b/litellm/responses/mcp/chat_stream_round_billing.py @@ -0,0 +1,281 @@ +import asyncio +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass, replace +from types import MappingProxyType +from typing import Final, TypeAlias + +from litellm.litellm_core_utils.asyncify import asyncify, is_event_loop_running +from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_params +from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, UncachedPrice +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.responses.mcp.parked_stream_dispatch import ( + DISPATCH_RELEASE, + HELD_ROUND_SPEND, + IN_FLIGHT_ROUND, + PARKED_DISPATCH, + SERVED_FROM_CACHE, + DeferredStreamRelease, + parked_success, +) +from litellm.responses.mcp.round_billing import billed_for_every_round, merged_chat_usage, summed_cost_breakdown +from litellm.responses.streaming_iterator import spawn_logging_task +from litellm.types.utils import ModelResponse, ModelResponseStream, Usage + +Parked: TypeAlias = tuple[object, ...] +PartialAnswer: TypeAlias = Callable[[], object] + + +@dataclass(frozen=True, slots=True) +class _AssembledRound: + response: ModelResponse + usage: Usage + cache_hit: bool + parked: Parked + + +@dataclass(frozen=True, slots=True) +class _PricedRound: + response: ModelResponse + usage: Usage + price: UncachedPrice + cache_hit: bool + + @property + def spend(self) -> BilledRoundsSpend: + return BilledRoundsSpend(self.usage, self.price.cost or 0.0, self.price.cost_breakdown) + + +@dataclass(frozen=True, slots=True) +class _HeldUsageChunk: + chunk: ModelResponseStream + usage: Usage + + +@dataclass(frozen=True, slots=True) +class _FirstRound: + priced: _PricedRound + parked: Parked + logged_alone: Parked + banked: LiteLLMLoggingObj | None = None + usage_chunk: _HeldUsageChunk | None = None + + def client_usage_chunk(self, billed: ModelResponse | None) -> ModelResponseStream | None: + if self.usage_chunk is None: + return None + usage: Final = ( + self.priced.usage.model_copy(update=MappingProxyType({"cost": self.priced.price.cost})) + if billed is None + else merged_chat_usage(self.priced.usage, self.usage_chunk.usage).model_copy( + update=MappingProxyType({"cost": _cost(billed)}) + ) + ) + return self.usage_chunk.chunk.model_copy( + update=MappingProxyType({"id": self.priced.response.id, "usage": usage}) + ) + + +def _cost(response: ModelResponse) -> object: + hidden_params: Final = get_hidden_params(response) + return None if hidden_params is None else hidden_params.get("response_cost") + + +def _assembled_round(parked: Parked | None) -> _AssembledRound | None: + match parked: + case (ModelResponse() as response, cache_hit) if isinstance(usage := getattr(response, "usage", None), Usage): + return _AssembledRound(response, usage, cache_hit is True, parked) + case _: + return None + + +def engaged_round_billing(logging_obj: object) -> "ChatStreamRoundBilling | None": + return ChatStreamRoundBilling(logging_obj) if isinstance(logging_obj, LiteLLMLoggingObj) else None + + +class ChatStreamRoundBilling: + def __init__(self, logging_obj: LiteLLMLoggingObj) -> None: + self._logging_obj: Final = logging_obj + self._first: _FirstRound | None = None + self._partial_follow_up: PartialAnswer | None = None + self._reading_loop: asyncio.AbstractEventLoop | None = None + + @contextmanager + def armed_while_reading(self) -> Generator[None]: + armed: Final = getattr(self._logging_obj, DISPATCH_RELEASE, None) is None + if armed: + setattr(self._logging_obj, DISPATCH_RELEASE, self._log_success) + self._reading_loop = asyncio.get_running_loop() + try: + yield + finally: + if armed and self._peek_parked() is None: + self._disarm() + + def complete_first_round(self) -> None: + assembled: Final = _assembled_round(self._peek_parked()) + if assembled is not None: + self._first = _FirstRound(self._priced(assembled), assembled.parked, assembled.parked) + setattr(self._logging_obj, HELD_ROUND_SPEND, self._held_round_one_spend) + + def bank_first_round(self) -> BilledRoundsSpend | None: + self._bank_apart_from_the_follow_up() + return None if self._first is None else self._first.priced.spend + + def client_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None: + usage: Final[object] = getattr(chunk, "usage", None) + if self._first is None or not isinstance(usage, Usage): + return chunk + self._first = replace(self._first, usage_chunk=_HeldUsageChunk(chunk, usage)) + return None + + def finish(self) -> ModelResponseStream | None: + first: Final = self._first + billed: Final = self._release_rounds() + return None if first is None else first.client_usage_chunk(billed) + + def fail(self) -> None: + self._bank_apart_from_the_follow_up() + first: Final = self._first + if first is None or first.banked is None or not self._holds_round_one_alone(first): + self.finish() + return + self._first = None + setattr(self._logging_obj, PARKED_DISPATCH, None) + self._log_through_the_release((*first.parked, first.banked.guardrail_scoped_copy())) + + def follow_up_started(self, partial_follow_up: PartialAnswer) -> None: + self._partial_follow_up = partial_follow_up + setattr(self._logging_obj, IN_FLIGHT_ROUND, self.bill_in_flight_round) + + async def bill_in_flight_round(self) -> None: + partial_follow_up: Final = self._partial_follow_up + if partial_follow_up is None or self._first_round_held_alone() is None: + return + final_round: Final = _assembled_round((await asyncify(partial_follow_up)(), False)) + first: Final = self._first_round_held_alone() + if final_round is None or first is None: + return + self._first = None + setattr( + self._logging_obj, PARKED_DISPATCH, self._billed_for_both_rounds(first.priced, self._priced(final_round)) + ) + + def release_abandoned(self) -> None: + if not self._owns_release(): + return + if is_event_loop_running() or self._idle_reading_loop() is not None: + self.finish() + return + setattr(self._logging_obj, PARKED_DISPATCH, None) + self._first = None + self._disarm() + + def _bank_apart_from_the_follow_up(self) -> None: + first: Final = self._first + if first is None or first.banked is not None or self._peek_parked() is not first.parked: + return + banked: Final = self._logging_obj.attempt_scoped_copy() + if not self._owns_release(): + self._first = replace(first, banked=banked) + return + logged_alone: Final = (*first.parked, banked) + self._first = replace(first, logged_alone=logged_alone, banked=banked) + setattr(self._logging_obj, PARKED_DISPATCH, logged_alone) + + def _release_rounds(self) -> ModelResponse | None: + parked: Final = self._take_parked() + first: Final = self._first + self._first = None + if parked is None: + self._disarm() + return None + if first is None or parked is first.logged_alone: + self._release(parked) + return None + final_round: Final = _assembled_round(parked) + if final_round is None: + self._release(first.logged_alone) + return None + billed, cache_hit = self._billed_for_both_rounds(first.priced, self._priced(final_round)) + self._release((billed, cache_hit)) + return billed + + def _billed_for_both_rounds(self, first: _PricedRound, final: _PricedRound) -> tuple[ModelResponse, bool]: + first_breakdown: Final = first.price.cost_breakdown + final_breakdown: Final = final.price.cost_breakdown + rounds: Final = billed_for_every_round(first.response, first_breakdown, final.response, final_breakdown) + usage: Final[object] = getattr(rounds, "usage", None) + charged_usage: Final = ( + usage.model_copy(update=MappingProxyType({"cost": _cost(rounds)})) if isinstance(usage, Usage) else usage + ) + billed: Final = rounds.model_copy(update=MappingProxyType({"id": first.response.id, "usage": charged_usage})) + cache_hit: Final = first.cache_hit and final.cache_hit + set_hidden_params(billed, {**(get_hidden_params(rounds) or {}), "cache_hit": cache_hit}) + self._logging_obj.cost_breakdown = summed_cost_breakdown( + first.response, first_breakdown, final.response, final_breakdown + ) + return billed, cache_hit + + def _priced(self, assembled: _AssembledRound) -> _PricedRound: + served: Final = assembled.cache_hit or served_from_cache(assembled.response) + price: Final = SERVED_FROM_CACHE if served else self._logging_obj.uncached_price(assembled.response) + priced: Final = assembled.response.model_copy() + set_hidden_params( + priced, + {**(get_hidden_params(assembled.response) or {}), "response_cost": price.cost, "cache_hit": served}, + ) + return _PricedRound(priced, assembled.usage, price, served) + + def _peek_parked(self) -> Parked | None: + parked: Final[Parked | None] = getattr(self._logging_obj, PARKED_DISPATCH, None) + return parked if isinstance(parked, tuple) else None + + def _take_parked(self) -> Parked | None: + parked: Final = self._peek_parked() + setattr(self._logging_obj, PARKED_DISPATCH, None) + return parked + + def _first_round_held_alone(self) -> _FirstRound | None: + first: Final = self._first + return first if first is not None and self._holds_round_one_alone(first) else None + + def _held_round_one_spend(self) -> BilledRoundsSpend | None: + first: Final = self._first_round_held_alone() + return None if first is None else first.priced.spend + + def _holds_round_one_alone(self, first: _FirstRound) -> bool: + parked: Final = self._peek_parked() + return parked is first.parked or parked is first.logged_alone + + def _owns_release(self) -> bool: + release: Final[object] = getattr(self._logging_obj, DISPATCH_RELEASE, None) + return release == self._log_success + + def _release(self, parked: Parked) -> None: + if getattr(self._logging_obj, DISPATCH_RELEASE, None) is not None and not self._owns_release(): + setattr(self._logging_obj, PARKED_DISPATCH, parked) + return + self._log_through_the_release(parked) + + def _log_through_the_release(self, parked: Parked) -> None: + armed: Final[DeferredStreamRelease | None] = getattr(self._logging_obj, DISPATCH_RELEASE, None) + setattr(self._logging_obj, DISPATCH_RELEASE, None) + logged: Final = (self._log_success if armed is None else armed)(*parked) + idle_reading_loop: Final = None if is_event_loop_running() else self._idle_reading_loop() + if idle_reading_loop is not None: + idle_reading_loop.run_until_complete(logged) + return + spawn_logging_task(asyncio.get_running_loop(), logged, task_name="Chat completions MCP stream success log") + + def _idle_reading_loop(self) -> asyncio.AbstractEventLoop | None: + loop: Final = self._reading_loop + return loop if loop is not None and not loop.is_closed() and not loop.is_running() else None + + def _disarm(self) -> None: + if self._owns_release(): + setattr(self._logging_obj, DISPATCH_RELEASE, None) + + async def _log_success(self, *parked: object) -> None: + success: Final = parked_success(parked) + if success is not None: + await success.dispatch(self._logging_obj) diff --git a/litellm/responses/mcp/parked_stream_dispatch.py b/litellm/responses/mcp/parked_stream_dispatch.py new file mode 100644 index 00000000000..792363f0bab --- /dev/null +++ b/litellm/responses/mcp/parked_stream_dispatch.py @@ -0,0 +1,72 @@ +from collections.abc import Coroutine +from dataclasses import dataclass +from typing import Final, Protocol + +from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, UncachedPrice +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +PARKED_DISPATCH: Final = "_deferred_stream_complete_args" +DISPATCH_RELEASE: Final = "_on_deferred_stream_complete" +IN_FLIGHT_ROUND: Final = "_bill_in_flight_round_on_disconnect" +HELD_ROUND_SPEND: Final = "_held_round_spend_for_failure" +SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None) + + +class DispatchesSuccessHandlers(Protocol): + async def dispatch_success_handlers( + self, + result: object = None, + start_time: object = None, + end_time: object = None, + cache_hit: object = None, + prefer_async_handlers: bool = False, + ) -> None: ... + + +class DeferredStreamRelease(Protocol): + def __call__(self, *parked: object) -> Coroutine[object, object, None]: ... + + +class InFlightRoundBiller(Protocol): + def __call__(self) -> Coroutine[object, object, None]: ... + + +class HeldRoundSpend(Protocol): + def __call__(self) -> BilledRoundsSpend | None: ... + + +def success_dispatcher(logger: LiteLLMLoggingObj) -> DispatchesSuccessHandlers: + return logger + + +@dataclass(frozen=True, slots=True) +class ParkedSuccess: + assembled: object + cache_hit: object + logged_on: LiteLLMLoggingObj | None = None + + async def dispatch(self, request_logger: LiteLLMLoggingObj) -> None: + logger: Final = request_logger if self.logged_on is None else self.logged_on + await success_dispatcher(logger).dispatch_success_handlers( + self.assembled, cache_hit=self.cache_hit, start_time=None, end_time=None, prefer_async_handlers=True + ) + + +def in_flight_round_biller(logging_obj: object) -> InFlightRoundBiller | None: + biller: Final[InFlightRoundBiller | None] = getattr(logging_obj, IN_FLIGHT_ROUND, None) + return biller if isinstance(logging_obj, LiteLLMLoggingObj) and callable(biller) else None + + +def held_round_spend(logging_obj: object) -> BilledRoundsSpend | None: + held: Final[HeldRoundSpend | None] = getattr(logging_obj, HELD_ROUND_SPEND, None) + return held() if isinstance(logging_obj, LiteLLMLoggingObj) and callable(held) else None + + +def parked_success(parked: tuple[object, ...]) -> ParkedSuccess | None: + match parked: + case (assembled, cache_hit): + return ParkedSuccess(assembled, cache_hit) + case (assembled, cache_hit, LiteLLMLoggingObj() as logged_on): + return ParkedSuccess(assembled, cache_hit, logged_on) + case _: + return None diff --git a/litellm/responses/mcp/stream_round_billing.py b/litellm/responses/mcp/stream_round_billing.py index 443147230bf..5cb256a8ac0 100644 --- a/litellm/responses/mcp/stream_round_billing.py +++ b/litellm/responses/mcp/stream_round_billing.py @@ -2,11 +2,18 @@ import asyncio from collections.abc import Callable, Coroutine, Sequence from dataclasses import dataclass from datetime import datetime -from typing import Final, Protocol, TypeAlias +from typing import Final, TypeAlias from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, BilledRoundsSpend, UncachedPrice from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.responses.mcp.parked_stream_dispatch import ( + DISPATCH_RELEASE, + PARKED_DISPATCH, + SERVED_FROM_CACHE, + parked_success, + success_dispatcher, +) from litellm.responses.mcp.round_billing import billed_for_every_round, merged_chat_usage, summed_cost_breakdown from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator, spawn_logging_task from litellm.responses.utils import ResponseAPILoggingUtils @@ -16,25 +23,6 @@ from litellm.types.utils import CostBreakdown, ModelResponse, Usage Parked: TypeAlias = tuple[object, ...] RestoresHeaders: TypeAlias = Callable[[ResponsesAPIStreamingResponse], None] -_PARKED_DISPATCH: Final = "_deferred_stream_complete_args" -_DISPATCH_RELEASE: Final = "_on_deferred_stream_complete" -_SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None) - - -class _DispatchesSuccessHandlers(Protocol): - async def dispatch_success_handlers( - self, - result: object = None, - start_time: object = None, - end_time: object = None, - cache_hit: object = None, - prefer_async_handlers: bool = False, - ) -> None: ... - - -def _success_dispatcher(logger: LiteLLMLoggingObj) -> _DispatchesSuccessHandlers: - return logger - def _event_loop_running() -> bool: try: @@ -198,8 +186,8 @@ class StreamedRoundBilling: self._billed: _BilledRounds | None = None self._banked: Parked | None = None self._banked_row: _RoundRow | None = None - if getattr(logging_obj, _DISPATCH_RELEASE, None) is None: - setattr(logging_obj, _DISPATCH_RELEASE, self._run_parked) + if getattr(logging_obj, DISPATCH_RELEASE, None) is None: + setattr(logging_obj, DISPATCH_RELEASE, self._run_parked) def complete_round( self, @@ -286,7 +274,7 @@ class StreamedRoundBilling: def _price(self, response: ResponsesAPIResponse, round_parked: Parked | None) -> UncachedPrice: if served_from_cache(response): - return _SERVED_FROM_CACHE + return SERVED_FROM_CACHE match round_parked: case (ModelResponse() as assembled, bool()): return self._logging_obj.uncached_price(assembled) @@ -297,7 +285,7 @@ class StreamedRoundBilling: self._banked = parked self._banked_row = row if parked is not None: - setattr(self._logging_obj, _PARKED_DISPATCH, parked) + setattr(self._logging_obj, PARKED_DISPATCH, parked) def _bank_apart_from_later_rounds(self, billed: _BilledRounds) -> None: row: Final = self._banked_row @@ -320,7 +308,7 @@ class StreamedRoundBilling: @staticmethod async def _log_success(success: _SuccessLog) -> None: success.carry_bridged_usage() - await _success_dispatcher(success.logger).dispatch_success_handlers( + await success_dispatcher(success.logger).dispatch_success_handlers( success.row.logged, start_time=success.logger.start_time, end_time=success.row.end_time, @@ -329,10 +317,10 @@ class StreamedRoundBilling: ) def _take_parked(self) -> Parked | None: - parked: Final[Parked | None] = getattr(self._logging_obj, _PARKED_DISPATCH, None) + parked: Final[Parked | None] = getattr(self._logging_obj, PARKED_DISPATCH, None) if not isinstance(parked, tuple): return None - setattr(self._logging_obj, _PARKED_DISPATCH, None) + setattr(self._logging_obj, PARKED_DISPATCH, None) return parked async def _run_parked(self, *parked: object) -> None: @@ -341,29 +329,27 @@ class StreamedRoundBilling: await dispatch case (dispatch,) if asyncio.iscoroutine(dispatch): await dispatch - case (assembled, cache_hit): - await _success_dispatcher(self._logging_obj).dispatch_success_handlers( - assembled, cache_hit=cache_hit, start_time=None, end_time=None, prefer_async_handlers=True - ) + case _ if (success := parked_success(parked)) is not None: + await success.dispatch(self._logging_obj) case _: return def _owns_release(self) -> bool: - release: Final[object] = getattr(self._logging_obj, _DISPATCH_RELEASE, None) + release: Final[object] = getattr(self._logging_obj, DISPATCH_RELEASE, None) return release == self._run_parked def _release(self, parked: Parked) -> None: - if getattr(self._logging_obj, _DISPATCH_RELEASE, None) is not None and not self._owns_release(): - setattr(self._logging_obj, _PARKED_DISPATCH, parked) + if getattr(self._logging_obj, DISPATCH_RELEASE, None) is not None and not self._owns_release(): + setattr(self._logging_obj, PARKED_DISPATCH, parked) return - setattr(self._logging_obj, _DISPATCH_RELEASE, None) + setattr(self._logging_obj, DISPATCH_RELEASE, None) spawn_logging_task( asyncio.get_running_loop(), self._run_parked(*parked), task_name="Responses MCP stream success logging" ) def _disarm(self) -> None: if self._owns_release(): - setattr(self._logging_obj, _DISPATCH_RELEASE, None) + setattr(self._logging_obj, DISPATCH_RELEASE, None) @staticmethod def _close(parked: Parked | None) -> None: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index f0bb2201162..fb91ba6a618 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -2145,18 +2145,32 @@ async def _enforce_frame_project_quota( ) -async def _admit_frame_turn(turn_admission: Callable[[], Awaitable[None]] | None, raw_message: str) -> Exception | None: +@dataclass(frozen=True, slots=True) +class _AdmittedTurn: + reservation: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) + + +@dataclass(frozen=True, slots=True) +class _RefusedTurn: + rejection: Exception + + +ResponsesWsTurnAdmission: TypeAlias = Callable[[], Awaitable[Mapping[str, object]]] + + +async def _admit_frame_turn( + turn_admission: ResponsesWsTurnAdmission | None, raw_message: str +) -> _AdmittedTurn | _RefusedTurn: frame: Final = _json_object_or_none(raw_message) if turn_admission is None or frame is None or frame.get("type") != "response.create": - return None + return _AdmittedTurn() try: - await turn_admission() + return _AdmittedTurn(await turn_admission()) except RateLimitError as refusal: - return refusal + return _RefusedTurn(refusal) except Exception as e: verbose_logger.exception("Responses WS turn admission failed: %s", e) - return e - return None + return _RefusedTurn(e) def _turn_refusal_frame(refusal: RateLimitError) -> str: @@ -2191,6 +2205,7 @@ RESPONSES_WS_MASKABLE_TEXT_BLOCK_TYPES: Final = frozenset({"input_text", "output _RESPONSES_WS_FAILURE_EVENT_TYPES: Final = frozenset({"error", "response.failed"}) _PER_TURN_METADATA_KEYS: Final = frozenset({"metadata", "litellm_metadata"}) +_NO_IO_TOKEN_RESERVATION: Final[Mapping[str, object]] = MappingProxyType({}) _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -2318,6 +2333,17 @@ def _set_turn_guardrail_information( _replace_guardrail_information(litellm_params.get(key), guardrail_information) +def _set_turn_io_token_reservation(turn_logging_obj: LiteLLMLoggingObj, reservation: Mapping[str, object]) -> None: + litellm_params: Final = _json_object_or_none( + turn_logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped + ) + if litellm_params is None or not reservation: + return + for bucket in (litellm_params.get(key) for key in _PER_TURN_METADATA_KEYS): + if _is_json_object(bucket): + bucket.update(reservation) + + def _replace_guardrail_information(bucket: object, guardrail_information: tuple[Mapping[str, object], ...]) -> None: if not _is_json_object(bucket): return @@ -2458,7 +2484,7 @@ class ResponsesWebSocketStreaming: authorized_model: str | None = None, custom_llm_provider: str | None = None, request_defaults: ResponsesWebSocketRequestDefaults | None = None, - turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_admission: ResponsesWsTurnAdmission | None = None, frame_guard: ResponsesWsFrameGuard | None = None, ): self.websocket = websocket @@ -2480,10 +2506,13 @@ class ResponsesWebSocketStreaming: # response.create frame to prevent deployment-substitution attacks. self.authorized_model: str | None = authorized_model self.request_defaults: ResponsesWebSocketRequestDefaults | None = request_defaults - self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission + self.turn_admission: ResponsesWsTurnAdmission | None = turn_admission def _start_turn( - self, client_frame: str | None, guardrail_information: tuple[Mapping[str, object], ...] = () + self, + client_frame: str | None, + guardrail_information: tuple[Mapping[str, object], ...] = (), + io_token_reservation: Mapping[str, object] = _NO_IO_TOKEN_RESERVATION, ) -> None: start_time: Final = datetime.now() # noqa: DTZ005 # naive to match the logging pipeline start_time turn_logging_obj: Final = self.logging_obj.copy_for_new_call( @@ -2491,6 +2520,7 @@ class ResponsesWebSocketStreaming: ) self._detach_budget_reservation() _set_turn_guardrail_information(turn_logging_obj, guardrail_information) + _set_turn_io_token_reservation(turn_logging_obj, io_token_reservation) turn_logging_obj.model_call_details["messages"] = list(_logged_user_messages(client_frame)) turn_logging_obj.pre_call(input=client_frame, api_key="") self._open_turns = (*self._open_turns, _ResponsesWsTurn(logging_obj=turn_logging_obj)) @@ -2510,11 +2540,14 @@ class ResponsesWebSocketStreaming: } def _record_client_frame( - self, client_frame: str, guardrail_information: tuple[Mapping[str, object], ...] = () + self, + client_frame: str, + guardrail_information: tuple[Mapping[str, object], ...] = (), + io_token_reservation: Mapping[str, object] = _NO_IO_TOKEN_RESERVATION, ) -> None: frame: Final = _json_object_or_none(client_frame) if frame is not None and frame.get("type") == "response.create": - self._start_turn(client_frame, guardrail_information) + self._start_turn(client_frame, guardrail_information, io_token_reservation) async def _guard_frame(self, message: str) -> GuardedResponsesWsFrame | None: """Run the proxy's pre_call guardrails on one ``response.create`` frame. @@ -2533,15 +2566,18 @@ class ResponsesWebSocketStreaming: return None async def _forward_guarded_frame( - self, message: str, turn_admission: Callable[[], Awaitable[None]] | None = None + self, message: str, turn_admission: ResponsesWsTurnAdmission | None = None ) -> None: guarded: Final = await self._guard_frame(message) - if guarded is None or not await self._admit_or_reject_turn(guarded, message, turn_admission): + if guarded is None: + return + admitted: Final = await self._admit_or_reject_turn(guarded, message, turn_admission) + if admitted is None: return if guarded.pii_tokens: self._remember_pii_tokens(guarded.pii_tokens) masked: Final = await self._mask_response_create(guarded.frame) - self._record_client_frame(masked, guarded.guardrail_information) + self._record_client_frame(masked, guarded.guardrail_information, admitted.reservation) await self.backend_ws.send(masked) def _remember_pii_tokens(self, pii_tokens: Mapping[str, str]) -> None: @@ -2941,15 +2977,17 @@ class ResponsesWebSocketStreaming: return True async def _admit_or_reject_turn( - self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: Callable[[], Awaitable[None]] | None - ) -> bool: - rejection: Final = await _admit_frame_turn(turn_admission, message) - if rejection is None: - return True - if self._failure is None: - self._failure = rejection - await _reject_guarded_turn(self.websocket, guarded, rejection) - return False + self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: ResponsesWsTurnAdmission | None + ) -> _AdmittedTurn | None: + verdict: Final = await _admit_frame_turn(turn_admission, message) + match verdict: + case _AdmittedTurn(): + return verdict + case _RefusedTurn(rejection=rejection): + if self._failure is None: + self._failure = rejection + await _reject_guarded_turn(self.websocket, guarded, rejection) + return None async def _send_turn_refusal(self, refusal: RateLimitError) -> None: try: @@ -3052,7 +3090,7 @@ class ManagedResponsesWebSocketHandler: custom_llm_provider: str | None = None, first_message: str | None = None, quota_callbacks: Sequence[ProjectQuotaCallback] | None = None, - turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_admission: ResponsesWsTurnAdmission | None = None, frame_guard: ResponsesWsFrameGuard | None = None, guardrail_callbacks: Sequence[PresidioGuardrailCallback] | None = None, **kwargs: object, @@ -3076,7 +3114,7 @@ class ManagedResponsesWebSocketHandler: self._connection_provider = self._resolve_provider(model) or custom_llm_provider self.first_message = first_message self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else () - self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission + self.turn_admission: ResponsesWsTurnAdmission | None = turn_admission self._failure: Exception | None = None # Carry through safe pass-through kwargs (e.g. extra_headers) self.extra_kwargs: dict[str, object] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS} @@ -3473,7 +3511,7 @@ class ManagedResponsesWebSocketHandler: # ------------------------------------------------------------------ async def _process_response_create( - self, raw_message: str, turn_admission: Callable[[], Awaitable[None]] | None = None + self, raw_message: str, turn_admission: ResponsesWsTurnAdmission | None = None ) -> None: """ Parse one ``response.create`` event, call ``litellm.aresponses(stream=True)``, @@ -3520,11 +3558,11 @@ class ManagedResponsesWebSocketHandler: guarded: Final = await self._guard_frame(raw_message) if guarded is None: return - rejection: Final = await _admit_frame_turn(turn_admission, raw_message) - if rejection is not None: + verdict: Final = await _admit_frame_turn(turn_admission, raw_message) + if isinstance(verdict, _RefusedTurn): if self._failure is None: - self._failure = rejection - await _reject_guarded_turn(self.websocket, guarded, rejection) + self._failure = verdict.rejection + await _reject_guarded_turn(self.websocket, guarded, verdict.rejection) return if guarded.pii_tokens: self._pii_tokens = {**self._pii_tokens, **guarded.pii_tokens} @@ -3555,6 +3593,11 @@ class ManagedResponsesWebSocketHandler: ) self._update_proxy_request(call_kwargs, requested_model or self.model_group or model) call_kwargs.update(self.extra_kwargs) + turn_litellm_metadata: Final = call_kwargs.get("litellm_metadata") + call_kwargs["litellm_metadata"] = { + **(turn_litellm_metadata if _is_json_object(turn_litellm_metadata) else {}), + **verdict.reservation, + } try: terminal_event: Final = await self._stream_and_forward(model, call_kwargs) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index b0db0516658..1a97c1cb48c 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1,10 +1,11 @@ import base64 import os import re -from collections.abc import Callable, Collection, Iterable, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass from functools import reduce from types import MappingProxyType -from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload +from typing import Any, Final, NoReturn, Optional, TypeVar, Union, cast, get_type_hints, overload from pydantic import BaseModel, TypeAdapter from typing_extensions import TypedDict as ExtraItemsTypedDict @@ -15,7 +16,13 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils -from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel +from litellm.llms.base_llm.responses.transformation import ( + ANY_REASONING_FIELD_VALUES, + REASONING_SUMMARY_VALUES, + BaseResponsesAPIConfig, + ReasoningFieldValues, + ReasoningSummaryLevel, +) from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.openai import ( AllMessageValues, @@ -1411,9 +1418,6 @@ def reasoning_with_summary_default( return {**reasoning} if summary is None else {**reasoning, "summary": summary} -_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None)) - - def _openai_model_info(model: str) -> ModelInfo | None: from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration @@ -1425,24 +1429,115 @@ def _declared_values(values: Sequence[str] | None) -> frozenset[str | None] | No return None if values is None else frozenset(values) -def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | None] | None]: +_REFUSABLE_REASONING_FIELDS: Final = frozenset({"mode", "context"}) + + +def _openai_reasoning_field_values(model: str) -> ReasoningFieldValues: model_info: Final = _openai_model_info(model) if model_info is None: - return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None}) + return ANY_REASONING_FIELD_VALUES if not model_info.get("supports_reasoning"): return MappingProxyType({}) return MappingProxyType( { - "summary": _SUMMARY_VALUES, + "summary": REASONING_SUMMARY_VALUES, "mode": _declared_values(model_info.get("reasoning_modes")), "context": _declared_values(model_info.get("reasoning_contexts")), } ) -def _openai_accepts_reasoning_field( - key: str, value: object, values_by_field: Mapping[str, frozenset[str | None] | None] -) -> bool: +_CATALOG_REASONING_FIELDS: Final = (("mode", "reasoning_modes"), ("context", "reasoning_contexts")) + + +def _catalog_values(values: object) -> frozenset[str | None]: + return frozenset(value for value in iter_items(values) if isinstance(value, str)) + + +def _declared_reasoning_values(row: Mapping[str, object] | None) -> Mapping[str, frozenset[str | None]]: + if row is None: + return {} + return { + field: _catalog_values(row[key]) for field, key in _CATALOG_REASONING_FIELDS if isinstance(row.get(key), list) + } + + +def _with_declared_reasoning_values( + defaults: ReasoningFieldValues | None, + model: str, + custom_llm_provider: str, + deployment_model_info: Mapping[str, object] | None, +) -> ReasoningFieldValues | None: + catalog: Final = cast(Mapping[str, object], litellm.model_cost) # cast-ok: read-only view of the legacy registry + catalog_row: Final = as_str_mapping(catalog.get(f"{custom_llm_provider}/{model.removeprefix('responses/')}")) + overrides: Final = { + **_declared_reasoning_values(catalog_row), + **_declared_reasoning_values(deployment_model_info), + } + if not overrides: + return defaults + return MappingProxyType({**(ANY_REASONING_FIELD_VALUES if defaults is None else defaults), **overrides}) + + +def _is_version_number(part: str) -> bool: + return part.isdigit() and len(part) <= 2 + + +def _splits_a_version(parts: Sequence[str], end: int) -> bool: + return end < len(parts) and _is_version_number(parts[end - 1]) and _is_version_number(parts[end]) + + +def _hyphen_runs_longest_first(name: str) -> Iterator[str]: + parts: Final = name.rsplit("/", 1)[-1].split("-") + for length in range(len(parts), 0, -1): + yield from ( + "-".join(parts[start : start + length]) + for start in range(len(parts) - length + 1) + if not _splits_a_version(parts, start + length) + ) + + +def _openai_catalog_family(name: str) -> str | None: + runs: Final = _hyphen_runs_longest_first(name) + return next((run for run in runs if OpenAIGPTConfig.is_openai_catalog_model(run)), None) + + +def _openai_catalog_name(name: str) -> str | None: + last_segment: Final = name.rsplit("/", 1)[-1] + return last_segment if OpenAIGPTConfig.is_openai_catalog_model(last_segment) else None + + +def _openai_catalog_model(model: str, base_model: str | None, names_wrap_models: bool) -> str | None: + find: Final = _openai_catalog_family if names_wrap_models else _openai_catalog_name + families: Final = (find(name) for name in (base_model, model) if name) + return next((family for family in families if family is not None), None) + + +def _default_reasoning_field_values( + model: str, base_model: str | None, responses_config: BaseResponsesAPIConfig | None +) -> ReasoningFieldValues | None: + declared: Final = None if responses_config is None else responses_config.reasoning_field_values() + if declared is not None: + return declared + names_wrap_models: Final = responses_config is not None and responses_config.deployment_names_wrap_openai_models() + catalog_model: Final = _openai_catalog_model(model, base_model, names_wrap_models) + if catalog_model is not None: + return _openai_reasoning_field_values(catalog_model) + return None if responses_config is None else responses_config.unresolved_model_reasoning_field_values(model) + + +def _reasoning_field_values( + model: str, + base_model: str | None, + custom_llm_provider: str, + responses_config: BaseResponsesAPIConfig | None, + deployment_model_info: Mapping[str, object] | None, +) -> ReasoningFieldValues | None: + defaults: Final = _default_reasoning_field_values(model, base_model, responses_config) + return _with_declared_reasoning_values(defaults, model, custom_llm_provider, deployment_model_info) + + +def _accepts_reasoning_field(key: str, value: object, values_by_field: ReasoningFieldValues) -> bool: if key == "effort": return True if key not in values_by_field: @@ -1451,11 +1546,8 @@ def _openai_accepts_reasoning_field( return allowed is None or (isinstance(value, str | None) and value in allowed) -def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]: - values_by_field: Final = _openai_reasoning_field_values(model) - return { - key: value for key, value in reasoning.items() if _openai_accepts_reasoning_field(key, value, values_by_field) - } +def _accepted_reasoning(reasoning: Mapping[str, object], values_by_field: ReasoningFieldValues) -> Mapping[str, object]: + return {key: value for key, value in reasoning.items() if _accepts_reasoning_field(key, value, values_by_field)} def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]: @@ -1464,26 +1556,66 @@ def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]: return tuple(path for path in iter_items(additional_drop_params) if isinstance(path, str)) -def openai_bridge_reasoning( +@dataclass(frozen=True, slots=True) +class RefusedReasoningFields: + refused: tuple[str, ...] + + +def raise_refused_reasoning_fields(refusal: RefusedReasoningFields, model: str, custom_llm_provider: str) -> NoReturn: + raise litellm.UnsupportedParamsError( + message=( + f"{custom_llm_provider} does not support parameters: " + f"{[f'reasoning_effort.{key}' for key in refusal.refused]}, for model={model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=custom_llm_provider, + model=model, + ) + + +def bridge_reasoning( *, requested: Mapping[str, object], effort_kept: bool, summary_alias: object, model: str, + base_model: str | None, + custom_llm_provider: str, + api_base: str | None, + deployment_model_info: Mapping[str, object] | None, drop_params: bool, additional_drop_params: object, -) -> Mapping[str, object] | None: +) -> Mapping[str, object] | RefusedReasoningFields | None: + from litellm.utils import ProviderConfigManager + drop_paths: Final = _additional_drop_paths(additional_drop_params) if "reasoning_effort" in drop_paths: return None kept: Final = requested if effort_kept else {key: value for key, value in requested.items() if key != "effort"} - with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, "openai") - accepted: Final = ( - _openai_accepted_reasoning(with_summary, model) - if drop_params and OpenAIGPTConfig.is_openai_catalog_model(model) - else {**with_summary} + with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, custom_llm_provider) + sent: Final = ( + as_str_mapping( + _apply_nested_drop_params({"reasoning_effort": {**with_summary}}, list(drop_paths)).get("reasoning_effort") + ) + or {} ) - remaining: Final = as_str_mapping( - _apply_nested_drop_params({"reasoning_effort": accepted}, list(drop_paths)).get("reasoning_effort") + responses_config: Final = ProviderConfigManager.get_provider_responses_api_config( + custom_llm_provider, model, api_base ) - return remaining or None + values_by_field: Final = _reasoning_field_values( + model, base_model, custom_llm_provider, responses_config, deployment_model_info + ) + accepted: Final = sent if values_by_field is None else _accepted_reasoning(sent, values_by_field) + refused: Final = tuple( + key + for key, value in sent.items() + if key in _REFUSABLE_REASONING_FIELDS and key not in accepted and value is not None + ) + if ( + refused + and not drop_params + and responses_config is not None + and responses_config.ignores_unsupported_reasoning_fields() + ): + return RefusedReasoningFields(refused) + return (accepted if drop_params else sent) or None diff --git a/litellm/router.py b/litellm/router.py index 20af2e5649e..233e7581a40 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -255,6 +255,7 @@ from litellm.router_utils.clientside_credential_handler import ( from litellm.router_utils.common_utils import ( MID_STREAM_CONTINUATION_KWARG, as_retry_skipped_deployment_ids, + deployment_model_id_pin, deployments_the_caller_may_use, filter_team_based_models, filter_web_search_deployments, @@ -377,6 +378,7 @@ from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check impor EncryptedContentAffinityCheck, ) from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( + async_admit_io_token_turn, async_io_token_refund_failure, build_io_token_rate_limit_headers, deployment_has_io_token_limits, @@ -5112,6 +5114,7 @@ class Router(DeprecatedRouterMembers): kwargs: dict, function_name: str | None = None, splits_stream_timeout: bool = False, + opens_responses_socket: bool = False, ) -> None: """ 5 jobs: @@ -5201,7 +5204,12 @@ class Router(DeprecatedRouterMembers): # here before it's wiped below, instead of relying on that attempt's # (possibly still-pending) failure event to do it. refund_stale_reservation_before_retry(self._core_cache, kwargs) - set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment)) + set_io_token_rate_limit_request_kwargs( + kwargs, + store_in_context=deployment_has_io_token_limits(deployment), + opens_responses_socket=opens_responses_socket, + metadata_channel=metadata_variable_name, + ) kwargs[metadata_variable_name].setdefault( ROUTING_REQUEST_TAGS_METADATA_KEY, @@ -5316,6 +5324,9 @@ class Router(DeprecatedRouterMembers): ): kwargs[STREAM_USAGE_INJECTED_FIELD] = True self._update_kwargs_with_default_litellm_params(kwargs=kwargs, metadata_variable_name=metadata_variable_name) + pinned_model_id: Final = deployment_model_id_pin(deployment_context) + if pinned_model_id is not None: + kwargs["model_id"] = pinned_model_id # rebind-ok: this method's contract is filling the caller's kwargs hop_kwargs: Final = _FALLBACK_HOP_ADAPTER.validate_python(kwargs) if _is_fallback_hop(hop_kwargs): EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( @@ -6242,6 +6253,9 @@ class Router(DeprecatedRouterMembers): kwargs[k] = v elif k == "metadata": kwargs[k].update(v) + pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data)) + if pinned_model_id is not None: + kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults # call via litellm.completion() return litellm.text_completion( @@ -6608,6 +6622,7 @@ class Router(DeprecatedRouterMembers): kwargs=kwargs, function_name=function_name, splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + opens_responses_socket=handler_name == "_aresponses_websocket", ) data: Final = deployment["litellm_params"].copy() @@ -10458,9 +10473,20 @@ class Router(DeprecatedRouterMembers): return deployment if current is None else current.model_dump(exclude_none=True) async def async_admit_deployment_turn( - self, model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object] - ) -> None: + self, + model_group: str, + opened_on: Mapping[str, object], + request_kwargs: Mapping[str, object], + ) -> Mapping[str, object]: deployment: Final = self._deployment_as_configured_now(opened_on) + rate_limit_check: Final = next( + (callback for callback in self._routing_callbacks() if isinstance(callback, ModelRateLimitingCheck)), None + ) + io_token_admission: Final[Mapping[str, object]] = ( + MappingProxyType({}) + if rate_limit_check is None + else await async_admit_io_token_turn(rate_limit_check.dual_cache, deployment, get_utc_datetime()) + ) budget_limiter: Final = self._core_get_router_deployment_budget_limiter() if budget_limiter is not None: try: @@ -10481,6 +10507,7 @@ class Router(DeprecatedRouterMembers): await self.async_routing_strategy_pre_call_checks( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict deployment=dict(without_io_token_limits(deployment)), parent_otel_span=None ) + return io_token_admission async def _admit_turn_through_routing_strategy( self, model_group: str, deployment: Mapping[str, object], request_kwargs: Mapping[str, object] diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 2451422b5e6..a06860e7786 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -16,6 +16,7 @@ from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure +from litellm.llms.sagemaker.common_utils import deployment_inference_component from litellm.types.router import ( CredentialLiteLLMParams, Deployment, @@ -522,6 +523,16 @@ PROVIDER_SPECIFIC_CAPABILITY_FIELDS: Final = frozenset(ProviderSpecificModelInfo _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +def deployment_model_id_pin(deployment_litellm_params: Mapping[str, object]) -> object | None: + clientside_params: Final = deployment_litellm_params.get("configurable_clientside_auth_params") + if isinstance(clientside_params, list | tuple) and "model_id" in clientside_params: + return None + own_model_id: Final = deployment_litellm_params.get("model_id") + if own_model_id is not None: + return own_model_id + return deployment_inference_component(deployment_litellm_params) + + def is_model_capability_field(name: str) -> bool: return name in PROVIDER_SPECIFIC_CAPABILITY_FIELDS or ( name.startswith("supports_") and name in GetModelCostMap.loaded_model_cost_map_fields() diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 7580c7dadd3..80e1f3fe29c 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -14,6 +14,9 @@ from __future__ import annotations import contextlib import contextvars from collections.abc import Mapping, MutableMapping +from dataclasses import dataclass +from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx @@ -23,7 +26,15 @@ from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache +from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION +from litellm.litellm_core_utils.internal_call_metadata import ( + IO_TOKEN_RESERVATION_KEYS, + ITPM_CACHE_KEY, + ITPM_RESERVED_KEY, + OTPM_CACHE_KEY, + OTPM_RESERVED_KEY, +) from litellm.litellm_core_utils.token_counter import deployment_openai_model, offload_token_count from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET @@ -39,18 +50,27 @@ else: RoutingArgsTTL: Final = 60 -_io_token_rate_limit_request_kwargs: Final[contextvars.ContextVar[dict[str, object] | None]] = contextvars.ContextVar( - "io_token_rate_limit_request_kwargs", + +@dataclass(frozen=True, slots=True) +class _IoTokenRateLimitRequest: + kwargs: dict[str, object] | None + opens_responses_socket: bool + metadata_channel: str + + +_io_token_rate_limit_request: Final[contextvars.ContextVar[_IoTokenRateLimitRequest | None]] = contextvars.ContextVar( + "io_token_rate_limit_request", default=None, ) -ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved" -OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved" -ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key" -OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key" - -def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, store_in_context: bool = True) -> None: +def set_io_token_rate_limit_request_kwargs( + kwargs: dict[str, object] | None, + store_in_context: bool = True, + *, + opens_responses_socket: bool = False, + metadata_channel: str, +) -> None: # The reservation sentinels are server-only, but `metadata` is caller # controlled on proxy requests. Strip any client-supplied copies here (this # runs before the router stashes its own reservation) so a forged @@ -64,11 +84,14 @@ def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, sto # every other deployment overwrite the slot with None instead of the # kwargs; overwriting (rather than skipping) also releases a previous # request's kwargs when a context is reused. - _io_token_rate_limit_request_kwargs.set(kwargs if store_in_context else None) + _io_token_rate_limit_request.set( + _IoTokenRateLimitRequest(kwargs, opens_responses_socket, metadata_channel) if store_in_context else None + ) def get_io_token_rate_limit_request_kwargs() -> dict[str, object] | None: - return _io_token_rate_limit_request_kwargs.get() + request: Final = _io_token_rate_limit_request.get() + return None if request is None else request.kwargs def seconds_until_minute_reset() -> int: @@ -224,27 +247,32 @@ def _otpm_reservation( return default if from_deployment is None else min(from_deployment, default) +def _usage_field(container: object, name: str) -> object: + if is_str_keyed_mapping(container): + return container.get(name) + return getattr(container, name, None) + + +def _cached_prompt_tokens(usage: object) -> int: + reported: Final = ( + _usage_field(_usage_field(usage, "prompt_tokens_details"), "cached_tokens"), + _usage_field(_usage_field(usage, "input_tokens_details"), "cached_tokens"), + _usage_field(usage, "cache_read_input_tokens"), + ) + return next((int(tokens) for tokens in reported if isinstance(tokens, int | float) and tokens), 0) + + def _get_usage_tokens(usage: object) -> tuple[int, int, int]: if usage is None: return 0, 0, 0 if hasattr(usage, "prompt_tokens") or hasattr(usage, "input_tokens"): prompt = int(getattr(usage, "prompt_tokens", None) or getattr(usage, "input_tokens", 0) or 0) completion = int(getattr(usage, "completion_tokens", None) or getattr(usage, "output_tokens", 0) or 0) - cached = 0 - details: object = getattr(usage, "prompt_tokens_details", None) - if details is not None: - cached = int(getattr(details, "cached_tokens", 0) or 0) - if not cached: - cached = int(getattr(usage, "cache_read_input_tokens", 0) or 0) - return prompt, completion, cached + return prompt, completion, _cached_prompt_tokens(usage) if isinstance(usage, dict): prompt = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0) completion = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0) - details = usage.get("prompt_tokens_details") or {} - cached = int(details.get("cached_tokens", 0) or 0) if isinstance(details, dict) else 0 - if not cached: - cached = int(usage.get("cache_read_input_tokens") or 0) - return prompt, completion, cached + return prompt, completion, _cached_prompt_tokens(usage) return 0, 0, 0 @@ -298,19 +326,36 @@ def _resolve_reconcile_usage_tokens( if isinstance(standard_logging_object, dict): prompt_tokens = int(standard_logging_object.get("prompt_tokens") or 0) completion_tokens = int(standard_logging_object.get("completion_tokens") or 0) - cached_tokens = 0 - metadata: Final = standard_logging_object.get("metadata") - if isinstance(metadata, dict): - cached_tokens = int(metadata.get("cache_read_input_tokens") or 0) + logged_usage: Final = _usage_field(standard_logging_object.get("metadata"), "usage_object") + logged_cached_tokens: Final = _cached_prompt_tokens(logged_usage) # Same rationale as _usage_is_present: a bare total_tokens with no # prompt/completion breakdown can't be split, so it isn't treated # as resolved usage - the reservation is kept instead of refunded. if prompt_tokens or completion_tokens: - return max(0, prompt_tokens - cached_tokens), completion_tokens, True + return max(0, prompt_tokens - logged_cached_tokens), completion_tokens, True return 0, 0, False +def _reservation( + itpm_reserved: int, otpm_reserved: int, itpm_cache_key: str | None, otpm_cache_key: str | None +) -> Mapping[str, object]: + return MappingProxyType( + { + ITPM_RESERVED_KEY: itpm_reserved, + OTPM_RESERVED_KEY: otpm_reserved, + ITPM_CACHE_KEY: itpm_cache_key, + OTPM_CACHE_KEY: otpm_cache_key, + } + ) + + +def _stash_reservation_in_metadata(request: _IoTokenRateLimitRequest | None, reservation: Mapping[str, object]) -> None: + metadata: Final = None if request is None or not request.kwargs else request.kwargs.get(request.metadata_channel) + if isinstance(metadata, dict): + metadata.update(reservation) + + def _key_for_this_minute(reserved_key: str) -> str: counter, _, _ = reserved_key.rpartition(":") return f"{counter}:{get_utc_datetime().strftime('%H-%M')}" @@ -326,30 +371,6 @@ def _reconcile_increments( ) -def _stash_reservation_in_metadata( - request_kwargs: dict[str, object] | None, - *, - itpm_reserved: int, - otpm_reserved: int, - itpm_cache_key: str | None, - otpm_cache_key: str | None, -) -> None: - if not request_kwargs: - return - reservation: Final = { - ITPM_RESERVED_KEY: itpm_reserved, - OTPM_RESERVED_KEY: otpm_reserved, - ITPM_CACHE_KEY: itpm_cache_key, - OTPM_CACHE_KEY: otpm_cache_key, - } - for channel in ("metadata", "litellm_metadata"): - existing = request_kwargs.get(channel) - if isinstance(existing, dict): - existing.update(reservation) - elif channel == "metadata": - request_kwargs[channel] = dict(reservation) - - def _extract_reservation(reservation: Mapping[str, int | str | None]) -> tuple[int, int, str | None, str | None]: itpm_cache_key: Final = reservation.get(ITPM_CACHE_KEY) otpm_cache_key: Final = reservation.get(OTPM_CACHE_KEY) @@ -399,7 +420,7 @@ def _clear_reservation_from_kwargs(kwargs: Mapping[str, object] | None) -> None: for channel in _reservation_channels(kwargs): channel_dict = _as_mutable_mapping(channel) if channel_dict is not None: - for key in (ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, ITPM_CACHE_KEY, OTPM_CACHE_KEY): + for key in IO_TOKEN_RESERVATION_KEYS: channel_dict.pop(key, None) @@ -499,7 +520,8 @@ def io_token_pre_call_check( if itpm_limit is None and otpm_limit is None: return deployment - request_kwargs: Final = get_io_token_rate_limit_request_kwargs() + request: Final = _io_token_rate_limit_request.get() + request_kwargs: Final = None if request is None else request.kwargs _model, served_model = _tokenizer_and_served_models(deployment) estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model) otpm_to_reserve: Final = ( @@ -546,11 +568,13 @@ def io_token_pre_call_check( raise _stash_reservation_in_metadata( - request_kwargs, - itpm_reserved=itpm_reserved, - otpm_reserved=otpm_reserved, - itpm_cache_key=itpm_key if itpm_limit is not None else None, - otpm_cache_key=otpm_key if otpm_limit is not None else None, + request, + _reservation( + itpm_reserved, + otpm_reserved, + itpm_key if itpm_limit is not None else None, + otpm_key if otpm_limit is not None else None, + ), ) return deployment @@ -565,7 +589,14 @@ async def async_io_token_pre_call_check( if itpm_limit is None and otpm_limit is None: return deployment - request_kwargs: Final = get_io_token_rate_limit_request_kwargs() + request: Final = _io_token_rate_limit_request.get() + request_kwargs: Final = None if request is None else request.kwargs + if request is not None and request.opens_responses_socket: + _stash_reservation_in_metadata( + request, + await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span), + ) + return deployment _model, served_model = _tokenizer_and_served_models(deployment) estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model) otpm_to_reserve: Final = ( @@ -618,15 +649,65 @@ async def async_io_token_pre_call_check( raise _stash_reservation_in_metadata( - request_kwargs, - itpm_reserved=itpm_reserved, - otpm_reserved=otpm_reserved, - itpm_cache_key=itpm_key if itpm_limit is not None else None, - otpm_cache_key=otpm_key if otpm_limit is not None else None, + request, + _reservation( + itpm_reserved, + otpm_reserved, + itpm_key if itpm_limit is not None else None, + otpm_key if otpm_limit is not None else None, + ), ) return deployment +async def _async_redis_usage(redis_cache: RedisCache | None, key: str, parent_otel_span: Span | None) -> int | None: + if redis_cache is None: + return None + with contextlib.suppress(RedisCircuitBreakerOpenError): + return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span) + return None + + +@with_service_target(ROUTER_USAGE_TARGET) +async def _async_shared_usage(dual_cache: DualCache, key: str, parent_otel_span: Span | None) -> int | None: + shared: Final = await _async_redis_usage(dual_cache.redis_cache, key, parent_otel_span) + if shared is not None: + return shared + return await dual_cache.async_get_cache(key=key, local_only=True) + + +async def _raise_if_at_limit( + dual_cache: DualCache, key: str, limit: int | None, limit_label: str, parent_otel_span: Span | None +) -> None: + if limit is None: + return + usage: Final = await _async_shared_usage(dual_cache, key, parent_otel_span) + if usage is not None and usage >= limit: + raise _rate_limit_error(limit_label, limit, usage) + + +@with_service_target(ROUTER_USAGE_TARGET) +async def async_admit_io_token_turn( + dual_cache: DualCache, + deployment: Mapping[str, object], + now: datetime, + parent_otel_span: Span | None = None, +) -> Mapping[str, object]: + configured: Final = dict(deployment) + itpm_limit, otpm_limit = get_deployment_io_token_limits(configured) + if itpm_limit is None and otpm_limit is None: + return MappingProxyType({}) + cache_keys: Final = _get_cache_keys(configured, now.strftime("%H-%M")) + if cache_keys is None: + return MappingProxyType({}) + itpm_key, otpm_key = cache_keys + await _raise_if_at_limit(dual_cache, itpm_key, itpm_limit, "ITPM", parent_otel_span) + await _raise_if_at_limit(dual_cache, otpm_key, otpm_limit, "OTPM", parent_otel_span) + return _reservation( + 0, 0, itpm_key if itpm_limit is not None else None, otpm_key if otpm_limit is not None else None + ) + + @with_service_target(ROUTER_USAGE_TARGET) def io_token_reconcile_success( dual_cache: DualCache, diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 3563fcd96ae..c315a31b65d 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -38,7 +38,7 @@ from litellm.litellm_core_utils.reasoning_effort_strength import ( ReasoningEffortRounding, nearest_declared_reasoning_effort, ) -from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking +from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking_silently from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping from litellm.types.llms.anthropic import AnthropicThinkingParam from litellm.types.llms.openai import REASONING_EFFORT @@ -143,7 +143,7 @@ def with_adaptive_thinking_as_effort( params: Mapping[str, object], *, model: str, custom_llm_provider: str, drop_params: bool ) -> Mapping[str, object]: translated: Final = dict(params) - AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently( model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params ) return translated @@ -151,7 +151,7 @@ def with_adaptive_thinking_as_effort( def thinking_budget_effort(thinking: object, *, model: str, custom_llm_provider: str, drop_params: bool) -> str | None: if custom_llm_provider in _PROVIDERS_SENDING_THINKING_AS_EFFORT: - return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params) adaptive_config: Final = as_str_mapping( with_adaptive_thinking_as_effort( {"thinking": thinking}, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params @@ -162,7 +162,7 @@ def thinking_budget_effort(thinking: object, *, model: str, custom_llm_provider: return adaptive_effort if isinstance(adaptive_effort, str) else None if "thinking" in (get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider) or ()): return None - return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params) def provider_reads_thinking(thinking: object, *, model: str, custom_llm_provider: str, drop_params: bool) -> bool: diff --git a/litellm/types/llms/gemini.py b/litellm/types/llms/gemini.py index 2ad81e64e05..67d90eefab0 100644 --- a/litellm/types/llms/gemini.py +++ b/litellm/types/llms/gemini.py @@ -81,6 +81,14 @@ class BidiGenerateContentServerMessage(TypedDict, total=False): """Output only. The setup complete message.""" +class BidiGenerateContentActivityStart(TypedDict): + pass + + +class BidiGenerateContentActivityEnd(TypedDict): + pass + + class BidiGenerateContentRealtimeInput(TypedDict, total=False): text: str """The text to be sent to the model.""" @@ -94,11 +102,11 @@ class BidiGenerateContentRealtimeInput(TypedDict, total=False): audioStreamEnd: bool """Output only. If true, indicates that the audio stream has ended.""" - activityStart: bool - """Output only. If true, indicates that the activity has started.""" + activityStart: BidiGenerateContentActivityStart + """Marks the start of user activity when automatic activity detection is disabled.""" - activityEnd: bool - """Output only. If true, indicates that the activity has ended.""" + activityEnd: BidiGenerateContentActivityEnd + """Marks the end of user activity when automatic activity detection is disabled.""" StartOfSpeechSensitivityEnum = Literal[ diff --git a/litellm/types/llms/mistral.py b/litellm/types/llms/mistral.py index cd61f32f430..68a9d26827b 100644 --- a/litellm/types/llms/mistral.py +++ b/litellm/types/llms/mistral.py @@ -23,3 +23,22 @@ class MistralThinkingBlock(TypedDict): type: Literal["thinking"] thinking: list[MistralTextBlock] closed: NotRequired[ReadOnly[bool]] + + +class MistralRequestRouting(TypedDict, total=False): + api_base: ReadOnly[object] + drop_params: ReadOnly[object] + allowed_openai_params: ReadOnly[object] + custom_llm_provider: ReadOnly[object] + + +class MistralCacheKeyParams(TypedDict, total=False): + prompt_cache_key: ReadOnly[object] + + +class MistralReasoningEffortParams(TypedDict, total=False): + reasoning_effort: ReadOnly[object] + + +class MistralFilePart(TypedDict, total=False): + file_id: ReadOnly[object] diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index d9d86cf54b3..a54d66c4beb 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,4 +1,5 @@ import builtins +import math from collections.abc import Iterable, Mapping, Sequence from enum import Enum from os import PathLike @@ -83,6 +84,7 @@ from typing_extensions import ( override, ) +from litellm._logging import verbose_logger from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBaseModel from litellm.types.responses.main import ( CustomToolCallOutputItem, @@ -1446,9 +1448,11 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject): @classmethod def parse_cost(cls, v: object) -> object: """Normalise cost: accept either a float or a dict with a ``total_cost`` key.""" - if isinstance(v, dict): - return v.get("total_cost") - return v + cost: Final = v.get("total_cost") if isinstance(v, dict) else v + if isinstance(cost, float) and not math.isfinite(cost): + verbose_logger.warning("Dropping a non-finite provider usage cost") + return None + return cost model_config = ConfigDict(extra="allow") diff --git a/litellm/types/proxy/management_endpoints/key_requests.py b/litellm/types/proxy/management_endpoints/key_requests.py index 172357d8ac4..b49cb63c8e0 100644 --- a/litellm/types/proxy/management_endpoints/key_requests.py +++ b/litellm/types/proxy/management_endpoints/key_requests.py @@ -1,14 +1,14 @@ import enum -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from datetime import datetime from types import MappingProxyType -from typing import Any, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers +from typing import Any, Final, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers from pydantic import Field, PositiveInt, field_validator, model_validator from typing_extensions import TypeIs # noqa: TID251 # narrows the raw payload a before-validator receives from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase -from litellm.types.router import UpdateRouterConfig +from litellm.types.router import ContentPolicyFallbackRules, UpdateRouterConfig from litellm.types.utils import LiteLLMPydanticObjectBase @@ -17,6 +17,12 @@ class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase): index_permissions: list[Literal["read", "write"]] +def _fallback_targets(rules: ContentPolicyFallbackRules) -> Iterator[object]: + for rule in rules: + for targets in rule.values(): + yield from targets + + class KeyRequestBase(GenerateRequestBase): key: str | None = None tpd_limit: int | None = None @@ -42,6 +48,18 @@ class KeyRequestBase(GenerateRequestBase): router_settings: UpdateRouterConfig | None = None access_group_ids: list[str] | None = None + @field_validator("router_settings") + @classmethod + def content_policy_fallbacks_name_models( + cls, router_settings: UpdateRouterConfig | None + ) -> UpdateRouterConfig | None: + rules: Final = None if router_settings is None else router_settings.content_policy_fallbacks + if rules is not None and not all(isinstance(target, str) for target in _fallback_targets(rules)): + raise ValueError( + 'A key\'s content_policy_fallbacks targets must be model names, e.g. [{"gpt-4": ["gpt-4o"]}]' + ) + return router_settings + class LiteLLMKeyType(str, enum.Enum): """ diff --git a/litellm/types/proxy/management_endpoints/subscription_accounts.py b/litellm/types/proxy/management_endpoints/subscription_accounts.py new file mode 100644 index 00000000000..a924b97034b --- /dev/null +++ b/litellm/types/proxy/management_endpoints/subscription_accounts.py @@ -0,0 +1,62 @@ +from datetime import date +from typing import Final + +from pydantic import BaseModel, ConfigDict, Field + +from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable + +CURRENCY_CODE_PATTERN: Final = r"^[A-Z]{3}$" + + +class SubscriptionAccountFeeRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + custom_llm_provider: str = Field(min_length=1) + account_id: str = Field(min_length=1) + monthly_fee: float = Field(gt=0, allow_inf_nan=False) + currency: str = Field(pattern=CURRENCY_CODE_PATTERN) + billing_period_start: date + label: str | None = None + + +class UpdateSubscriptionAccountFeeRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + subscription_account_id: str + monthly_fee: float | None = Field(default=None, gt=0, allow_inf_nan=False) + currency: str | None = Field(default=None, pattern=CURRENCY_CODE_PATTERN) + billing_period_start: date | None = None + label: str | None = None + + +class DeleteSubscriptionAccountRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + subscription_account_id: str + + +class BillingPeriod(BaseModel): + model_config = ConfigDict(frozen=True) + + start: date + end: date + + +class SubscriptionAccountUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + custom_llm_provider: str + account_id: str | None + deployments: tuple[str, ...] + fee: LiteLLM_SubscriptionAccountTable | None + billing_periods: tuple[BillingPeriod, ...] + fixed_cost: float | None + + +class SubscriptionUsageResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + start_date: date + end_date: date + subscription_providers: tuple[str, ...] + accounts: tuple[SubscriptionAccountUsage, ...] diff --git a/litellm/types/router.py b/litellm/types/router.py index f927ae0475b..726a328e913 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -192,6 +192,11 @@ OptionalPreCallChecks = list[ ] +FallbackTargets: TypeAlias = Sequence[str | Mapping[str, JsonValue]] +ContentPolicyFallbackRule: TypeAlias = Annotated[Mapping[str, FallbackTargets], Field(min_length=1, max_length=1)] +ContentPolicyFallbackRules: TypeAlias = Sequence[ContentPolicyFallbackRule] + + class UpdateRouterConfig(LiteLLMBaseModel): """ Set of params that you can modify via `router.update_settings()`. @@ -211,6 +216,7 @@ class UpdateRouterConfig(LiteLLMBaseModel): retry_after: float | None = None fallbacks: list[dict] | None = None context_window_fallbacks: list[dict] | None = None + content_policy_fallbacks: ContentPolicyFallbackRules | None = None model_group_alias: dict[str, str | dict] | None = Field(default={}) enable_tag_filtering: bool | None = None weights: RouterWeights | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5905aaa3380..55aaf606175 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1,4 +1,5 @@ import json +import math import re import time from collections.abc import Collection, Mapping, Sequence @@ -7,6 +8,7 @@ from enum import Enum from types import MappingProxyType from typing import ( TYPE_CHECKING, + Annotated, Any, Final, Literal, @@ -38,6 +40,7 @@ from openai.types.moderation_create_response import ( ) from pydantic import ( BaseModel, + BeforeValidator, ConfigDict, Field, FieldSerializationInfo, @@ -45,6 +48,7 @@ from pydantic import ( JsonValue, PrivateAttr, SkipValidation, + TypeAdapter, ValidationError, field_serializer, field_validator, @@ -1199,7 +1203,7 @@ class ChatCompletionTokenLogprob(OpenAIObject): unlikely. """ - top_logprobs: list[TopLogprob] + top_logprobs: list[TopLogprob] = Field(default_factory=list) """List of the most likely tokens and their log probability, at this token position. @@ -1236,9 +1240,13 @@ class ChatCompletionTokenLogprob(OpenAIObject): class ChoiceLogprobs(OpenAIObject): + model_config = ConfigDict(defer_build=False) + content: list[ChatCompletionTokenLogprob] | None = None """A list of message content tokens with log probability information.""" + refusal: Sequence[ChatCompletionTokenLogprob] | None = None + def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator return hasattr(self, key) @@ -1252,6 +1260,21 @@ class ChoiceLogprobs(OpenAIObject): return getattr(self, key) +def _sdk_logprobs_as_dict(logprobs: object) -> object: + if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel): + return logprobs + return logprobs.model_dump(exclude_unset=True) + + +CHOICE_LOGPROBS_OR_JSON: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter( + Annotated[ + ChoiceLogprobs | JsonValue, + Field(union_mode="left_to_right"), + BeforeValidator(_sdk_logprobs_as_dict), + ] +) + + class FunctionCall(OpenAIObject): arguments: str name: str | None = None @@ -1765,7 +1788,7 @@ class Choices(SafeAttributeModel, OpenAIObject): params["message"] = Message(**dump) if logprobs is not None: if isinstance(logprobs, dict): - params["logprobs"] = ChoiceLogprobs(**logprobs) + params["logprobs"] = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs) else: params["logprobs"] = logprobs else: @@ -1949,6 +1972,19 @@ class ServerToolUse(LiteLLMBaseModel): return getattr(self, key) +_COST_BREAKDOWN_ADAPTER: Final = TypeAdapter(Mapping[object, object]) + + +def _is_non_finite_float(value: object) -> bool: + return isinstance(value, float) and not math.isfinite(value) + + +def _carries_non_finite_number(cost: object) -> bool: + if not isinstance(cost, Mapping): + return _is_non_finite_float(cost) + return any(_is_non_finite_float(member) for member in _COST_BREAKDOWN_ADAPTER.validate_python(cost).values()) + + class Usage(SafeAttributeModel, CompletionUsage): _cache_creation_input_tokens: int = PrivateAttr( 0 @@ -1956,6 +1992,7 @@ class Usage(SafeAttributeModel, CompletionUsage): _cache_read_input_tokens: int = PrivateAttr( 0 ) # hidden param for prompt caching. Might change, once openai introduces their equivalent. + _dropped_non_finite_cost: bool = PrivateAttr(False) server_tool_use: ServerToolUse | None = None cost: float | None = None @@ -2062,10 +2099,14 @@ class Usage(SafeAttributeModel, CompletionUsage): else: # maintain openai compatibility in usage object if possible del self.server_tool_use - if cost is not None: - self.cost = cost - else: + dropped_non_finite_cost: Final = cost is not None and _carries_non_finite_number(cost) + if cost is None: del self.cost + elif dropped_non_finite_cost: + verbose_logger.warning("Dropping a non-finite provider usage cost") + del self.cost + else: + self.cost = cost ## ANTHROPIC MAPPING ## if "cache_creation_input_tokens" in params and isinstance(params["cache_creation_input_tokens"], int): @@ -2080,6 +2121,11 @@ class Usage(SafeAttributeModel, CompletionUsage): for k, v in params.items(): setattr(self, k, v) + self._dropped_non_finite_cost = dropped_non_finite_cost + + @staticmethod + def dropped_non_finite_cost(usage: "Usage") -> bool: + return usage._dropped_non_finite_cost def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator @@ -2131,8 +2177,8 @@ class StreamingChoices(OpenAIObject): if enhancements is not None: self.enhancements = enhancements - if logprobs is not None and isinstance(logprobs, dict): - self.logprobs = ChoiceLogprobs(**logprobs) + if isinstance(logprobs, (dict, BaseModel)): + self.logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs) else: self.logprobs = logprobs @@ -4155,6 +4201,8 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( "supports_assistant_prefill", "supports_bedrock_native_structured_output", "supports_pdf_input", + "reasoning_modes", + "reasoning_contexts", } ) diff --git a/litellm/utils.py b/litellm/utils.py index e85e089289a..6f56cc65f45 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10911,7 +10911,7 @@ class ProviderConfigManager: if LlmProviders.GEMINI == provider: from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig - return GeminiRealtimeConfig() + return GeminiRealtimeConfig(drop_params=drop_params) if LlmProviders.META == provider: from litellm.llms.meta.realtime.transformation import MetaRealtimeConfig diff --git a/schema.prisma b/schema.prisma index 2965174114b..179b741fe7c 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery { @@id([event_id, target, version]) @@index([target, state, next_attempt_at]) } + +model LiteLLM_SubscriptionAccountTable { + subscription_account_id String @id @default(uuid()) + custom_llm_provider String + account_id String + label String? + monthly_fee Float + currency String + billing_period_start String + created_at DateTime @default(now()) @map("created_at") + created_by String + updated_at DateTime @default(now()) @updatedAt @map("updated_at") + updated_by String + + @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") +} diff --git a/tests/benchmarks/test_realtime_session_merge_benchmarks.py b/tests/benchmarks/test_realtime_session_merge_benchmarks.py new file mode 100644 index 00000000000..29ee1b638ad --- /dev/null +++ b/tests/benchmarks/test_realtime_session_merge_benchmarks.py @@ -0,0 +1,21 @@ +from typing import Final + +import pytest + +from litellm.litellm_core_utils.realtime_streaming import _merge_session_fields + +NESTED_LEVELS: Final = 8 +LEAVES: Final = 2000 + + +def _nested_session(depth: int) -> dict[str, object]: + return {"child": _nested_session(depth - 1)} if depth else {f"k{index}": index for index in range(LEAVES)} + + +@pytest.mark.benchmark +def test_merging_a_session_update_that_repeats_every_nested_level() -> None: + session: Final = _nested_session(NESTED_LEVELS) + + merged: Final = _merge_session_fields(session, session) + + assert merged == session diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index c943ff2cbd0..9b61c1d39ae 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -85,6 +85,7 @@ IGNORE_FUNCTIONS = [ "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. + "_merge_session_fields", # max depth set (_SESSION_MERGE_MAX_DEPTH); deeper mappings are replaced wholesale by the later session.update. ] diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index 4d8336775ef..6bd5d61d721 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -47,6 +47,7 @@ class RealtimeProvider: id: str alias: str litellm_params: LiteLLMParamsBody + modalities: tuple[str, ...] = ("text",) PROVIDERS = ( @@ -75,6 +76,7 @@ PROVIDERS = ( model="gemini/gemini-3.1-flash-live-preview", api_key="os.environ/GEMINI_API_KEY", ), + modalities=("audio",), ), RealtimeProvider( "vertex_ai", @@ -84,6 +86,7 @@ PROVIDERS = ( vertex_location="us-central1", vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", ), + modalities=("audio",), ), RealtimeProvider( "bedrock", diff --git a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py index f5592abb34d..3b7ba3a2430 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -124,7 +124,8 @@ def test_text_conversation( session.send( SessionUpdate( session=SessionConfig( - instructions="You are a terse assistant. Reply in one short sentence." + modalities=list(provider.modalities), + instructions="You are a terse assistant. Reply in one short sentence.", ) ) ) @@ -159,6 +160,7 @@ def test_tool_call_round_trip( session.send( SessionUpdate( session=SessionConfig( + modalities=list(provider.modalities), tools=[WEATHER_TOOL], tool_choice="auto", instructions=( diff --git a/tests/integration/_support/hook_count_probe.py b/tests/integration/_support/hook_count_probe.py index 0d4056e8fb9..3cd72f1a6ba 100644 --- a/tests/integration/_support/hook_count_probe.py +++ b/tests/integration/_support/hook_count_probe.py @@ -75,6 +75,24 @@ class BlockingPostCallGuardrail(CustomGuardrail): raise HTTPException(status_code=400, detail="blocked by the hook-count guardrail") +class UsageRecordingPostCallGuardrail(CustomGuardrail): + """Records the token usage of the response it is handed in the request's guardrail information, so the spend row + shows which response the guardrail saw.""" + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: object, response: object + ) -> None: + usage: Final = getattr(response, "usage", None) + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response={ + "prompt_tokens": getattr(usage, "prompt_tokens", None), + "completion_tokens": getattr(usage, "completion_tokens", None), + }, + request_data=data, + guardrail_status="success", + ) + + probe: Final = HookCountProbe() sync_only_probe: Final = SyncOnlyHookCountProbe() diff --git a/tests/integration/authorization/test_claude_code_gateway_oauth.py b/tests/integration/authorization/test_claude_code_gateway_oauth.py index 4dedbdcd8e4..23476c439de 100644 --- a/tests/integration/authorization/test_claude_code_gateway_oauth.py +++ b/tests/integration/authorization/test_claude_code_gateway_oauth.py @@ -636,3 +636,55 @@ def test_device_flow_mints_the_session_for_the_team_the_developer_picks_in_the_b served_models: Final = {model.id for model in sdk.models.list()} assert served_models == {team_models[chosen_team]} assert len(identity_provider.drain()) == 3 + + +@pytest.mark.timeout(300) +def test_gateway_login_lists_and_defaults_to_the_first_team_on_the_user_record_after_a_write_to_its_row( + enabled_gateway: Gateway, identity_provider: Wire +) -> None: + run_id: Final = uuid.uuid4().hex + with enabled_gateway.scenario() as scenario: + user_id: Final = scenario.user(user_id=_USER_ID, user_email=_USER_EMAIL, user_role="internal_user") + first_team: Final = scenario.team(team_id=f"zeta-{run_id}") + second_team: Final = scenario.team(team_id=f"alpha-{run_id}") + for team_id in (first_team, second_team): + enabled_gateway.post( + "/team/member_add", {"team_id": team_id, "member": {"role": "user", "user_id": user_id}} + ) + enabled_gateway.post("/team/update", {"team_id": first_team, "metadata": {"updated_by_run": run_id}}) + assert enabled_gateway.rows('SELECT teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (user_id,)) == [ + {"teams": [first_team, second_team]} + ] + + authorization_response: Final = enabled_gateway.client.post( + "/claude_code_gateway/oauth/device_authorization", data={"client_id": _CLIENT_ID} + ) + assert authorization_response.status_code == 200, authorization_response.text + authorization: Final = _DeviceAuthorization.model_validate_json(authorization_response.content) + verification: Final = enabled_gateway.client.get(authorization.verification_uri, follow_redirects=False) + assert verification.status_code == 303, verification.text + identity_provider_redirect: Final = enabled_gateway.client.get( + verification.headers["location"], follow_redirects=False + ) + assert identity_provider_redirect.status_code == 302, identity_provider_redirect.text + callback: Final = enabled_gateway.client.get( + identity_provider_redirect.headers["location"], follow_redirects=False + ) + assert callback.status_code == 200, callback.text + assert tuple(_TEAM_OPTION.findall(callback.text)) == (first_team, second_team), callback.text + browser_token: Final = _BROWSER_TOKEN.search(callback.text) + assert browser_token is not None, callback.text + completion_without_a_pick: Final = enabled_gateway.client.post( + f"/sso/cli/complete/{authorization.device_code.partition('.')[0]}", + data={ + "user_code": authorization.user_code, + "browser_complete_token": html.unescape(browser_token.group(1)), + }, + ) + assert completion_without_a_pick.status_code == 200, completion_without_a_pick.text + token_response: Final = _device_code_grant(enabled_gateway, authorization.device_code) + assert token_response.status_code == 200, token_response.text + token: Final = _AccessToken.model_validate_json(token_response.content) + identity_provider.drain() + + assert (token.user_id, token.team_id) == (user_id, first_team) diff --git a/tests/integration/authorization/test_guardrail_submissions.py b/tests/integration/authorization/test_guardrail_submissions.py index 16b9cbf31dc..fb095010eb2 100644 --- a/tests/integration/authorization/test_guardrail_submissions.py +++ b/tests/integration/authorization/test_guardrail_submissions.py @@ -432,7 +432,7 @@ def test_team_guardrail_submissions_require_admin_approval( f"Guardrail '{first_name}' not found. " "Please ensure the guardrail is configured in your LiteLLM proxy." ), - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } @@ -481,7 +481,7 @@ def test_team_guardrail_submissions_require_admin_approval( f"Guardrail '{second_name}' not found. " "Please ensure the guardrail is configured in your LiteLLM proxy." ), - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } diff --git a/tests/integration/authorization/test_user_self_update_guard.py b/tests/integration/authorization/test_user_self_update_guard.py index 2c495beb77e..4294e8a088d 100644 --- a/tests/integration/authorization/test_user_self_update_guard.py +++ b/tests/integration/authorization/test_user_self_update_guard.py @@ -279,7 +279,7 @@ def test_new_user_with_admin_role_requires_proxy_admin(gateway: Gateway, role: s assert refused.json() == { "error": { "message": f"Only proxy admins can create administrative users (proxy_admin, proxy_admin_viewer). Attempted to create user with role: {role}. Your role: internal_user", - "type": "internal_server_error", + "type": "auth_error", "param": None, "code": "403", } diff --git a/tests/integration/management/test_added_backup_deployment_serves_peer_fallbacks.py b/tests/integration/management/test_added_backup_deployment_serves_peer_fallbacks.py new file mode 100644 index 00000000000..942bed6c165 --- /dev/null +++ b/tests/integration/management/test_added_backup_deployment_serves_peer_fallbacks.py @@ -0,0 +1,109 @@ +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Final + +import httpx +import pytest +from pydantic import TypeAdapter +from redis import Redis + +from tests.integration._support.client import Gateway, eventually, list_value, object_value, string_value +from tests.integration._support.database import scratch_database +from tests.integration._support.process import graceful_stop_seconds, owned_proxy +from tests.integration._support.redis_process import owned_redis + +pytestmark: Final = [pytest.mark.timeout(int(4 * graceful_stop_seconds() + 120))] + +COORDINATION_CONFIG: Final = Path(__file__).resolve().parents[1] / "coordination_redis_proxy_config.yaml" +CONFIG_SYNC_CHANNEL: Final = "litellm_proxy.config_change" +SUBSCRIBER_COUNTS: Final = TypeAdapter(tuple[tuple[str, int], ...]) +PERIODIC_RELOAD_SECONDS: Final = 3600 +PEER_PICKUP_SECONDS: Final = 4 +PROVIDER_KEY: Final = "integration-provider-key" + + +@contextmanager +def _proxy_pair(gateway: Gateway, directory: Path) -> Iterator[tuple[Gateway, Gateway]]: + with scratch_database("integration_model_add_sync") as database, owned_redis(directory) as coordination: + shared_stores: Final = { + "DATABASE_URL": database, + "REDIS_HOST": coordination.host, + "REDIS_PORT": str(coordination.port), + "PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": str(PERIODIC_RELOAD_SECONDS), + } + with ( + owned_proxy(gateway, directory / "writer", shared_stores, config=COORDINATION_CONFIG) as writer, + owned_proxy(gateway, directory / "peer", shared_stores, config=COORDINATION_CONFIG) as peer, + Redis(host=coordination.host, port=coordination.port, socket_timeout=2) as redis, + ): + eventually( + lambda: SUBSCRIBER_COUNTS.validate_python(redis.pubsub_numsub(CONFIG_SYNC_CHANNEL))[0][1], + lambda count: count >= 2, + seconds=30, + ) + yield writer, peer + + +@contextmanager +def _failing_upstream_model(gateway: Gateway, upstream_model: str) -> Iterator[None]: + script: Final = f"{gateway.upstream_url}/__scripts/{upstream_model}" + httpx.post(script, json={"statuses": [500]}, timeout=5, trust_env=False).raise_for_status() + try: + yield + finally: + httpx.delete(script, timeout=5, trust_env=False).raise_for_status() + + +def _add_deployment(writer: Gateway, group: str, upstream_model: str, order: int) -> None: + writer.post( + "/model/new", + { + "model_name": group, + "litellm_params": { + "model": f"openai/{upstream_model}", + "api_key": PROVIDER_KEY, + "api_base": f"{writer.upstream_url}/v1", + "order": order, + "num_retries": 0, + }, + }, + ) + + +def _listed_models(proxy: Gateway) -> frozenset[str]: + return frozenset(string_value(object_value(entry)["id"]) for entry in list_value(proxy.get("/v1/models")["data"])) + + +def _chat(proxy: Gateway, group: str) -> httpx.Response: + return proxy.request( + "POST", + "/v1/chat/completions", + {"model": group, "messages": [{"role": "user", "content": f"order fallback {uuid.uuid4().hex}"}]}, + ) + + +def test_a_backup_added_on_one_proxy_catches_the_peers_failing_primary_right_away( + gateway: Gateway, tmp_path: Path +) -> None: + group: Final = f"integration-{uuid.uuid4().hex}" + primary: Final = f"integration-primary-{uuid.uuid4().hex}" + backup: Final = f"integration-backup-{uuid.uuid4().hex}" + with _proxy_pair(gateway, tmp_path) as (writer, peer), _failing_upstream_model(gateway, primary): + _add_deployment(writer, group, primary, order=1) + eventually(lambda: _listed_models(peer), lambda listed: group in listed, seconds=30) + + _add_deployment(writer, group, backup, order=2) + on_writer: Final = _chat(writer, group) + on_peer: Final = eventually( + lambda: _chat(peer, group), + lambda response: response.status_code == 200, + seconds=PEER_PICKUP_SECONDS, + return_last_on_timeout=True, + ) + + assert on_writer.status_code == 200, on_writer.text + assert object_value(on_writer.json())["model"] == backup, on_writer.text + assert on_peer.status_code == 200, on_peer.text + assert object_value(on_peer.json())["model"] == backup, on_peer.text diff --git a/tests/integration/management/test_user_delete_revokes_keys.py b/tests/integration/management/test_user_delete_revokes_keys.py index 513a80b7319..2a1bd210b49 100644 --- a/tests/integration/management/test_user_delete_revokes_keys.py +++ b/tests/integration/management/test_user_delete_revokes_keys.py @@ -170,7 +170,7 @@ def test_user_delete_revokes_keys_and_memberships(gateway: Gateway, peer: Gatewa assert info.json() == { "error": { "message": f"User {user_id} not found", - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } @@ -337,7 +337,7 @@ def test_scim_user_delete_revokes_keys(gateway: Gateway, peer: Gateway, monkeypa expected_info: Final = { "error": { "message": f"User {user_id} not found", - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index 4b967802893..f6743a8751a 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -46,9 +46,10 @@ from pydantic import JsonValue, TypeAdapter pytestmark: Final = [pytest.mark.xdist_group("mcp_llm_private_gateway"), pytest.mark.shares_worker_state] -@pytest.fixture(scope="session") -def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, Mapping[str, str]]]: - directory: Final = tmp_path_factory.mktemp("mcp-llm-gateway") +@contextmanager +def _private_gateway( + directory: Path, guardrails: Sequence[Mapping[str, JsonValue]] = () +) -> Iterator[tuple[Gateway, Mapping[str, str]]]: stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) config: Final = directory / "stored-responses.yaml" (directory / f"{STREAM_BLOCK_MODULE}.py").write_text(STREAM_BLOCK_SOURCE) @@ -68,7 +69,8 @@ def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gat { "guardrail_name": STREAM_BLOCK, "litellm_params": {"guardrail": f"{STREAM_BLOCK_MODULE}.BlockGuard", "mode": "post_call"}, - } + }, + *guardrails, ], } ) @@ -81,15 +83,61 @@ def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gat yield private, services -@pytest.fixture -def gateway(_llm_gateway: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway]: - private, services = _llm_gateway +@contextmanager +def _serving(served: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway]: + private, services = served with pytest.MonkeyPatch.context() as environment: for name, value in services.items(): environment.setenv(name, value) yield private +@pytest.fixture(scope="session") +def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, Mapping[str, str]]]: + with _private_gateway(tmp_path_factory.mktemp("mcp-llm-gateway")) as served: + yield served + + +@pytest.fixture +def gateway(_llm_gateway: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway]: + with _serving(_llm_gateway) as private: + yield private + + +ROUND_USAGE_GUARDRAIL: Final = "mcp-llm-round-usage" +CONTENT_FILTER: Final = f"mcp-llm-content-filter-{uuid.uuid4().hex[:8]}" + + +@pytest.fixture(scope="module") +def _guarded_llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, Mapping[str, str]]]: + round_usage: Final[Mapping[str, JsonValue]] = { + "guardrail_name": ROUND_USAGE_GUARDRAIL, + "litellm_params": { + "guardrail": "integration._support.hook_count_probe.UsageRecordingPostCallGuardrail", + "mode": "post_call", + "default_on": True, + }, + } + content_filter: Final[Mapping[str, JsonValue]] = { + "guardrail_name": CONTENT_FILTER, + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "post_call", + "default_on": False, + "blocked_words": [{"keyword": "forbidden-keeper", "action": "BLOCK"}], + }, + } + guardrails: Final = (round_usage, content_filter) + with _private_gateway(tmp_path_factory.mktemp("mcp-llm-guarded-gateway"), guardrails) as served: + yield served + + +@pytest.fixture +def guarded_gateway(_guarded_llm_gateway: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway]: + with _serving(_guarded_llm_gateway) as private: + yield private + + Surface = Literal["chat", "responses", "messages", "messages_bridge", "responses_bridge"] SURFACES: Final[tuple[Surface, ...]] = ("chat", "responses", "messages", "messages_bridge") ADD: Final = {"a": 2, "b": 3} @@ -1032,6 +1080,184 @@ def test_nonstream_failed_follow_up_bills_round_one_on_the_failure_row( ) +def _streamed_chunks(response: httpx.Response) -> tuple[dict[str, object], ...]: + return tuple( + json.loads(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + + +def test_streaming_chat_auto_execute_logs_the_answer_and_bills_both_rounds(gateway: Gateway) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND) + with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO], stream=True, stream_options={"include_usage": True}) + assert response.status_code == 200, response.text + chunks: Final = _streamed_chunks(response) + text: Final = "".join( + str(chunk["choices"][0]["delta"].get("content") or "") for chunk in chunks if chunk.get("choices") + ) + assert text == ANSWER, response.text + usages: Final = tuple(chunk["usage"] for chunk in chunks if chunk.get("usage")) + assert usages, response.text + rows: Final = _spend_rows(response.headers["x-litellm-call-id"], "acompletion", "success") + assert len(rows) == 1, rows + row: Final = rows[0] + assert row["request_id"] == chunks[0]["id"], (row, chunks[0]) + assert (row["prompt_tokens"], row["completion_tokens"]) == ( + FIRST_ROUND[0] + FOLLOW_UP_ROUND[0], + FIRST_ROUND[1] + FOLLOW_UP_ROUND[1], + ), row + assert float(str(row["spend"])) == pytest.approx(_price(FIRST_ROUND, FOLLOW_UP_ROUND)), row + final_usage: Final = usages[-1] + assert (final_usage["prompt_tokens"], final_usage["completion_tokens"], final_usage["total_tokens"]) == ( + row["prompt_tokens"], + row["completion_tokens"], + sum((*FIRST_ROUND, *FOLLOW_UP_ROUND)), + ), usages + + +def test_streaming_chat_auto_execute_whose_follow_up_fails_bills_round_one_once(gateway: Gateway) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_fails=True) + with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO], stream=True, stream_options={"include_usage": True}) + assert FOLLOW_UP_OUTAGE in response.text, response.text + call_id: Final = response.headers["x-litellm-call-id"] + succeeded: Final = _spend_rows(call_id, "acompletion", "success") + assert len(succeeded) == 1, succeeded + assert (succeeded[0]["prompt_tokens"], succeeded[0]["completion_tokens"]) == FIRST_ROUND, succeeded + assert float(str(succeeded[0]["spend"])) == pytest.approx(_price(FIRST_ROUND)), succeeded + failed: Final = _spend_rows(call_id, "acompletion", "failure") + assert [float(str(row["spend"])) for row in failed] == [0.0], failed + + +def _guardrail_information(metadata: JsonValue) -> tuple[JsonValue, ...]: + entries: Final = object_value(metadata).get("guardrail_information") + return tuple(entries) if isinstance(entries, list) else () + + +def _round_usage_seen_by_the_guardrail(call_id: str) -> list[JsonValue]: + rows: Final = read_rows( + 'SELECT metadata FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s AND call_type=%s AND status=%s', + (call_id, "acompletion", "success"), + ) + entries: Final = tuple(chain.from_iterable(_guardrail_information(row["metadata"]) for row in rows)) + return [ + entry.get("guardrail_response") + for entry in entries + if isinstance(entry, dict) and entry.get("guardrail_name") == ROUND_USAGE_GUARDRAIL + ] + + +def test_streaming_chat_auto_execute_whose_follow_up_fails_runs_the_post_call_guardrail_on_round_one( + guarded_gateway: Gateway, +) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_fails=True) + with _rig(guarded_gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO], stream=True, stream_options={"include_usage": True}) + assert FOLLOW_UP_OUTAGE in response.text, response.text + call_id: Final = response.headers["x-litellm-call-id"] + succeeded: Final = _spend_rows(call_id, "acompletion", "success") + assert len(succeeded) == 1, succeeded + assert (succeeded[0]["prompt_tokens"], succeeded[0]["completion_tokens"]) == FIRST_ROUND, succeeded + assert _round_usage_seen_by_the_guardrail(call_id) == [ + {"prompt_tokens": FIRST_ROUND[0], "completion_tokens": FIRST_ROUND[1]} + ] + + +def _chat_rows(call_id: str, statuses: Sequence[str]) -> list[dict[str, JsonValue]]: + return eventually( + lambda: read_rows( + 'SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s AND call_type=%s', + (call_id, "acompletion"), + ), + lambda found: _statuses(found) == sorted(statuses), + seconds=70, + ) + + +def _entries_of(guardrail_name: str, row: Mapping[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]: + return tuple( + entry + for entry in _guardrail_information(row["metadata"]) + if isinstance(entry, dict) and entry.get("guardrail_name") == guardrail_name + ) + + +def test_streaming_chat_auto_execute_whose_follow_up_fails_counts_its_post_call_content_filter_once( + guarded_gateway: Gateway, +) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_fails=True) + with _rig(guarded_gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send( + _granted_key(rig), + [AUTO], + stream=True, + stream_options={"include_usage": True}, + guardrails=[CONTENT_FILTER], + ) + assert FOLLOW_UP_OUTAGE in response.text, response.text + rows: Final = _chat_rows(response.headers["x-litellm-call-id"], ("success", "failure")) + filtered: Final = tuple( + (row["status"], entry.get("guardrail_status")) for row in rows for entry in _entries_of(CONTENT_FILTER, row) + ) + assert filtered == (("failure", "guardrail_failed_to_respond"),), rows + (guardrail_id,) = frozenset( + str(entry.get("guardrail_id") or CONTENT_FILTER) + for row in rows + for entry in _entries_of(CONTENT_FILTER, row) + ) + counted: Final = eventually( + lambda: read_rows( + 'SELECT requests_evaluated, flagged_count FROM "LiteLLM_DailyGuardrailMetrics" WHERE guardrail_id=%s', + (guardrail_id,), + ), + lambda found: bool(found), + seconds=70, + ) + assert counted == [{"requests_evaluated": 1, "flagged_count": 1}], counted + + +@pytest.mark.parametrize("include_usage", (True, False), ids=("include-usage", "no-usage")) +def test_streaming_chat_auto_execute_answered_in_one_round_ends_with_the_usage_it_billed( + gateway: Gateway, include_usage: bool +) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND) + with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send( + _granted_key(rig), + [AUTO], + prompt=f"{DIRECT} {rig.alias}", + stream=True, + stream_options={"include_usage": include_usage}, + ) + assert response.status_code == 200, response.text + assert response.text.rstrip().endswith("data: [DONE]"), response.text + chunks: Final = _streamed_chunks(response) + text: Final = "".join( + str(chunk["choices"][0]["delta"].get("content") or "") for chunk in chunks if chunk.get("choices") + ) + assert text == ANSWER, response.text + rows: Final = _spend_rows(response.headers["x-litellm-call-id"], "acompletion", "success") + assert len(rows) == 1 and (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == FIRST_ROUND, rows + usage_at: Final = tuple(index for index, chunk in enumerate(chunks) if chunk.get("usage")) + if not include_usage: + assert usage_at == (), response.text + return + assert usage_at == (len(chunks) - 1,), response.text + finished_at: Final = tuple( + index + for index, chunk in enumerate(chunks) + if chunk.get("choices") and chunk["choices"][0].get("finish_reason") is not None + ) + assert finished_at and finished_at[-1] < usage_at[0], response.text + usage: Final = chunks[-1]["usage"] + assert (usage["prompt_tokens"], usage["completion_tokens"], usage["total_tokens"]) == ( + *FIRST_ROUND, + sum(FIRST_ROUND), + ), usage + + NO_CACHE: Final = {"no-cache": True} @@ -1269,6 +1495,27 @@ def test_streaming_responses_follow_up_broken_mid_stream_bills_round_one_once_an assert float(str(failure["spend"])) == pytest.approx(_price(failed_round)), rows +def test_streaming_chat_follow_up_broken_mid_answer_bills_round_one_and_its_partial_answer_with_one_fixed_margin( + gateway: Gateway, +) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_breaks=True) + with _rig(gateway, "chat", upstream="messages_bridge", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO], stream=True, stream_options={"include_usage": True}) + assert "the sum" in response.text, response.text + assert ANSWER not in response.text, response.text + assert len(rig.upstream_tools()) == 2 + round_one: Final = _price(FIRST_ROUND) + FIXED_MARGIN + rows: Final = _model_rows(response.headers["x-litellm-call-id"], _logs_a_failure_billing(round_one)) + assert _statuses(rows) == ["failure", "success"], rows + (success,) = (row for row in rows if row["status"] == "success") + (failure,) = (row for row in rows if row["status"] == "failure") + assert (success["prompt_tokens"], success["completion_tokens"]) == FIRST_ROUND, rows + assert float(str(success["spend"])) == pytest.approx(round_one), rows + partial_answer: Final = (int(str(failure["prompt_tokens"])), int(str(failure["completion_tokens"]))) + assert min(partial_answer) > 0, rows + assert float(str(failure["spend"])) == pytest.approx(_price(partial_answer)), rows + + @pytest.mark.parametrize("upstream", STREAMED_UPSTREAMS, ids=STREAMED_IDS) def test_streaming_responses_blocked_by_a_post_call_guardrail_bills_every_round_once( gateway: Gateway, upstream: StreamedUpstream diff --git a/tests/integration/observability/test_apply_guardrail_endpoint.py b/tests/integration/observability/test_apply_guardrail_endpoint.py index 9fa7d1a74db..f50679bd54e 100644 --- a/tests/integration/observability/test_apply_guardrail_endpoint.py +++ b/tests/integration/observability/test_apply_guardrail_endpoint.py @@ -156,7 +156,7 @@ def test_apply_guardrail_returns_a_block_error_for_generic_guardrails(gateway: G assert error.model_dump() == { "error": { "message": _BLOCK_REASON, - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": 400, } diff --git a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py index 854da6fce0b..0772e928b35 100644 --- a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py +++ b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py @@ -1037,7 +1037,7 @@ def test_s07_a_5kb_caller_model_under_the_wildcard_is_a_bounded_4xx_and_liveline error: Final = _JSON.validate_python(_JSON.validate_python(json.loads(outcome.raw))["error"]) assert "a" * 5120 in str(error["message"]), outcome.raw[:300] assert_within_factor_of_control(oversized.seconds, control=control.seconds, factor=20) - assert rig.wire.drain() == () + assert len(rig.wire.drain()) == 1 _failure_row(outcome.call_id) diff --git a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py index ad58a400cec..2712d6e30cd 100644 --- a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py +++ b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py @@ -355,7 +355,13 @@ def _voice_scenario_created_after_the_first_frame(transcript: str) -> RealtimeRe def _live_scenario() -> RealtimeResponse: - return RealtimeResponse(content_type="application/x-realtime", events=()) + return RealtimeResponse( + content_type="application/x-realtime", + events=( + {"serverContent": {"modelTurn": {"parts": [{"text": "scripted $REQUEST_ID"}]}}}, + {"serverContent": {"turnComplete": True}}, + ), + ) def _realtime_update(turn_detection: JsonValue) -> dict[str, JsonValue]: @@ -1348,10 +1354,14 @@ def test_gemini_ai_studio_voice_session_with_vad_tuning_connects_under_the_guard _ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, - (_step(_update_frame(update), until="session.updated"),), + ( + _step(_update_frame(update), until="session.updated"), + _step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE), + ), ) observed: Final = _observed(upstream_url, handle.scenario_id) - assert session.types == ("session.created", "session.updated"), session + assert session.types[:2] == ("session.created", "session.updated"), session + assert session.types[-1] == RESPONSE_DONE, session assert _setup_detection(observed) == {"disabled": True}, _sent(observed) @@ -1365,10 +1375,14 @@ def test_vertex_live_voice_session_keeps_vad_tuning_under_the_guardrail(guardrai _ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, - (_step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),), + ( + _step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"), + _step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE), + ), ) observed: Final = _observed(upstream_url, handle.scenario_id) - assert session.types == ("session.created", "session.updated"), session + assert session.types[:2] == ("session.created", "session.updated"), session + assert session.types[-1] == RESPONSE_DONE, session assert _setup_detection(observed) == GEMINI_TUNED_DETECTION, _sent(observed) diff --git a/tests/integration/providers/test_sagemaker_chat_wire.py b/tests/integration/providers/test_sagemaker_chat_wire.py index fdf0d94f7c7..784e43f85e2 100644 --- a/tests/integration/providers/test_sagemaker_chat_wire.py +++ b/tests/integration/providers/test_sagemaker_chat_wire.py @@ -243,7 +243,7 @@ def test_sagemaker_chat_signs_the_inference_component_header_and_sends_hf_model_ ) assert "x-amzn-sagemaker-inference-component" in signed_headers.removeprefix("SignedHeaders=").split(";") assert JSON_OBJECT.validate_json(request.body) == { - **container_body(wire.url, PROMPT, stream=False, model_id=_INFERENCE_COMPONENT), + **container_body(wire.url, PROMPT, stream=False), "model": SERVED_MODEL, }, request.body diff --git a/tests/integration/providers/test_sagemaker_embedding_wire.py b/tests/integration/providers/test_sagemaker_embedding_wire.py index 74f3c8c8a00..332f473aeaf 100644 --- a/tests/integration/providers/test_sagemaker_embedding_wire.py +++ b/tests/integration/providers/test_sagemaker_embedding_wire.py @@ -668,12 +668,12 @@ def test_an_extra_body_that_is_not_a_populated_object_adds_nothing_to_the_body( assert _body(request) == {"input": [text]}, request.body -def test_a_caller_sent_model_id_overrides_the_deployments_inference_component(proxy: Gateway, wire: Wire) -> None: +def test_a_caller_sent_model_id_cannot_override_the_deployments_inference_component(proxy: Gateway, wire: Wire) -> None: text: Final = _text() response: Final = _embed(proxy, "sm-openai", [text], model_id="caller-chosen-component") assert response.status_code == 200, response.text request: Final = _only_request(wire) - assert request.headers[COMPONENT_HEADER] == "caller-chosen-component", dict(request.headers) + assert request.headers[COMPONENT_HEADER] == COMPONENT, dict(request.headers) assert _body(request) == {"input": [text], "model": SERVED_MODEL}, request.body diff --git a/tests/integration/routing/test_responses_websocket_turn_admission.py b/tests/integration/routing/test_responses_websocket_turn_admission.py index 87b56a92979..7b0e14d044f 100644 --- a/tests/integration/routing/test_responses_websocket_turn_admission.py +++ b/tests/integration/routing/test_responses_websocket_turn_admission.py @@ -518,9 +518,7 @@ def test_a_tpm_limit_lowered_while_the_socket_is_open_applies_to_its_next_turn( assert deployment.forwarded_turns() == 1 -def test_a_socket_on_an_otpm_limited_deployment_reserves_once_and_serves_every_turn( - limited: Gateway, cert: tuple[Path, Path] -) -> None: +def test_a_socket_on_an_otpm_limited_deployment_serves_every_turn(limited: Gateway, cert: tuple[Path, Path]) -> None: with responses_peer(cert) as peer, limited.scenario() as scenario: deployment: Final = _native(scenario, peer, otpm=10_000) key: Final = scenario.key(models=[deployment.name]) @@ -536,6 +534,115 @@ def test_a_socket_on_an_otpm_limited_deployment_reserves_once_and_serves_every_t assert deployment.forwarded_turns() == 5 +IO_TOKEN_LIMIT: Final = 10_000 +TOKENS_PER_TURN: Final = MappingProxyType({"itpm": USAGE["input_tokens"], "otpm": USAGE["output_tokens"]}) + + +def _io_tokens_counted(identity: str, kind: str, minute: str) -> int: + with _redis() as redis: + return sum(int(redis.get(key) or 0) for key in redis.scan_iter(f"global_router:{identity}:*:{kind}:{minute}")) + + +def _add_io_tokens(deployment: Deployment, kind: str, minute: str, tokens: int) -> None: + with _redis() as redis: + redis.incrby(f"global_router:{deployment.identity}:openai/{PROVIDER_MODEL}:{kind}:{minute}", tokens) + + +async def _say_until_counted( + connection: websockets.ClientConnection, deployment: Deployment, counted: Callable[[], float], expected: int +) -> tuple[dict[str, JsonValue], ...]: + received: Final = await _say(connection, deployment.name) + await _until(counted, lambda tokens: tokens == expected) + return received + + +@pytest.mark.parametrize("kind", ("otpm", "itpm")) +def test_a_turn_is_refused_once_earlier_turns_fill_the_deployment_io_token_limit( + limited: Gateway, cert: tuple[Path, Path], kind: str +) -> None: + per_turn: Final = TOKENS_PER_TURN[kind] + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, **{kind: IO_TOKEN_LIMIT}) + key: Final = scenario.key(models=[deployment.name]) + minute: Final = _fresh_minute() + counted: Final = partial(_io_tokens_counted, deployment.identity, kind, minute) + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name) as connection: + first: Final = await _say_until_counted(connection, deployment, counted, per_turn) + await asyncio.to_thread(_add_io_tokens, deployment, kind, minute, IO_TOKEN_LIMIT - 5 * per_turn) + answered: Final = tuple( + [ + await _say_until_counted( + connection, deployment, counted, IO_TOKEN_LIMIT - (5 - number) * per_turn + ) + for number in range(2, 6) + ] + ) + return (first, *answered, await _say(connection, deployment.name)) + + turns: Final = asyncio.run(session()) + + assert [completed(received)["status"] for received in turns[:5]] == ["completed"] * 5 + _assert_refused(turns[5]) + assert deployment.forwarded_turns() == 5 + assert held(counted, lambda tokens: tokens == IO_TOKEN_LIMIT, holding=3, seconds=20) == IO_TOKEN_LIMIT + + +def test_a_socket_at_half_its_deployment_otpm_serves_ten_turns(limited: Gateway, cert: tuple[Path, Path]) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, otpm=IO_TOKEN_LIMIT) + key: Final = scenario.key(models=[deployment.name]) + minute: Final = _fresh_minute() + _add_io_tokens(deployment, "otpm", minute, IO_TOKEN_LIMIT // 2) + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name) as connection: + return tuple([await _say(connection, deployment.name) for _ in range(10)]) + + turns: Final = asyncio.run(session()) + + assert [completed(received)["status"] for received in turns] == ["completed"] * 10 + assert deployment.forwarded_turns() == 10 + expected: Final = IO_TOKEN_LIMIT // 2 + 10 * USAGE["output_tokens"] + counted: Final = partial(_io_tokens_counted, deployment.identity, "otpm", minute) + assert held(counted, lambda tokens: tokens == expected, holding=3, seconds=20) == expected + + +@pytest.mark.parametrize("relay", ("native", "managed")) +def test_every_turn_counts_its_actual_tokens_toward_the_deployment_itpm_and_otpm( + limited: Gateway, cert: tuple[Path, Path], relay: str +) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + limits: Final = {"itpm": 1_000_000, "otpm": 1_000_000} + deployment: Final = _native(scenario, peer, **limits) if relay == "native" else _managed(scenario, **limits) + key: Final = scenario.key(models=[deployment.name]) + minute: Final = _fresh_minute() + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name) as connection: + return tuple([await _say(connection, deployment.name) for _ in range(3)]) + + turns: Final = asyncio.run(session()) + + usages: Final = tuple(object_value(completed(received)["usage"]) for received in turns) + expected: Final = { + "itpm": sum(int(str(usage["input_tokens"])) for usage in usages), + "otpm": sum(int(str(usage["output_tokens"])) for usage in usages), + } + assert any(tokens > 0 for tokens in expected.values()), usages + counted: Final = { + kind: held( + partial(_io_tokens_counted, deployment.identity, kind, minute), + lambda tokens, kind=kind: tokens == expected[kind], + holding=3, + seconds=20, + ) + for kind in expected + } + assert counted == expected + + def test_usage_based_routing_refuses_a_turn_only_once_earlier_turns_used_up_the_deployment_tpm( usage_routed: Gateway, cert: tuple[Path, Path] ) -> None: diff --git a/tests/integration/spend/test_chatgpt_subscription_usage.py b/tests/integration/spend/test_chatgpt_subscription_usage.py new file mode 100644 index 00000000000..8f1fb2d0f74 --- /dev/null +++ b/tests/integration/spend/test_chatgpt_subscription_usage.py @@ -0,0 +1,322 @@ +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Final + +import httpx +import pytest +from integration._support import codex_vendor as cv +from integration._support import responses_vendor as rv +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, string_value +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Wire, wire_server +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +_MODEL: Final = "chatgpt/gpt-5.5" +_PROVIDER: Final = "chatgpt" +_FEE_TABLE_QUERY: Final = ( + 'SELECT subscription_account_id FROM "LiteLLM_SubscriptionAccountTable" WHERE custom_llm_provider = %s' + " AND account_id = %s" +) +_FEE_ROW_QUERY: Final = ( + 'SELECT subscription_account_id FROM "LiteLLM_SubscriptionAccountTable" WHERE subscription_account_id = %s' +) +_MONTHLY_FEE: Final = 200.0 +_BILLING_PERIOD_START: Final = "2026-01-15" +_RANGE: Final = {"start_date": "2026-03-01", "end_date": "2026-03-31"} +_PERIOD_IN_RANGE: Final = ("2026-03-15", "2026-04-14") +_TOKENS: Final = TypeAdapter(int) + + +class _Metrics(BaseModel): + model_config = ConfigDict(frozen=True) + + spend: float + prompt_tokens: int + completion_tokens: int + api_requests: int + successful_requests: int + failed_requests: int + + +class _ProviderUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + metrics: _Metrics + metadata: Mapping[str, JsonValue] + + +class _Breakdown(BaseModel): + model_config = ConfigDict(frozen=True) + + providers: Mapping[str, _ProviderUsage] + + +class _Day(BaseModel): + model_config = ConfigDict(frozen=True) + + breakdown: _Breakdown + + +class _DailyActivity(BaseModel): + model_config = ConfigDict(frozen=True) + + results: tuple[_Day, ...] + + +class _Fee(BaseModel): + model_config = ConfigDict(frozen=True) + + subscription_account_id: str + custom_llm_provider: str + account_id: str + monthly_fee: float + currency: str + billing_period_start: str + + +class _Period(BaseModel): + model_config = ConfigDict(frozen=True) + + start: str + end: str + + +class _Account(BaseModel): + model_config = ConfigDict(frozen=True) + + custom_llm_provider: str + account_id: str | None + deployments: tuple[str, ...] + fee: _Fee | None + billing_periods: tuple[_Period, ...] + fixed_cost: float | None + + +class _SubscriptionUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + accounts: tuple[_Account, ...] + + +@dataclass(frozen=True, slots=True) +class _Rig: + wire: Wire + proxy: OwnedProxy + model: str + + @property + def gateway(self) -> Gateway: + return self.proxy.gateway + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("chatgpt-subscription-usage") + no_default_login: Final = directory / "no-default-login" + no_default_login.mkdir() + deployment_login: Final = cv.login(directory) / "auth.json" + with gateway_from_environment() as gateway, wire_server(cv.CodexVendor().respond) as wire: + overrides: Final = {"CHATGPT_TOKEN_DIR": str(no_default_login), "CHATGPT_API_BASE": wire.url} + config: Final = cv.proxy_config(directory, probe=False) + with ( + owned_proxy_process(gateway, directory, overrides, config=config, workers=2) as owned, + owned.gateway.scenario() as scenario, + ): + model: Final = scenario.model( + model=_MODEL, api_base=wire.url, api_key=None, chatgpt_auth_file=str(deployment_login) + ) + yield _Rig(wire, owned, model) + + +def _responses_body(model: str, marker: str, *, stream: bool) -> Mapping[str, JsonValue]: + return { + "model": model, + "input": [{"role": "user", "content": f"Say marker-{marker}"}], + "stream": stream, + "cache": {"no-cache": True}, + } + + +def _assert_answered(response: httpx.Response, marker: str, content_type: str) -> None: + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith(content_type), response.headers + assert rv.answer(marker) in response.text, response.text + + +def _today() -> str: + return datetime.now(UTC).date().isoformat() + + +def _chatgpt_usage_between(gateway: Gateway, first_day: str, last_day: str) -> tuple[_ProviderUsage, ...]: + response: Final = gateway.request( + "GET", "/user/daily/activity", params={"start_date": first_day, "end_date": last_day} + ) + assert response.status_code == 200, response.text + activity: Final = _DailyActivity.model_validate_json(response.content) + return tuple( + result.breakdown.providers[_PROVIDER] for result in activity.results if _PROVIDER in result.breakdown.providers + ) + + +def _summed(buckets: tuple[_ProviderUsage, ...]) -> _Metrics: + return _Metrics( + spend=sum(bucket.metrics.spend for bucket in buckets), + prompt_tokens=sum(bucket.metrics.prompt_tokens for bucket in buckets), + completion_tokens=sum(bucket.metrics.completion_tokens for bucket in buckets), + api_requests=sum(bucket.metrics.api_requests for bucket in buckets), + successful_requests=sum(bucket.metrics.successful_requests for bucket in buckets), + failed_requests=sum(bucket.metrics.failed_requests for bucket in buckets), + ) + + +def _usage(gateway: Gateway, *, key: str | None = None) -> httpx.Response: + return gateway.request("GET", "/subscription_accounts/usage", params=_RANGE, key=key) + + +def _accounts(gateway: Gateway) -> tuple[_Account, ...]: + response: Final = _usage(gateway) + assert response.status_code == 200, response.text + return _SubscriptionUsage.model_validate_json(response.content).accounts + + +def _deployment_account(rig: _Rig) -> _Account: + (account,) = _accounts(rig.gateway) + assert (account.custom_llm_provider, account.account_id, account.deployments) == ( + _PROVIDER, + cv.ACCOUNT, + (rig.model,), + ), account + return account + + +def _assert_no_fee(account: _Account) -> None: + assert (account.fee, account.billing_periods, account.fixed_cost) == (None, (), None), account + + +def _fee_body(account_id: str, monthly_fee: float = _MONTHLY_FEE) -> Mapping[str, JsonValue]: + return { + "custom_llm_provider": _PROVIDER, + "account_id": account_id, + "monthly_fee": monthly_fee, + "currency": "USD", + "billing_period_start": _BILLING_PERIOD_START, + "label": "integration", + } + + +def _delete_fee_if_present(gateway: Gateway, identity: str) -> None: + if gateway.rows(_FEE_ROW_QUERY, (identity,)): + gateway.post("/subscription_accounts/delete", {"subscription_account_id": identity}) + assert gateway.rows(_FEE_ROW_QUERY, (identity,)) == [] + + +def _set_fee(scenario: Scenario, account_id: str) -> str: + created: Final = scenario.gateway.post("/subscription_accounts/new", _fee_body(account_id)) + identity: Final = string_value(created["subscription_account_id"]) + scenario.cleanups.callback(_delete_fee_if_present, scenario.gateway, identity) + return identity + + +def _assert_fee_reported(account: _Account, identity: str) -> None: + assert account.fee is not None, account + assert ( + account.fee.subscription_account_id, + account.fee.account_id, + account.fee.monthly_fee, + account.fee.currency, + account.fee.billing_period_start, + ) == (identity, cv.ACCOUNT, _MONTHLY_FEE, "USD", _BILLING_PERIOD_START), account + assert tuple((period.start, period.end) for period in account.billing_periods) == (_PERIOD_IN_RANGE,), account + assert account.fixed_cost == _MONTHLY_FEE, account + + +def test_responses_calls_on_a_deployment_login_count_as_subscription_covered_chatgpt_usage(rig: _Rig) -> None: + first_day: Final = _today() + rig.wire.drain() + plain_marker: Final = uuid.uuid4().hex + stream_marker: Final = uuid.uuid4().hex + plain: Final = rig.gateway.request("POST", "/v1/responses", _responses_body(rig.model, plain_marker, stream=False)) + _assert_answered(plain, plain_marker, "application/json") + streamed: Final = rig.gateway.request( + "POST", "/v1/responses", _responses_body(rig.model, stream_marker, stream=True) + ) + _assert_answered(streamed, stream_marker, "text/event-stream") + forwarded: Final = rig.wire.drain() + assert len(forwarded) == 2, [request.target for request in forwarded] + for request, marker in zip(forwarded, (plain_marker, stream_marker), strict=True): + cv.forwarded(request, marker) + last_day: Final = _today() + buckets: Final = eventually( + lambda: _chatgpt_usage_between(rig.gateway, first_day, last_day), + lambda found: _summed(found).api_requests >= 2, + seconds=70, + ) + totals: Final = _summed(buckets) + assert (totals.api_requests, totals.successful_requests, totals.failed_requests, totals.spend) == ( + 2, + 2, + 0, + 0, + ), buckets + assert (totals.prompt_tokens, totals.completion_tokens) == ( + 2 * _TOKENS.validate_python(cv.USAGE["input_tokens"]), + 2 * _TOKENS.validate_python(cv.USAGE["output_tokens"]), + ), buckets + assert all(bucket.metadata.get("subscription_covered") is True for bucket in buckets), buckets + + +def test_fee_on_the_deployment_account_reports_its_fixed_cost_until_deleted(rig: _Rig) -> None: + _assert_no_fee(_deployment_account(rig)) + with rig.gateway.scenario() as scenario: + identity: Final = _set_fee(scenario, cv.ACCOUNT) + _assert_fee_reported(_deployment_account(rig), identity) + deleted: Final = rig.gateway.post("/subscription_accounts/delete", {"subscription_account_id": identity}) + assert deleted["subscription_account_id"] == identity, deleted + assert rig.gateway.rows(_FEE_ROW_QUERY, (identity,)) == [] + _assert_no_fee(_deployment_account(rig)) + + +def test_second_fee_for_the_same_account_answers_400_and_keeps_the_first(rig: _Rig) -> None: + with rig.gateway.scenario() as scenario: + identity: Final = _set_fee(scenario, cv.ACCOUNT) + duplicate: Final = rig.gateway.request( + "POST", "/subscription_accounts/new", _fee_body(cv.ACCOUNT, monthly_fee=3 * _MONTHLY_FEE) + ) + assert duplicate.status_code == 400, duplicate.text + assert rig.gateway.rows(_FEE_TABLE_QUERY, (_PROVIDER, cv.ACCOUNT)) == [{"subscription_account_id": identity}] + _assert_fee_reported(_deployment_account(rig), identity) + + +def test_update_and_delete_of_an_unknown_fee_answer_404(rig: _Rig) -> None: + unknown: Final = str(uuid.uuid4()) + updated: Final = rig.gateway.request( + "POST", "/subscription_accounts/update", {"subscription_account_id": unknown, "monthly_fee": _MONTHLY_FEE} + ) + assert updated.status_code == 404, updated.text + deleted: Final = rig.gateway.request("POST", "/subscription_accounts/delete", {"subscription_account_id": unknown}) + assert deleted.status_code == 404, deleted.text + assert rig.gateway.rows(_FEE_ROW_QUERY, (unknown,)) == [] + + +def test_internal_user_is_refused_setting_a_fee_and_reading_usage(rig: _Rig) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(user_id=scenario.user(user_role="internal_user")) + created: Final = rig.gateway.request("POST", "/subscription_accounts/new", _fee_body(cv.ACCOUNT), key=key) + assert created.status_code == 401, created.text + assert rig.gateway.rows(_FEE_TABLE_QUERY, (_PROVIDER, cv.ACCOUNT)) == [] + usage: Final = _usage(rig.gateway, key=key) + assert usage.status_code == 401, usage.text + + +def test_fee_on_an_account_no_deployment_uses_stays_listed_and_counted(rig: _Rig) -> None: + unused: Final = f"acct-unused-{uuid.uuid4().hex}" + with rig.gateway.scenario() as scenario: + identity: Final = _set_fee(scenario, unused) + accounts: Final = _accounts(rig.gateway) + assert frozenset(account.account_id for account in accounts) == {cv.ACCOUNT, unused}, accounts + (unused_account,) = (account for account in accounts if account.account_id == unused) + assert unused_account.fee is not None, unused_account + assert (unused_account.deployments, unused_account.fee.subscription_account_id) == ((), identity) + assert unused_account.fixed_cost == _MONTHLY_FEE, unused_account diff --git a/tests/integration/spend/test_non_finite_cost_spend_writes.py b/tests/integration/spend/test_non_finite_cost_spend_writes.py new file mode 100644 index 00000000000..9a4d2dd45a4 --- /dev/null +++ b/tests/integration/spend/test_non_finite_cost_spend_writes.py @@ -0,0 +1,166 @@ +import json +from hashlib import sha256 +from pathlib import Path +from typing import Final, Literal +from uuid import uuid4 + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.openai_wire import answering_model_discovery +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.shares_worker_state + +PROMPT_TOKENS: Final = 5 +COMPLETION_TOKENS: Final = 3 +INPUT_RATE: Final = 0.001 +OUTPUT_RATE: Final = 0.002 +TOKEN_PRICE: Final = PROMPT_TOKENS * INPUT_RATE + COMPLETION_TOKENS * OUTPUT_RATE +OVERFLOWING_INPUT_RATE: Final = 1e308 +POISONED_UPSTREAM: Final = "poisoned-upstream" +CONTROL_UPSTREAM: Final = "control-upstream" +DAILY_TOTALS: Final = "SELECT model_group, SUM(spend)::float8 AS spend, SUM(api_requests)::int8 AS api_requests" +DAILY_USER_QUERY: Final = f'{DAILY_TOTALS} FROM "LiteLLM_DailyUserSpend" WHERE api_key=%s GROUP BY model_group' +DAILY_TEAM_QUERY: Final = f'{DAILY_TOTALS} FROM "LiteLLM_DailyTeamSpend" WHERE api_key=%s GROUP BY model_group' +DAILY_TAG_QUERY: Final = f'{DAILY_TOTALS} FROM "LiteLLM_DailyTagSpend" WHERE api_key=%s AND tag=%s GROUP BY model_group' + +Poison = Literal["reported_nan_cost", "overflowing_price"] + + +def _frame(payload: dict[str, JsonValue]) -> bytes: + return b"data: " + json.dumps(payload).encode() + b"\n\n" + + +def _stream(upstream_model: str, reported_cost: float | None) -> tuple[bytes, ...]: + chunk: Final = {"id": f"chatcmpl-{uuid4().hex}", "object": "chat.completion.chunk", "created": 1} + usage: Final[dict[str, JsonValue]] = { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + } + reported_usage: Final = usage if reported_cost is None else {**usage, "cost": reported_cost} + return ( + _frame({**chunk, "model": upstream_model, "choices": [{"index": 0, "delta": {"content": "ok"}}]}), + _frame( + { + **chunk, + "model": upstream_model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": reported_usage, + } + ), + b"data: [DONE]\n\n", + ) + + +def _deployment(alias: str, upstream_model: str, wire_url: str, input_rate: float) -> dict[str, JsonValue]: + return { + "model_name": alias, + "litellm_params": { + "model": f"openai/{upstream_model}", + "api_base": f"{wire_url}/v1", + "api_key": "integration-provider-key", + "input_cost_per_token": input_rate, + "output_cost_per_token": OUTPUT_RATE, + }, + } + + +def _proxy_config(directory: Path, deployments: tuple[dict[str, JsonValue], ...]) -> Path: + config: Final = directory / "non_finite_cost_config.yaml" + config.write_text( + json.dumps( + { + "model_list": list(deployments), + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "disable_spend_logs": False, + "proxy_batch_write_at": 1, + "proxy_batch_polling_interval": 1, + }, + "router_settings": {"disable_cooldowns": True}, + } + ) + ) + return config + + +def _by_model_group(rows: list[dict[str, JsonValue]]) -> dict[str, float]: + return {str(row["model_group"]): float(str(row["spend"])) for row in rows} + + +def _both_requests_rows(query: str, parameters: tuple[str, ...]) -> list[dict[str, JsonValue]]: + return eventually(lambda: read_rows(query, parameters), lambda rows: len(rows) == 2, seconds=70) + + +@pytest.mark.parametrize("poison", ["reported_nan_cost", "overflowing_price"]) +@pytest.mark.timeout(240) +def test_a_non_finite_cost_never_blocks_the_next_requests_spend_rows( + gateway: Gateway, tmp_path: Path, poison: Poison +) -> None: + poisoned_alias: Final = f"non-finite-cost-{uuid4().hex}" + control_alias: Final = f"finite-cost-{uuid4().hex}" + tag: Final = f"non-finite-cost-{uuid4().hex}" + poisoned_spend: Final = TOKEN_PRICE if poison == "reported_nan_cost" else 0.0 + poisoned_input_rate: Final = INPUT_RATE if poison == "reported_nan_cost" else OVERFLOWING_INPUT_RATE + + def respond(request: Request) -> Reply: + assert request.target.endswith("/chat/completions"), request.target + upstream_model: Final = json.loads(request.body)["model"] + reported_cost: Final = ( + float("nan") if upstream_model == POISONED_UPSTREAM and poison == "reported_nan_cost" else None + ) + return Reply(content_type="text/event-stream", chunks=_stream(upstream_model, reported_cost)) + + with wire_server(answering_model_discovery(respond)) as wire: + config: Final = _proxy_config( + tmp_path, + ( + _deployment(poisoned_alias, POISONED_UPSTREAM, wire.url, poisoned_input_rate), + _deployment(control_alias, CONTROL_UPSTREAM, wire.url, INPUT_RATE), + ), + ) + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate, candidate.scenario() as scenario: + team: Final = scenario.team() + key: Final = scenario.key(user_id=f"non-finite-cost-{uuid4().hex}", team_id=team) + digest: Final = sha256(key.encode()).hexdigest() + responses: Final = tuple( + candidate.request( + "POST", + "/v1/chat/completions", + { + "model": alias, + "messages": [{"role": "user", "content": f"non-finite cost {uuid4().hex}"}], + "stream": True, + "metadata": {"tags": [tag]}, + }, + key=key, + ) + for alias in (poisoned_alias, control_alias) + ) + assert all(response.status_code == 200 for response in responses), [response.text for response in responses] + + expected: Final = {poisoned_alias: pytest.approx(poisoned_spend), control_alias: pytest.approx(TOKEN_PRICE)} + spend_logs: Final = _both_requests_rows( + 'SELECT model_group, spend FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (digest,) + ) + assert _by_model_group(spend_logs) == expected, spend_logs + daily_rows: Final = ( + _both_requests_rows(DAILY_USER_QUERY, (digest,)), + _both_requests_rows(DAILY_TEAM_QUERY, (digest,)), + _both_requests_rows(DAILY_TAG_QUERY, (digest, tag)), + ) + assert [_by_model_group(rows) for rows in daily_rows] == [expected] * 3, daily_rows + assert [{row["api_requests"] for row in rows} for rows in daily_rows] == [{1}] * 3, daily_rows + key_total: Final = poisoned_spend + TOKEN_PRICE + key_spend: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(key_total), + seconds=70, + return_last_on_timeout=True, + ) + assert float(str(key_spend[0]["spend"])) == pytest.approx(key_total), key_spend diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index c70c7080ab8..fa03879c05a 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -3998,6 +3998,77 @@ class TestChatPathProviderStamp: assert params["prompt_cache_options"] == {"mode": "explicit"} +class TestChatPathResolvesTheDeploymentProvider: + MODEL = "azure_ai/gpt-6-astra" + AMBIENT_API_BASE = "https://other-deployment.openai.azure.com" + FOUNDRY_API_BASE = "https://foundry.services.ai.azure.com" + UPSTREAM_PROXY_API_BASE = "https://upstream-proxy.example.com" + POINTS = [{"location": "message", "role": "system"}] + MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + + def _pin_ambient_api_base(self, monkeypatch: pytest.MonkeyPatch, pinned: bool) -> None: + if pinned: + monkeypatch.setenv("AZURE_AI_API_BASE", self.AMBIENT_API_BASE) + else: + monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) + + def _chat( + self, custom_llm_provider: str | None, api_base: str, model: str = MODEL + ) -> tuple[list[AllMessageValues], dict[str, object]]: + seeded: Final[dict[str, object]] = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=seeded, + messages=copy.deepcopy(self.MESSAGES), + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + ) + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=copy.deepcopy(self.MESSAGES), + non_default_params=seeded, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return out, params + + def test_chat_path_stamps_the_breakpoint_on_a_foundry_deployment_while_the_ambient_base_is_azure_openai( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._pin_ambient_api_base(monkeypatch, True) + messages, params = self._chat(None, self.FOUNDRY_API_BASE) + assert (messages, params) == self._chat("azure_ai", self.FOUNDRY_API_BASE) + assert AnthropicCacheControlHook.count_request_cache_breakpoints(messages) == 1 + assert ("prompt_cache_options" in params) == supports_openai_prompt_cache_breakpoint(self.MODEL, "azure_ai") + + @pytest.mark.parametrize("pinned", [True, False]) + @pytest.mark.parametrize( + "api_base", + [FOUNDRY_API_BASE, "https://resource.cognitiveservices.azure.com", "https://resource.openai.azure.com"], + ) + def test_chat_path_marks_the_deployment_like_the_provider_its_api_base_dispatches_to( + self, monkeypatch: pytest.MonkeyPatch, api_base: str, pinned: bool + ) -> None: + self._pin_ambient_api_base(monkeypatch, pinned) + dispatched_provider: Final = litellm.get_llm_provider(model=self.MODEL, api_base=api_base)[1] + assert self._chat(None, api_base) == self._chat(dispatched_provider, api_base) + + @pytest.mark.parametrize("model", [MODEL, "bedrock_mantle/openai.gpt-5.6-sol"]) + @pytest.mark.parametrize("set_globally", ["env", "module"]) + def test_chat_path_marks_the_deployment_like_litellm_proxy_when_calls_route_to_one( + self, monkeypatch: pytest.MonkeyPatch, model: str, set_globally: str + ) -> None: + self._pin_ambient_api_base(monkeypatch, False) + if set_globally == "env": + monkeypatch.setenv("USE_LITELLM_PROXY", "True") + else: + monkeypatch.setattr(litellm, "use_litellm_proxy", True) + assert self._chat(None, self.UPSTREAM_PROXY_API_BASE, model) == self._chat( + "litellm_proxy", self.UPSTREAM_PROXY_API_BASE, model + ) + + class TestClientBreakpointsCountedOnce: def test_client_message_breakpoints_are_not_double_counted(self): messages = [ diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index faecb8d8ef6..d0f5bfc6b9f 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -1,6 +1,7 @@ import asyncio import importlib import json +from collections.abc import Mapping from datetime import datetime, timedelta from typing import Final @@ -1111,6 +1112,44 @@ def test_convert_to_model_response_object_with_logprobs(): assert result.system_fingerprint is None assert result._hidden_params is not None + +@pytest.mark.parametrize( + "upstream_logprobs", + [ + pytest.param({"content": [{"token": "Hi", "logprob": None}]}, id="null-content-logprob"), + pytest.param( + { + "content": [{"token": "Hi", "logprob": -0.1, "top_logprobs": []}], + "refusal": [{"token": "I", "logprob": None}], + }, + id="null-refusal-logprob", + ), + ], +) +def test_convert_to_model_response_object_passes_nonconforming_logprobs_through( + upstream_logprobs: Mapping[str, object], +) -> None: + result: Final = convert_to_model_response_object( + response_object={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1702685778, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hi"}, + "logprobs": upstream_logprobs, + "finish_reason": "stop", + } + ], + }, + model_response_object=ModelResponse(), + ) + + assert json.loads(result.model_dump_json())["choices"][0]["logprobs"] == upstream_logprobs + + def test_convert_to_model_response_object_error(): """Test error handling for None response object.""" with pytest.raises(Exception, match="Error in response object format"): @@ -3050,6 +3089,50 @@ def test_convert_to_model_response_object_function_output(): assert result._hidden_params is not None +@pytest.mark.parametrize( + "reported_cost", + [ + float("nan"), + float("inf"), + {"input_tokens_cost": 0.0001, "request_cost": 0.005, "total_cost": float("nan")}, + {"input_tokens_cost": float("inf"), "request_cost": 0.005, "total_cost": 0.0051}, + ], + ids=["nan", "inf", "breakdown-with-nan-total", "breakdown-with-inf-part"], +) +def test_a_non_finite_provider_usage_cost_is_dropped_so_the_response_is_strict_json(reported_cost: object): + response: Final = convert_to_model_response_object( + response_object={ + "id": "chatcmpl-non-finite-cost", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4, "cost": reported_cost}, + }, + model_response_object=ModelResponse(), + ) + + usage: Final = json.loads(json.dumps(response.model_dump(), allow_nan=False))["usage"] + assert "cost" not in usage + assert usage["total_tokens"] == 4 + + +def test_a_finite_provider_cost_breakdown_with_non_string_keys_is_kept(): + response: Final = convert_to_model_response_object( + response_object={ + "id": "chatcmpl-int-key-cost", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4, "cost": {1: 0.5}}, + }, + model_response_object=ModelResponse(), + ) + + assert response.usage.cost == {1: 0.5} + + _URL_CITATION: Final = { "type": "url_citation", "url_citation": { diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 53be3608979..778c9d68be5 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1246,6 +1246,80 @@ def test_anthropic_messages_pt_keeps_signed_thinking_block(): assert thinking_blocks[0]["thinking"] == "genuine anthropic reasoning" +@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai")) +def test_anthropic_messages_pt_converts_pdf_image_url_data_uri_to_document(llm_provider: str) -> None: + pdf_data = "JVBERi0xLjQK" + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "image_url", "image_url": {"url": f"data:application/pdf;base64,{pdf_data}"}}, + ], + } + ], + model="claude-opus-5-5", + llm_provider=llm_provider, + ) + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data}, + } + + +@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai")) +def test_anthropic_messages_pt_uses_pdf_format_for_generic_mime_data_uri(llm_provider: str) -> None: + pdf_data = "JVBERi0xLjQK" + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + { + "type": "image_url", + "image_url": { + "url": f"data:application/octet-stream;base64,{pdf_data}", + "format": "application/pdf", + }, + }, + ], + } + ], + model="claude-opus-5-5", + llm_provider=llm_provider, + ) + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data}, + } + + +def test_anthropic_messages_pt_keeps_png_image_url_as_image() -> None: + image_data = "iVBORw0KGgo=" + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": f"data:image/png;base64,{image_data}"}, + ], + } + ], + model="claude-opus-5-5", + llm_provider="anthropic", + ) + + assert result[0]["content"][1] == { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": image_data}, + } + + EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize( "empty_user_message", [{"role": "user", "content": None}, {"role": "user", "content": []}, {"role": "user"}], diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 1c86c4e30d8..93defc0f04f 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -26,15 +26,24 @@ from litellm.litellm_core_utils.core_helpers import ( normalize_drop_params, preserve_upstream_non_openai_attributes, process_response_headers, + provider_reported_usage_cost, qualify_agentic_followup_model, qualify_provider_stripped_model, reconstruct_model_name, redact_nested_match_and_regex_keys, + response_cost_in_hidden_params, set_provider_response_headers_in_hidden_params, unbind_budget_reservation_from_callbacks, ) from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.utils import Delta, ImageResponse, ModelResponseStream, StreamingChoices, TranscriptionResponse +from litellm.types.utils import ( + Delta, + ImageResponse, + ModelResponse, + ModelResponseStream, + StreamingChoices, + TranscriptionResponse, +) def test_parent_otel_span_reads_dict_like_metadata_by_index(): @@ -790,6 +799,23 @@ class TestReceiveStampSurvivesASecondMetadataBucket: assert get_litellm_metadata_from_kwargs(kwargs)["litellm_received_at"] is own +@pytest.mark.parametrize("reported", [float("nan"), float("inf"), {"total_cost": float("nan")}]) +def test_a_non_finite_provider_reported_cost_counts_as_unreported(reported: object) -> None: + assert provider_reported_usage_cost({"cost": reported}, "openrouter") is None + assert provider_reported_usage_cost({"cost": {"total_cost": 0.25}}, "openrouter") == 0.25 + + +@pytest.mark.parametrize("header_cost", [float("nan"), float("inf")]) +def test_a_non_finite_response_cost_header_counts_as_unreported(header_cost: float) -> None: + response: Final = ModelResponse() + response._hidden_params = {"additional_headers": {RESPONSE_COST_HEADER: header_cost}} + finite: Final = ModelResponse() + finite._hidden_params = {"additional_headers": {RESPONSE_COST_HEADER: 0.25}} + + assert response_cost_in_hidden_params(response) is None + assert response_cost_in_hidden_params(finite) == 0.25 + + def test_billed_usage_cost_bills_any_chat_upstream_without_its_own_reader() -> None: from types import SimpleNamespace diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 88501a23235..efb9648cf13 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -60,7 +60,6 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st assert caught.value.response.headers["x-request-id"] == "cyber-policy-request" - def _debug_info_for_api_base(configured_api_base: str) -> str: request: Final = httpx.Request("POST", "https://gateway.example/v1/chat/completions") original: Final = OpenAIError( @@ -1602,6 +1601,75 @@ def test_bedrock_mantle_400_maps_to_bad_request(): assert type(excinfo.value) is litellm.BadRequestError +class _ErrorWithoutResponse(Exception): + def __init__(self, message: str, status_code: int) -> None: + super().__init__(message) + self.message = message + self.status_code = status_code + + +def test_bedrock_mantle_context_overflow_without_a_provider_response_answers_400(): + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="openai.gpt-5.6-sol", + original_exception=_ErrorWithoutResponse( + '{"error":{"code":"context_length_exceeded","message":"too long"}}', 400 + ), + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.response.status_code == 400 + + +def _invalid_model_identifier_from(url: str) -> BedrockError: + return BedrockError( + status_code=400, + message='{"message":"The provided model identifier is invalid."}', + response=httpx.Response(status_code=400, request=httpx.Request(method="POST", url=url)), + ) + + +@pytest.mark.parametrize( + "path", + ["/model/us.newvendor.typo-1/converse", "/model/us.newvendor.typo-1/converse-stream"], +) +def test_bedrock_converse_rejecting_an_id_with_no_invoke_provider_answers_404(path): + with pytest.raises(litellm.NotFoundError) as excinfo: + exception_type( + model="us.newvendor.typo-1", + original_exception=_invalid_model_identifier_from(f"https://bedrock-runtime.us-east-1.amazonaws.com{path}"), + custom_llm_provider="bedrock", + ) + + assert excinfo.value.response.status_code == 404 + + +@pytest.mark.parametrize( + "original_exception", + [ + _invalid_model_identifier_from("https://bedrock-runtime.us-east-1.amazonaws.com/model/claude-typo-9/invoke"), + _ErrorWithoutResponse("The provided model identifier is invalid.", 400), + ], + ids=["invoke", "no-response"], +) +def test_bedrock_invalid_model_identifier_not_from_converse_stays_a_bad_request(original_exception): + with pytest.raises(litellm.BadRequestError) as excinfo: + exception_type(model="claude-typo-9", original_exception=original_exception, custom_llm_provider="bedrock") + + assert excinfo.value.status_code == 400 + + +def test_bedrock_invalid_model_identifier_without_a_model_stays_a_bad_request(): + with pytest.raises(litellm.BadRequestError) as excinfo: + exception_type( + model=None, + original_exception=_ErrorWithoutResponse("The provided model identifier is invalid.", 400), + custom_llm_provider="bedrock", + ) + + assert excinfo.value.status_code == 400 + + def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/litellm_core_utils/test_fallback_utils.py b/tests/unit/litellm_core_utils/test_fallback_utils.py index 1e3ca0ba90d..c0a344cc6c3 100644 --- a/tests/unit/litellm_core_utils/test_fallback_utils.py +++ b/tests/unit/litellm_core_utils/test_fallback_utils.py @@ -1,6 +1,7 @@ """Tests for litellm.litellm_core_utils.fallback_utils.""" import json +import pathlib from typing import Final import httpx @@ -8,12 +9,14 @@ import pytest import respx import litellm +from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.fallback_utils import ( async_completion_with_fallbacks, ) from litellm.main import acompletion as python_acompletion from litellm.main import completion as python_completion +from litellm.vector_stores.vector_store_registry import LiteLLM_ManagedVectorStore, VectorStoreRegistry @pytest.mark.asyncio @@ -444,3 +447,187 @@ def test_completion_fallbacks_sync( assert response.choices[0].message.content == "sync fallback answer" assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("first", "second") + + +CACHE_POINTS: Final = ({"location": "message", "role": "system"},) +CACHED_MESSAGES: Final = ( + {"role": "system", "content": "You are a careful assistant."}, + {"role": "user", "content": "hello"}, +) +OPENAI_ANSWER: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-6.1-sol", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +ANTHROPIC_ANSWER: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, +} + + +def _cached_request(**kwargs: object) -> dict[str, object]: + return { + "messages": [dict(message) for message in CACHED_MESSAGES], + "cache_control_injection_points": [dict(point) for point in CACHE_POINTS], + **kwargs, + } + + +@pytest.mark.asyncio +async def test_acompletion_fallback_sends_the_fallback_model_the_request_a_direct_call_sends( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + openai_route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 401, + json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}}, + ) + ) + anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=ANTHROPIC_ANSWER) + ) + + await python_acompletion(**_cached_request(model="anthropic/claude-haiku-4-5", api_key="anthropic-key")) + await python_acompletion( + **_cached_request( + model="openai/gpt-6.1-sol", + api_key="openai-key", + fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}], + ) + ) + + assert "prompt_cache_options" in json.loads(openai_route.calls[0].request.content) + direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls) + assert fallback_body == direct_body + + +@pytest.mark.parametrize("is_async", [True, False]) +@pytest.mark.asyncio +async def test_cache_dialect_follows_base_url_over_an_api_base_the_call_does_not_use( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, is_async: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=OPENAI_ANSWER) + ) + + async def send(**kwargs: object) -> None: + request: Final = _cached_request( + model="openai/gpt-6.1-sol", api_key="openai-key", base_url="https://api.openai.com/v1", **kwargs + ) + if is_async: + await python_acompletion(**request) + else: + python_completion(**request) + + await send() + await send(api_base="https://unused-gateway.example.com/v1") + + base_url_only, with_unused_api_base = (json.loads(call.request.content) for call in route.calls) + assert with_unused_api_base == base_url_only + + +@pytest.mark.asyncio +async def test_acompletion_fallback_sends_the_context_a_vector_store_tool_retrieves( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr( + litellm, + "vector_store_registry", + VectorStoreRegistry( + vector_stores=[ + LiteLLM_ManagedVectorStore( + vector_store_id="vs_docs", custom_llm_provider="openai", litellm_params={"api_key": "openai-key"} + ) + ] + ), + ) + search_route: Final = respx_mock.post("https://api.openai.com/v1/vector_stores/vs_docs/search").mock( + return_value=httpx.Response( + 200, + json={ + "object": "vector_store.search_results.page", + "search_query": "what is litellm?", + "data": [{"score": 1.0, "content": [{"type": "text", "text": "litellm is an LLM gateway"}]}], + }, + ) + ) + openai_route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 401, + json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}}, + ) + ) + anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=ANTHROPIC_ANSWER) + ) + + async def send(**kwargs: object) -> None: + await python_acompletion( + messages=[{"role": "user", "content": "what is litellm?"}], + tools=[{"type": "file_search", "vector_store_ids": ["vs_docs"]}], + **kwargs, + ) + + await send(model="anthropic/claude-haiku-4-5", api_key="anthropic-key") + await send( + model="openai/gpt-6.1-sol", + api_key="openai-key", + fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}], + ) + + direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls) + assert "litellm is an LLM gateway" in json.dumps(direct_body) + assert fallback_body == direct_body + assert search_route.call_count <= openai_route.call_count + anthropic_route.call_count + + +@pytest.mark.asyncio +async def test_acompletion_fallback_sends_the_prompt_the_primary_model_prefix_template_renders( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path +) -> None: + _ = (tmp_path / "brief.prompt").write_text( + "---\ninput:\n schema:\n question: string\n---\nSystem: Answer in one word.\n\nUser: {{question}}\n" + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "global_prompt_directory", str(tmp_path)) + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", []) + _ = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 401, + json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}}, + ) + ) + anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=ANTHROPIC_ANSWER) + ) + + async def send(**kwargs: object) -> None: + await python_acompletion( + prompt_id="brief", + prompt_variables={"question": "What color is the sky?"}, + messages=[{"role": "user", "content": "placeholder"}], + **kwargs, + ) + + await send(model="dotprompt/anthropic/claude-haiku-4-5", api_key="anthropic-key") + await send( + model="dotprompt/openai/gpt-6.1-sol", + api_key="openai-key", + fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}], + ) + + direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls) + assert "What color is the sky?" in json.dumps(direct_body) + assert fallback_body == direct_body diff --git a/tests/unit/litellm_core_utils/test_image_handling.py b/tests/unit/litellm_core_utils/test_image_handling.py index 9d9aef0c22a..d1bebde9917 100644 --- a/tests/unit/litellm_core_utils/test_image_handling.py +++ b/tests/unit/litellm_core_utils/test_image_handling.py @@ -288,6 +288,37 @@ def test_convert_url_to_base64_keeps_a_real_content_type(monkeypatch): assert result.startswith("data:image/jpeg;base64,") +@pytest.mark.parametrize( + ("url", "content_type", "expected"), + [ + ( + "https://docs.example/document.pdf", + "application/pdf; qs=0.001", + "data:application/pdf;base64,cGRm", + ), + ( + "https://img.example/image.png", + "image/png; charset=binary", + "data:image/png;base64,cGRm", + ), + ( + "https://docs.example/document.pdf", + "application/octet-stream; charset=binary", + "data:application/pdf;base64,cGRm", + ), + ], +) +def test_convert_url_to_base64_uses_bare_content_type( + monkeypatch: pytest.MonkeyPatch, + url: str, + content_type: str, + expected: str, +) -> None: + monkeypatch.setattr(litellm, "module_level_client", _stub_image_client(b"pdf", content_type)) + + assert convert_url_to_base64(url) == expected + + def test_convert_url_to_base64_raises_when_no_content_type_is_determinable(monkeypatch): monkeypatch.setattr( litellm, diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index b3ae3bf6240..336e9f7d24a 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6308,6 +6308,60 @@ def test_standard_logging_payload_resolves_kubernetes_pod_routing_from_metadata_ assert payload["metadata"][KUBERNETES_POD_ROUTING_KEY] == expected +@pytest.mark.parametrize("response_cost", [float("nan"), float("inf"), float("-inf")]) +def test_get_standard_logging_object_payload_logs_a_non_finite_response_cost_as_zero( + logging_obj: LitellmLogging, response_cost: float +) -> None: + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={"model": "gpt-4o", "messages": [], "litellm_params": {}, "response_cost": response_cost}, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["response_cost"] == 0 + + +def test_an_overflowing_custom_price_records_a_zero_response_cost(caplog: pytest.LogCaptureFixture) -> None: + messages: Final = [{"role": "user", "content": "hello world"}] + start_time: Final = datetime.datetime.now() + logging_obj: Final = Logging( + model="gpt-4o", + messages=messages, + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="overflow\nforged log line", + function_id="overflow", + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + response: Final = litellm.completion( + model="gpt-4o", + mock_response="hi there", + messages=messages, + input_cost_per_token=1e308, + output_cost_per_token=1e308, + litellm_logging_obj=logging_obj, + ) + logging_obj._process_hidden_params_and_response_cost( + response, start_time=start_time, end_time=datetime.datetime.now() + ) + + assert response._hidden_params["response_cost"] == 0.0 + payload: Final = logging_obj.model_call_details["standard_logging_object"] + assert json.loads(json.dumps(payload, allow_nan=False, default=str))["response_cost"] == 0.0 + warnings: Final = [r.getMessage() for r in caplog.records if "non-finite response_cost" in r.getMessage()] + assert warnings == ["Recording a non-finite response_cost as 0 for litellm_call_id=overflowforged log line"] + + def test_get_standard_logging_object_payload_carries_matched_access_groups(logging_obj): """Access groups stamped at auth time reach the logging payload, so integrations see what a request billed.""" from datetime import datetime diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index a8543ebcfc2..ba76138a268 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -1,7 +1,7 @@ import asyncio import base64 import json -from collections.abc import Coroutine, Mapping, Sequence +from collections.abc import Callable, Coroutine, Mapping, Sequence from dataclasses import dataclass from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -14,6 +14,7 @@ from websockets.frames import Close import litellm from litellm.constants import ( BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, + REALTIME_LAZY_SETUP_DEADLINE_SECONDS, REALTIME_METERED_USAGE_KEY, REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, @@ -21,6 +22,7 @@ from litellm.constants import ( from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.realtime_streaming import ( RealTimeStreaming, + _merge_session_fields, client_sent_openai_beta_realtime_header, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -1255,11 +1257,15 @@ async def test_client_ack_caches_setup_to_prevent_duplicate_session_update_setup side_effect=[ json.dumps({"type": "session.update", "session": {"tools": []}}), json.dumps({"type": "session.update", "session": {"tools": []}}), + _TEXT_ITEM_CREATE, + _RESPONSE_CREATE, Exception("client done"), ] ) + websocket.send_text = AsyncMock() provider_config = MagicMock() + provider_config.setup_session_update_rejection = MagicMock(return_value=None) def _transform(message: str, model: str, session_configuration_request=None): if session_configuration_request is None: @@ -2285,6 +2291,7 @@ async def test_guardrail_turn_detection_injected_into_first_session_update_defer logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.builds_setup_from_session_update = MagicMock(return_value=False) transformed_messages = [] def mock_transform(msg, model, session_config): @@ -2351,6 +2358,7 @@ async def test_guardrail_turn_detection_injection_tolerates_non_dict_value( logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.builds_setup_from_session_update = MagicMock(return_value=False) transformed_messages = [] def mock_transform(msg, model, session_config): @@ -2413,6 +2421,7 @@ async def test_subsequent_session_update_cannot_reenable_vad_when_guardrails_act logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.setup_session_update_rejection = MagicMock(return_value=None) transformed_messages = [] def mock_transform(msg, model, session_config): @@ -2465,6 +2474,7 @@ async def test_follow_up_setup_updates_cached_session_configuration_request(): logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.setup_session_update_rejection = MagicMock(return_value=None) follow_up_setup = json.dumps( { "setup": { @@ -2498,6 +2508,636 @@ async def test_follow_up_setup_updates_cached_session_configuration_request(): assert streaming.session_configuration_request == follow_up_setup +def _gemini_default_mode_streaming( + monkeypatch: pytest.MonkeyPatch, + client_messages: Sequence[str], + model: str = "gemini-live-2.5-flash-native-audio", + drop_params: bool = False, + defer_setup: bool = False, +) -> tuple[RealTimeStreaming, MagicMock]: + monkeypatch.setattr(litellm, "gemini_live_defer_setup", defer_setup, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws = MagicMock() + client_ws.receive_text = AsyncMock(side_effect=[*client_messages, ConnectionClosed(None, None)]) + client_ws.send_text = AsyncMock() + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + streaming = RealTimeStreaming( + client_ws, + backend_ws, + MagicMock(), + provider_config=GeminiRealtimeConfig(drop_params=drop_params), + model=model, + ) + return streaming, backend_ws + + +def _events_sent_to_client(streaming: RealTimeStreaming) -> list[dict]: + return [json.loads(call.args[0]) for call in streaming.websocket.send_text.await_args_list] + + +_TEXT_ITEM_CREATE = json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + } +) + +_RESPONSE_CREATE = json.dumps({"type": "response.create"}) + + +@pytest.mark.asyncio +async def test_default_mode_sends_default_setup_before_first_non_session_client_message(monkeypatch): + """Gemini Live setup is no longer pre-sent at connect, so a client that never + sends session.update still gets the provider default setup ahead of its + first content frame.""" + streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [_TEXT_ITEM_CREATE]) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"] + assert streaming.session_configuration_request == backend_ws.send.await_args_list[0].args[0] + + +@pytest.mark.asyncio +async def test_default_mode_session_update_first_builds_the_only_setup(monkeypatch): + session_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [session_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + + +@pytest.mark.asyncio +async def test_default_mode_merges_every_session_update_sent_before_the_first_content_frame(monkeypatch): + """Clients configure a session over several session.update messages. Gemini + Live takes one setup, so every update sent before the first content frame is + merged into that setup instead of only the first one applying.""" + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + modalities_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, modalities_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + + +@pytest.mark.asyncio +async def test_default_mode_merges_nested_audio_settings_across_session_updates(monkeypatch): + vad_update = json.dumps( + {"type": "session.update", "session": {"audio": {"input": {"turn_detection": {"silence_duration_ms": 1500}}}}} + ) + voice_update = json.dumps({"type": "session.update", "session": {"audio": {"output": {"voice": "Kore"}}}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [vad_update, voice_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + setup = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert setup["realtimeInputConfig"]["automaticActivityDetection"]["silenceDurationMs"] == 1500 + assert setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + + +def test_merge_session_fields_replaces_nested_mappings_wholesale_past_the_depth_cap(): + def nested(levels: int, key: str) -> dict: + return {key: levels, **({"child": nested(levels - 1, key)} if levels else {})} + + def level(node, depth: int): + return node if depth == 0 else level(node["child"], depth - 1) + + merged = _merge_session_fields(nested(10, "keep"), nested(10, "new")) + + assert all({"keep", "new"} <= set(level(merged, depth)) for depth in range(9)) + assert "keep" not in level(merged, 9) + assert level(merged, 9)["new"] == 1 + + +@pytest.mark.asyncio +async def test_default_mode_acknowledges_each_held_session_update_before_any_backend_frame(monkeypatch): + """A client that waits for session.updated before sending content must not + hang while its updates are held, so each one is acknowledged right away + with the modalities the merged setup will carry.""" + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + modalities_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, modalities_update], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + acks = [json.loads(call.args[0]) for call in streaming.websocket.send_text.await_args_list] + assert [ack["type"] for ack in acks] == ["session.updated", "session.updated"] + assert [ack["session"].get("output_modalities") or ack["session"].get("modalities") for ack in acks] == [ + ["audio"], + ["text"], + ] + assert acks[1]["session"]["instructions"] == "Be brief." + backend_ws.send.assert_not_awaited() + + +_LATE_INSTRUCTIONS_UPDATE = json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Be brief."}} +) + + +@pytest.mark.asyncio +async def test_default_mode_answers_a_session_update_after_the_first_content_with_an_error(monkeypatch): + streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [_TEXT_ITEM_CREATE, _LATE_INSTRUCTIONS_UPDATE]) + + await streaming.client_ack_messages() + + errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + + +@pytest.mark.asyncio +async def test_default_mode_drops_a_late_session_update_quietly_under_drop_params(monkeypatch): + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [_TEXT_ITEM_CREATE, _LATE_INSTRUCTIONS_UPDATE], drop_params=True + ) + + await streaming.client_ack_messages() + + assert _events_sent_to_client(streaming) == [] + assert backend_ws.send.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "requested_output"), + [ + ("gemini-live-2.5-flash-native-audio", {"modalities": ["text"]}), + ("gemini-live-2.5-flash", {"output_modalities": ["image"]}), + ], +) +async def test_default_mode_rejects_an_output_modality_the_model_cannot_produce(monkeypatch, model, requested_output): + update: Final = json.dumps( + {"type": "session.update", "session": {**requested_output, "instructions": "Be brief."}} + ) + streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [update, _TEXT_ITEM_CREATE], model=model) + + await streaming.client_ack_messages() + + events: Final = _events_sent_to_client(streaming) + assert [(event["type"], event["error"]["code"]) for event in events] == [("error", "invalid_value")] + setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert "systemInstruction" not in setup + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("modalities", "drop_params"), + [(["text", "audio"], False), (["text"], True)], + ids=["openai-default-pair", "text-dropped-under-drop-params"], +) +async def test_default_mode_applies_the_update_with_audio_output_on_an_audio_only_model( + monkeypatch, modalities, drop_params +): + update: Final = json.dumps( + {"type": "session.update", "session": {"modalities": modalities, "instructions": "Be brief."}} + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], drop_params=drop_params + ) + + await streaming.client_ack_messages() + + assert [event["type"] for event in _events_sent_to_client(streaming)] == ["session.updated"] + setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] + assert setup["systemInstruction"]["parts"][0]["text"] == "Be brief." + + +@pytest.mark.asyncio +async def test_default_mode_sends_no_extra_session_updated_when_gemini_confirms_the_setup(monkeypatch): + """Each held session.update was already acknowledged, so Gemini's setupComplete + must not reach the client as one more session.updated nobody asked for.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + model: Final = "gemini-live-2.5-flash-native-audio" + config: Final = GeminiRealtimeConfig() + client_ws: Final = MagicMock() + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=[json.dumps({"setupComplete": {}}).encode(), ConnectionClosed(None, None)]) + logging_obj: Final = _mock_logging() + logging_obj.litellm_trace_id = "trace_lazy" + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + streaming: Final = RealTimeStreaming(client_ws, backend_ws, logging_obj, provider_config=config, model=model) + streaming.session_created_sent_to_client = True + streaming.session_configuration_request = config.session_configuration_request(model) + + await streaming.backend_to_client_send_messages() + + assert [json.loads(call.args[0])["type"] for call in client_ws.send_text.await_args_list] == [] + + +@pytest.mark.asyncio +async def test_default_mode_folds_transcription_guardrail_vad_disable_into_the_merged_setup(monkeypatch): + """A transcription guardrail gates every turn, so its auto-response disable + has to land in the one setup built from the held session.updates, injected + once rather than into every update that gets merged.""" + monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()]) + debug = MagicMock() + monkeypatch.setattr("litellm.litellm_core_utils.realtime_streaming.verbose_logger.debug", debug) + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + vad_update = json.dumps( + {"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "silence_duration_ms": 900}}} + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, vad_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["realtimeInputConfig"]["automaticActivityDetection"] == {"disabled": True} + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + assert streaming._guardrail_turn_detection_update_sent is True + injected = [call for call in debug.call_args_list if str(call.args[0]).startswith("Injected turn_detection")] + assert len(injected) == 1 + + +def _blocking_text_guardrail() -> CustomGuardrail: + class _BlockingTextGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + if any("zebrafruit" in text for text in inputs.get("texts", [])): + raise ValueError("Content blocked") + return inputs + + return _BlockingTextGuardrail( + guardrail_name="test_text_guard", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + + +@pytest.mark.asyncio +async def test_default_mode_sends_the_setup_before_a_guardrail_blocks_the_first_message(monkeypatch): + """Gemini Live closes the socket when content arrives before the setup, so a + guardrail that blocks the first typed message must not reach the backend first.""" + monkeypatch.setattr(litellm, "callbacks", [_blocking_text_guardrail()]) + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + blocked_item = json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "zebrafruit"}]}, + } + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, blocked_item], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + assert "zebrafruit" not in json.dumps(frames[1]) + errors: Final = [event["error"]["type"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert errors == ["guardrail_violation"] + + +class _SteppedClock: + def __init__(self, epoch: float) -> None: + self.now = epoch + + def __call__(self) -> float: + return self.now + + +def _client_ws_receiving(*steps: Callable[[], Coroutine[object, object, str]]) -> MagicMock: + remaining: Final = iter(steps) + + async def receive_text() -> str: + return await next(remaining)() + + client_ws: Final = MagicMock() + client_ws.receive_text = receive_text + client_ws.send_text = AsyncMock() + return client_ws + + +@pytest.mark.asyncio +@pytest.mark.parametrize("epoch", [0.0, 1e9]) +async def test_default_mode_sends_the_held_setup_when_no_content_arrives_before_the_deadline(monkeypatch, epoch): + """Vertex AI refuses a setup sent minutes after connecting, so an idle session + sends what it holds at the deadline and answers any later session.update as late.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + clock: Final = _SteppedClock(epoch) + setup_sent: Final = asyncio.Event() + + async def instructions_update_then_the_deadline_passes() -> str: + clock.now += REALTIME_LAZY_SETUP_DEADLINE_SECONDS + return json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + + async def late_update_once_the_setup_is_out() -> str: + await setup_sent.wait() + return json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}} + ) + + async def disconnect() -> str: + raise ConnectionClosed(None, None) + + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock(side_effect=lambda frame: setup_sent.set()) + streaming: Final = RealTimeStreaming( + _client_ws_receiving(instructions_update_then_the_deadline_passes, late_update_once_the_setup_is_out, disconnect), + backend_ws, + MagicMock(), + provider_config=GeminiRealtimeConfig(), + model="gemini-live-2.5-flash", + clock=clock, + ) + + await asyncio.wait_for(streaming.client_ack_messages(), timeout=5) + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + events: Final = _events_sent_to_client(streaming) + assert [(event["type"], event.get("error", {}).get("code")) for event in events] == [ + ("session.updated", None), + ("error", "session_update_not_applied"), + ] + + +@pytest.mark.asyncio +async def test_default_mode_refuses_held_session_updates_past_the_size_cap(monkeypatch): + def instructions_update(text: str) -> str: + return json.dumps({"type": "session.update", "event_id": f"evt_{text[0]}", "session": {"instructions": text}}) + + half_the_cap: Final = RealTimeStreaming._MAX_BUFFERED_BYTES // 2 + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [instructions_update("a" * half_the_cap), instructions_update("b" * half_the_cap), _TEXT_ITEM_CREATE, _RESPONSE_CREATE], + model="gemini-live-2.5-flash", + ) + + await streaming.client_ack_messages() + + events: Final = _events_sent_to_client(streaming) + assert [(event["type"], event.get("error", {}).get("event_id")) for event in events] == [ + ("session.updated", None), + ("error", "evt_b"), + ] + assert events[1]["error"]["code"] == "session_update_too_large" + setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert setup["systemInstruction"]["parts"][0]["text"].startswith("a") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("defer_setup", [False, True]) +async def test_a_session_update_between_the_first_items_and_response_create_joins_the_setup(monkeypatch, defer_setup): + modalities_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + instructions_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [modalities_update, _TEXT_ITEM_CREATE, instructions_update, _RESPONSE_CREATE], + model="gemini-live-2.5-flash", + defer_setup=defer_setup, + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + assert [event["type"] for event in _events_sent_to_client(streaming)] == ["session.updated", "session.updated"] + + +@pytest.mark.asyncio +async def test_default_mode_applies_a_session_update_that_arrives_while_audio_streams_in(monkeypatch): + modalities_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"}) + instructions_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + commit: Final = json.dumps({"type": "input_audio_buffer.commit"}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [modalities_update, append, instructions_update, commit], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames][:2] == ["setup", "realtimeInput"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + assert frames[1]["realtimeInput"]["audio"]["data"] == "AAAA" + assert "error" not in [event["type"] for event in _events_sent_to_client(streaming)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("epoch", [0.0, 1e9]) +async def test_held_items_go_out_with_the_setup_when_no_response_create_follows(monkeypatch, epoch): + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + item_sent: Final = asyncio.Event() + + async def instructions_update() -> str: + return json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + + async def text_item() -> str: + return _TEXT_ITEM_CREATE + + async def disconnect_once_the_item_is_out() -> str: + await item_sent.wait() + raise ConnectionClosed(None, None) + + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock( + side_effect=lambda frame: item_sent.set() if "clientContent" in json.loads(frame) else None + ) + streaming: Final = RealTimeStreaming( + _client_ws_receiving(instructions_update, text_item, disconnect_once_the_item_is_out), + backend_ws, + MagicMock(), + provider_config=GeminiRealtimeConfig(), + model="gemini-live-2.5-flash", + lazy_setup_content_grace_seconds=0, + clock=_SteppedClock(epoch), + ) + + await asyncio.wait_for(streaming.client_ack_messages(), timeout=5) + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + + +@pytest.mark.asyncio +async def test_default_mode_acknowledges_an_unchanged_session_update_after_content(monkeypatch): + """Pipecat resends its unchanged session.update right after its first + conversation items and treats any error event as fatal.""" + session_update: Final = json.dumps( + {"type": "session.update", "session": {"modalities": ["text"], "instructions": "Be brief."}} + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [session_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE, session_update], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + events: Final = _events_sent_to_client(streaming) + assert [event["type"] for event in events] == ["session.updated", "session.updated"] + assert events[0]["session"] == events[1]["session"] + + +@pytest.mark.asyncio +async def test_deferred_setup_merges_session_updates_until_the_first_content(monkeypatch): + """Deferred setup merges every session.update sent before the first content, + the same as the default mode, and answers a changed one after it as late.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws: Final = MagicMock() + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}), + json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}), + _TEXT_ITEM_CREATE, + _RESPONSE_CREATE, + json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}} + ), + ConnectionClosed(None, None), + ] + ) + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, MagicMock(), provider_config=GeminiRealtimeConfig(), model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] + + +@pytest.mark.asyncio +async def test_deferred_setup_sends_the_default_setup_before_text_content_that_comes_first(monkeypatch): + """Gemini Live closes the socket on content sent before any setup, so text that arrives + before a session.update in deferred mode gets the model's default setup first.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws: Final = MagicMock() + client_ws.receive_text = AsyncMock( + side_effect=[ + _TEXT_ITEM_CREATE, + json.dumps({"type": "session.update", "event_id": "evt_late", "session": {"modalities": ["text"]}}), + ConnectionClosed(None, None), + ] + ) + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, MagicMock(), provider_config=GeminiRealtimeConfig(), model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] + + +@pytest.mark.asyncio +async def test_deferred_setup_sends_a_session_update_that_follows_buffered_audio_at_once(monkeypatch): + append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"}) + commit: Final = json.dumps({"type": "input_audio_buffer.commit"}) + session_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be a pirate."}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [append, commit, session_update], model="gemini-live-2.5-flash", defer_setup=True + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup"] + assert "Be a pirate." in json.dumps(frames[0]["setup"]["systemInstruction"]) + + +@pytest.mark.parametrize("defer_setup", [False, True]) +@pytest.mark.asyncio +async def test_a_transcription_guardrail_refuses_gemini_audio_it_cannot_screen(monkeypatch, defer_setup): + monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()]) + session_update: Final = json.dumps( + {"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "create_response": False}}} + ) + append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"}) + commit: Final = json.dumps({"type": "input_audio_buffer.commit", "event_id": "commit_1"}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [session_update, append, append, commit, json.dumps({"type": "response.create"}), _TEXT_ITEM_CREATE], + model="gemini-live-2.5-flash", + defer_setup=defer_setup, + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + refusals: Final = [ + event["error"]["event_id"] + for event in _events_sent_to_client(streaming) + if event["type"] == "error" and event["error"]["code"] == "audio_input_not_screenable" + ] + assert refusals == [None, "commit_1"] + + +@pytest.mark.parametrize("defer_setup", [False, True]) +@pytest.mark.parametrize("control_type", ["response.cancel", "conversation.item.delete", "input_audio_buffer.clear"]) +@pytest.mark.asyncio +async def test_a_control_event_before_session_update_leaves_the_setup_open(monkeypatch, defer_setup, control_type): + control_event: Final = json.dumps({"type": control_type, "item_id": "item_x"}) + session_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [control_event, session_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], + model="gemini-live-2.5-flash", + defer_setup=defer_setup, + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert [event["type"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] == [] + + @pytest.mark.asyncio async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeypatch): """Pipecat may send audio before session.update when setup is deferred.""" @@ -2533,14 +3173,18 @@ async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeyp @pytest.mark.asyncio -async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeypatch): +async def test_deferred_setup_sends_the_held_session_update_when_more_audio_arrives(monkeypatch): monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig client_ws = MagicMock() + client_ws.send_text = AsyncMock() audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AA=="}) session_update = json.dumps({"type": "session.update", "session": {"modalities": ["audio"]}}) - client_ws.receive_text = AsyncMock(side_effect=[audio_msg, session_update, ConnectionClosed(None, None)]) + later_audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AQ=="}) + client_ws.receive_text = AsyncMock( + side_effect=[audio_msg, session_update, later_audio_msg, ConnectionClosed(None, None)] + ) backend_ws = MagicMock() backend_ws.send = AsyncMock() logging_obj: Final = _mock_logging() @@ -2560,7 +3204,7 @@ async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeyp sent_payload = json.loads(backend_ws.send.await_args_list[0].args[0]) assert "setup" in sent_payload assert "realtimeInput" not in sent_payload - assert streaming._pending_messages_until_setup == [audio_msg] + assert streaming._pending_messages_until_setup == [audio_msg, later_audio_msg] @pytest.mark.asyncio @@ -3523,6 +4167,24 @@ async def test_upstream_refusal_after_a_synthetic_session_created_still_logs_a_f assert session.logging.logged_sessions == () +class _LazySetupRealtimeConfig(_PassthroughRealtimeConfig): + def builds_setup_from_session_update(self) -> bool: + return True + + +@pytest.mark.asyncio +async def test_upstream_outage_before_a_held_setup_was_sent_logs_the_session_not_a_refusal(): + upstream_close: Final = ConnectionClosed(Close(1011, "upstream outage"), None) + session: Final = _relay_session( + _client_ws_that_never_sends(), _backend_ws_closing_with(upstream_close), _LazySetupRealtimeConfig() + ) + + await session.run() + + assert session.logging.logged_failures == () + assert len(session.logging.logged_sessions) == 1 + + @pytest.mark.asyncio async def test_upstream_close_after_relayed_events_still_logs_the_session_as_success(): client_ws: Final = _client_ws_that_never_sends() diff --git a/tests/unit/litellm_core_utils/test_safe_json_dumps.py b/tests/unit/litellm_core_utils/test_safe_json_dumps.py index 1f2664f33cb..1b8bdd21326 100644 --- a/tests/unit/litellm_core_utils/test_safe_json_dumps.py +++ b/tests/unit/litellm_core_utils/test_safe_json_dumps.py @@ -169,6 +169,31 @@ def test_null_byte_stripped_in_fallback_str(): assert json.loads(out)["obj"] == "objrepr" +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_non_finite_floats_are_written_as_json_null(value: float) -> None: + out = safe_dumps( + {"cost_breakdown": {"total_cost": value, "output_cost": 0.006}, "costs": [value]}, non_finite_as_null=True + ) + + assert json.loads(out, parse_constant=lambda token: pytest.fail(f"non-JSON token {token}")) == { + "cost_breakdown": {"total_cost": None, "output_cost": 0.006}, + "costs": [None], + } + + +def test_a_set_holding_a_non_finite_float_sorts_its_finite_members_and_puts_null_last() -> None: + assert json.loads(safe_dumps({"costs": {2.0, float("inf"), 1.0}}, non_finite_as_null=True)) == { + "costs": [1.0, 2.0, None] + } + + +def test_non_finite_floats_keep_their_value_unless_the_caller_asks_for_null() -> None: + assert json.loads(safe_dumps({"timeout": float("inf"), "costs": [float("-inf")]})) == { + "timeout": float("inf"), + "costs": [float("-inf")], + } + + def test_clean_strings_are_not_run_through_replace(): """Regression for LIT-3910. diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index 28587bf4749..c8fa9c94c17 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1077,6 +1077,47 @@ def test_cost_field_in_usage_chunks(): assert usage.completion_tokens == 5 + +def _perplexity_usage_chunk(cost: Mapping[str, float], finish_reason: str | None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-pplx", + created=1745513206, + model="sonar", + choices=[StreamingChoices(finish_reason=finish_reason, index=0, delta=Delta(content="ok"))], + usage={"prompt_tokens": 26, "completion_tokens": 1, "total_tokens": 27, "cost": dict(cost)}, + ) + + +@pytest.mark.parametrize("poisoned_field", ["request_cost", "total_cost"]) +def test_a_non_finite_final_stream_cost_does_not_fall_back_to_an_earlier_chunk_cost(poisoned_field: str): + final_cost: Final = {"input_tokens_cost": 0.0012, "output_tokens_cost": 1e-05, "request_cost": 0.0025} + chunks: Final = [ + _perplexity_usage_chunk({"input_tokens_cost": 0, "output_tokens_cost": 0, "total_cost": 0}, None), + _perplexity_usage_chunk({**final_cost, "total_cost": 0.00371, poisoned_field: float("nan")}, "stop"), + ] + + response: Final = stream_chunk_builder(chunks=chunks) + + assert response is not None + assert getattr(response.usage, "cost", None) is None + assert response.usage.total_tokens == 27 + + +@pytest.mark.parametrize("provider_key", ["dropped_non_finite_cost", "_dropped_non_finite_cost"]) +def test_a_provider_usage_key_named_like_the_dropped_cost_flag_keeps_the_stream_cost(provider_key: str): + chunk: Final = ModelResponseStream( + id="chatcmpl-flag-key", + created=1745513206, + model="sonar", + choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="ok"))], + usage={"prompt_tokens": 26, "completion_tokens": 1, "total_tokens": 27, "cost": 0.00371, provider_key: True}, + ) + + response: Final = stream_chunk_builder(chunks=[chunk]) + + assert response is not None + assert getattr(response.usage, "cost", None) == 0.00371 + def test_stream_chunk_builder_tolerates_trailing_chunk_without_choices(): """Regression for https://github.com/BerriAI/litellm/issues/32051 diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 41e8c50fa85..b9c8feb223d 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -2,12 +2,15 @@ import asyncio import datetime import itertools import json +import os import queue +import subprocess import sys import threading import time from collections.abc import Mapping, Sequence from dataclasses import dataclass +from pathlib import Path from types import CodeType, FrameType, MappingProxyType from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -33,6 +36,7 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler from litellm.responses.mcp.chat_completions_handler import _add_mcp_metadata_to_response from litellm.types.utils import ( + ChoiceLogprobs, CompletionTokensDetailsWrapper, Delta, ModelResponse, @@ -1624,6 +1628,356 @@ async def test_stream_with_null_choices_chunk_finishes_normally( assert [choice.finish_reason for choice in streamed_choices if choice.finish_reason] == ["stop"] +def _token_logprobs(token: str) -> dict[str, object]: + return { + "content": [{"token": token, "logprob": -0.5, "bytes": list(token.encode()), "top_logprobs": []}], + "refusal": None, + } + + +@pytest.mark.parametrize("async_stream", [False, True]) +@pytest.mark.parametrize("transport", ["sdk", "http_handler"]) +@pytest.mark.parametrize( + ("finish_chunk_extra", "stream_options", "expected_usage_totals"), + [ + pytest.param({}, None, [], id="finish-chunk-with-logprobs"), + pytest.param( + {"usage": {"prompt_tokens": 3, "completion_tokens": 3, "total_tokens": 6}}, + {"include_usage": True}, + [6], + id="finish-chunk-with-logprobs-and-usage", + ), + ], +) +async def test_stream_finish_chunk_with_logprobs_reaches_caller_as_litellm_logprobs( + async_stream: bool, + transport: str, + finish_chunk_extra: dict[str, object], + stream_options: dict[str, bool] | None, + expected_usage_totals: list[int], + logging_obj: Logging, +) -> None: + upstream_tokens: Final = ("Hel", "lo", "<|return|>") + upstream_chunks: Final = ( + { + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "Hel"}, + "logprobs": _token_logprobs("Hel"), + "finish_reason": None, + } + ] + }, + { + "choices": [ + {"index": 0, "delta": {"content": "lo"}, "logprobs": _token_logprobs("lo"), "finish_reason": None} + ] + }, + { + "choices": [{"index": 0, "delta": {}, "logprobs": _token_logprobs("<|return|>"), "finish_reason": "stop"}], + **finish_chunk_extra, + }, + ) + body: Final = ( + "".join( + "data: " + + json.dumps( + { + "id": "chatcmpl-logprobs", + "object": "chat.completion.chunk", + "created": 1, + "model": "test-model", + **chunk, + } + ) + + "\n\n" + for chunk in upstream_chunks + ) + + "data: [DONE]\n\n" + ) + raw_response: Final = httpx.Response( + 200, + content=body, + headers={"content-type": "text/event-stream"}, + request=httpx.Request("POST", "https://provider.example/v1/chat/completions"), + ) + client: Final = AsyncOpenAI(api_key="unused") if async_stream else OpenAI(api_key="unused") + stream: Final = ( + ( + AsyncStream(cast_to=ChatCompletionChunk, response=raw_response, client=client) + if isinstance(client, AsyncOpenAI) + else Stream(cast_to=ChatCompletionChunk, response=raw_response, client=client) + ) + if transport == "sdk" + else OpenAIChatCompletionStreamingHandler( + streaming_response=raw_response.aiter_lines() if async_stream else raw_response.iter_lines(), + sync_stream=not async_stream, + ) + ) + response: Final = CustomStreamWrapper( + completion_stream=stream, + model="test-model", + logging_obj=logging_obj, + custom_llm_provider="openai" if transport == "sdk" else "hosted_vllm", + stream_options=stream_options, + ) + + try: + chunks: Final = [chunk async for chunk in response] if async_stream else list(response) + finally: + if isinstance(client, AsyncOpenAI): + await client.close() + else: + client.close() + + wire_choices: Final = tuple( + itertools.chain.from_iterable( + json.loads(chunk.model_dump_json(exclude_none=True, exclude_unset=True))["choices"] for chunk in chunks + ) + ) + finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")] + streamed_logprobs: Final = tuple( + choice.logprobs + for choice in itertools.chain.from_iterable(chunk.choices for chunk in chunks) + if choice.logprobs is not None + ) + streamed_tokens: Final = tuple( + entry.token for entry in itertools.chain.from_iterable(logprobs.content or () for logprobs in streamed_logprobs) + ) + + assert [choice["finish_reason"] for choice in finish_choices] == ["stop"] + assert [entry["token"] for entry in finish_choices[0]["logprobs"]["content"]] == [upstream_tokens[-1]] + assert all(isinstance(logprobs, ChoiceLogprobs) for logprobs in streamed_logprobs) + assert streamed_tokens == upstream_tokens + assert [chunk.usage.total_tokens for chunk in chunks if getattr(chunk, "usage", None)] == expected_usage_totals + + +async def _sdk_stream_chunks( + upstream_chunks: Sequence[Mapping[str, object]], + async_stream: bool, + logging_obj: Logging, + stream_options: dict[str, bool] | None = None, +) -> list[ModelResponseStream]: + body: Final = ( + "".join( + "data: " + + json.dumps( + { + "id": "chatcmpl-logprobs", + "object": "chat.completion.chunk", + "created": 1, + "model": "test-model", + **chunk, + } + ) + + "\n\n" + for chunk in upstream_chunks + ) + + "data: [DONE]\n\n" + ) + raw_response: Final = httpx.Response( + 200, + content=body, + headers={"content-type": "text/event-stream"}, + request=httpx.Request("POST", "https://provider.example/v1/chat/completions"), + ) + client: Final = AsyncOpenAI(api_key="unused") if async_stream else OpenAI(api_key="unused") + stream: Final = ( + AsyncStream(cast_to=ChatCompletionChunk, response=raw_response, client=client) + if isinstance(client, AsyncOpenAI) + else Stream(cast_to=ChatCompletionChunk, response=raw_response, client=client) + ) + response: Final = CustomStreamWrapper( + completion_stream=stream, + model="test-model", + logging_obj=logging_obj, + custom_llm_provider="openai", + stream_options=stream_options, + ) + try: + return [chunk async for chunk in response] if async_stream else list(response) + finally: + if isinstance(client, AsyncOpenAI): + await client.close() + else: + client.close() + + +@pytest.mark.parametrize("async_stream", [False, True]) +@pytest.mark.parametrize( + ("upstream_logprobs", "finish_chunk_extra", "stream_options", "expected_usage_totals"), + [ + pytest.param( + {"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None}, + {}, + None, + [], + id="null-logprob", + ), + pytest.param( + {"content": [{"token": "<|return|>", "logprob": None}]}, + {}, + None, + [], + id="null-logprob-keeps-only-the-keys-upstream-sent", + ), + pytest.param(-0.25, {}, None, [], id="scalar-logprobs"), + pytest.param( + {"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None}, + {"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}}, + {"include_usage": True}, + [5], + id="null-logprob-and-usage", + ), + ], +) +async def test_stream_with_nonstandard_logprobs_passes_them_through_and_finishes( + async_stream: bool, + upstream_logprobs: object, + finish_chunk_extra: dict[str, object], + stream_options: dict[str, bool] | None, + expected_usage_totals: list[int], + logging_obj: Logging, +) -> None: + chunks: Final = await _sdk_stream_chunks( + ( + { + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "hi"}, + "logprobs": upstream_logprobs, + "finish_reason": None, + } + ] + }, + { + "choices": [{"index": 0, "delta": {}, "logprobs": upstream_logprobs, "finish_reason": "stop"}], + **finish_chunk_extra, + }, + ), + async_stream, + logging_obj, + stream_options, + ) + + wire_choices: Final = tuple( + itertools.chain.from_iterable( + json.loads(chunk.model_dump_json(exclude_none=True, exclude_unset=True))["choices"] for chunk in chunks + ) + ) + logprobs_choices: Final = [choice for choice in wire_choices if "logprobs" in choice] + + assert [choice.get("finish_reason") for choice in logprobs_choices] == [None, "stop"] + assert [choice["logprobs"] for choice in logprobs_choices] == [upstream_logprobs, upstream_logprobs] + assert [chunk.usage.total_tokens for chunk in chunks if getattr(chunk, "usage", None)] == expected_usage_totals + + +@pytest.mark.parametrize("async_stream", [False, True]) +@pytest.mark.parametrize("logprobs_field", ["content", "refusal"]) +async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logprobs( + async_stream: bool, logprobs_field: Literal["content", "refusal"], logging_obj: Logging +) -> None: + chunks: Final = await _sdk_stream_chunks( + tuple( + { + "choices": [ + { + "index": 0, + "delta": delta, + "logprobs": {logprobs_field: [{"token": token, "logprob": -0.5}]}, + "finish_reason": finish_reason, + } + ] + } + for token, delta, finish_reason in ( + ("hi", {"role": "assistant", "content": "hi"}, None), + ("<|return|>", {}, "stop"), + ) + ), + async_stream, + logging_obj, + ) + + streamed_logprobs: Final = tuple( + choice.logprobs + for choice in itertools.chain.from_iterable(chunk.choices for chunk in chunks) + if choice.logprobs is not None + ) + streamed_entries: Final = tuple( + itertools.chain.from_iterable( + (logprobs.content if logprobs_field == "content" else logprobs.refusal) or () + for logprobs in streamed_logprobs + ) + ) + + assert [(entry.token, entry.logprob, entry.top_logprobs) for entry in streamed_entries] == [ + ("hi", -0.5, []), + ("<|return|>", -0.5, []), + ] + + +FINISH_ONLY_LOGPROBS_STREAM_SCRIPT: Final = """ +import json +import sys + +sys.path.insert(0, sys.argv[1]) + +import httpx +from openai import OpenAI + +import litellm + +upstream_choices = ( + {"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}, + { + "index": 0, + "delta": {}, + "logprobs": {"content": [{"token": "<|return|>", "logprob": -0.5, "top_logprobs": []}]}, + "finish_reason": "stop", + }, +) +body = "".join( + "data: " + + json.dumps({"id": "c", "object": "chat.completion.chunk", "created": 1, "model": "m", "choices": [choice]}) + + "\\n\\n" + for choice in upstream_choices +) + "data: [DONE]\\n\\n" +client = OpenAI( + api_key="unused", + base_url="https://provider.example/v1", + http_client=httpx.Client( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"}) + ) + ), +) +stream = litellm.completion( + model="openai/m", messages=[{"role": "user", "content": "hi"}], stream=True, logprobs=True, client=client +) +print(json.dumps([json.loads(chunk.model_dump_json(exclude_none=True)) for chunk in stream])) +""" + + +def test_stream_with_logprobs_only_on_finish_chunk_serializes_in_a_fresh_process() -> None: + result: Final = subprocess.run( + [sys.executable, "-I", "-c", FINISH_ONLY_LOGPROBS_STREAM_SCRIPT, str(Path(litellm.__file__).parent.parent)], + capture_output=True, + text=True, + timeout=120, + env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, + ) + + assert result.returncode == 0, result.stdout + result.stderr + wire_choices: Final = tuple( + itertools.chain.from_iterable(chunk["choices"] for chunk in json.loads(result.stdout.strip().splitlines()[-1])) + ) + finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")] + assert [choice["finish_reason"] for choice in finish_choices] == ["stop"] + assert [entry["token"] for entry in finish_choices[0]["logprobs"]["content"]] == ["<|return|>"] + + def _hosted_vllm_stream_wrapper(logging_obj: Logging, error_payload: dict) -> CustomStreamWrapper: """A CustomStreamWrapper over the real OpenAI-compatible line iterator, fed an HTTP 200 SSE body that carries an in-body error payload the way diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index d815e1eedd3..c2ee58b67ed 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -6,6 +6,7 @@ with guardrail transformations, specifically testing edge cases with empty choic """ import json +import logging import re from typing import Any, Literal, Optional, Final from unittest.mock import MagicMock, patch @@ -3391,3 +3392,18 @@ class TestAnthropicMessagesHandlerPostCallHookResponse: native = {"type": "message", "role": "assistant", "content": [{"type": "text", "text": "hi"}]} assert AnthropicMessagesHandler().post_call_hook_response(native) is native + + +def test_reading_a_dropped_budget_for_a_guardrail_logs_no_drop_warning(caplog: pytest.LogCaptureFixture) -> None: + data: Final = { + "model": "gemini-2.5-flash", + "max_tokens": 64, + "thinking": {"type": "enabled", "budget_tokens": "high"}, + "messages": [{"role": "user", "content": "hi"}], + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + messages: Final = AnthropicMessagesHandler().get_structured_messages(data) + + assert messages == [{"role": "user", "content": "hi"}] + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 4d5d6cc17a9..cbb60caa57b 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,5 +1,6 @@ import copy import json +import logging from itertools import chain from typing import Final from unittest.mock import MagicMock, patch @@ -3189,6 +3190,47 @@ def test_reasoning_effort_accepts_dict_shape_for_non_adaptive_model( ) +def _reasoning_effort_drop_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + record.getMessage() for record in caplog.records if "Dropping reasoning_effort=" in record.getMessage() + ) + + +@pytest.mark.parametrize("reasoning_effort_value", [5, {"effort": 5, "summary": "concise"}]) +def test_reasoning_effort_that_is_not_a_level_name_is_dropped_with_one_warning( + reasoning_effort_value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = AnthropicConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort_value}, + optional_params={}, + model="claude-sonnet-4-6-20260219", + drop_params=False, + ) + + warnings: Final = _reasoning_effort_drop_warnings(caplog) + assert "thinking" not in result + assert "output_config" not in result + assert len(warnings) == 1 + assert f"reasoning_effort={reasoning_effort_value!r}" in warnings[0] + assert "claude-sonnet-4-6-20260219" in warnings[0] + + +@pytest.mark.parametrize("reasoning_effort_value", [{"summary": "concise"}, {"effort": "low", "summary": "concise"}]) +def test_reasoning_effort_with_a_level_name_or_no_level_logs_no_drop( + reasoning_effort_value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + AnthropicConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort_value}, + optional_params={}, + model="claude-sonnet-4-6-20260219", + drop_params=False, + ) + + assert _reasoning_effort_drop_warnings(caplog) == () + + @pytest.mark.parametrize( "model,budget_tokens,expected", [ diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index c30644e7fdc..e0882653324 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -1,6 +1,7 @@ import base64 import functools import json +import logging from collections.abc import Mapping from types import MappingProxyType from typing import Any, Final, cast @@ -2871,14 +2872,21 @@ def test_translate_completion_input_params_rejects_a_non_numeric_budget_without_ @pytest.mark.parametrize("budget", ["2048", True]) def test_translate_completion_input_params_drops_a_non_numeric_budget_under_drop_params( - budget: object, monkeypatch: pytest.MonkeyPatch + budget: object, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ): monkeypatch.setattr(litellm, "drop_params", False) - translated: Final = AnthropicAdapter().translate_completion_input_params( - _gemini_request_with_budget(budget, drop_params=True) - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + translated: Final = AnthropicAdapter().translate_completion_input_params( + _gemini_request_with_budget(budget, drop_params=True) + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert translated is not None assert not {"reasoning_effort", "thinking"} & translated.keys() + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (f"thinking.budget_tokens={budget!r}", "gemini-2.5-flash", "(drop_params=True)") + ) def test_translate_openai_content_to_anthropic_reasoning_content_without_thinking_blocks(): diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py index ed6cd6134ff..77bdacc938f 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py @@ -1,6 +1,7 @@ import json import os import sys +from typing import Final import pytest @@ -70,3 +71,70 @@ def test_prepare_completion_kwargs_keeps_prompt_cache_key_through_responses_rero ) assert completion_kwargs["model"] == "openai/responses/gpt-5.6-luna" assert completion_kwargs["prompt_cache_key"] == "session-abc" + + +@pytest.mark.parametrize( + ("api_base", "azure_api_base_env", "derives_key"), + [ + (None, None, True), + ("https://api.mistral.ai/v1", None, True), + ("https://my-resource.openai.azure.com/openai/v1", None, False), + (None, "https://my-resource.services.ai.azure.com/openai", False), + ], +) +def test_prepare_completion_kwargs_derives_prompt_cache_key_only_for_a_mistral_hosted_endpoint( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, azure_api_base_env: str | None, derives_key: bool +) -> None: + if azure_api_base_env is None: + monkeypatch.delenv("MISTRAL_AZURE_API_BASE", raising=False) + else: + monkeypatch.setenv("MISTRAL_AZURE_API_BASE", azure_api_base_env) + + completion_kwargs: Final = _prepare( + "mistral/mistral-small-latest", {"custom_llm_provider": "mistral", "api_base": api_base} + ) + + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID + assert ("prompt_cache_key" in completion_kwargs) is derives_key + + +@pytest.mark.parametrize( + ("codestral_api_base_env", "derives_key"), + [(None, True), ("https://my-resource.services.ai.azure.com/openai/v1", False)], +) +def test_prepare_completion_kwargs_derives_prompt_cache_key_for_codestral_only_on_its_mistral_host( + monkeypatch: pytest.MonkeyPatch, codestral_api_base_env: str | None, derives_key: bool +) -> None: + monkeypatch.delenv("MISTRAL_AZURE_API_BASE", raising=False) + if codestral_api_base_env is None: + monkeypatch.delenv("CODESTRAL_API_BASE", raising=False) + else: + monkeypatch.setenv("CODESTRAL_API_BASE", codestral_api_base_env) + + completion_kwargs: Final = _prepare("codestral/codestral-latest", {"custom_llm_provider": "codestral"}) + + assert ("prompt_cache_key" in completion_kwargs) is derives_key + + +def test_prepare_completion_kwargs_derives_no_prompt_cache_key_mistral_cannot_take_from_a_non_ascii_session() -> None: + non_ascii_user_id: Final = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "sitzung-ü"}) + + mistral_kwargs: Final = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="mistral/mistral-small-latest", + metadata={"user_id": non_ascii_user_id}, + thinking=None, + extra_kwargs={"custom_llm_provider": "mistral"}, + )[0] + openai_kwargs: Final = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + metadata={"user_id": non_ascii_user_id}, + thinking=None, + extra_kwargs={"custom_llm_provider": "openai"}, + )[0] + + assert "prompt_cache_key" not in mistral_kwargs + assert openai_kwargs["prompt_cache_key"] == "sitzung-ü" diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index e504513dd36..27ed1f5ebd6 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -8,6 +8,7 @@ leaves the adapter, in the shape the target expects. import functools import json +import logging from collections.abc import Mapping from typing import Final @@ -510,26 +511,33 @@ class TestADeclaredDeploymentRoundsTheMessagesTierOntoItsLevels: @pytest.mark.asyncio @pytest.mark.parametrize("budget", ["2048", True]) @pytest.mark.parametrize("litellm_params, model_info, upstream_url, upstream_body", _MALFORMED_BUDGET_TARGETS) - async def test_drop_params_drops_a_non_numeric_budget( + async def test_drop_params_drops_a_non_numeric_budget_with_one_warning( self, local_model_cost_map: None, respx_mock: respx.MockRouter, + caplog: pytest.LogCaptureFixture, litellm_params: Mapping[str, object], model_info: Mapping[str, object], upstream_url: str, upstream_body: Mapping[str, object], budget: object, ) -> None: - sent: Final = await _body_sent_upstream( - respx_mock, - {**litellm_params, "drop_params": True}, - model_info, - upstream_url, - upstream_body, - thinking={"type": "enabled", "budget_tokens": budget}, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + sent: Final = await _body_sent_upstream( + respx_mock, + {**litellm_params, "drop_params": True}, + model_info, + upstream_url, + upstream_body, + thinking={"type": "enabled", "budget_tokens": budget}, + ) assert not {"reasoning_effort", "reasoning", "thinking"} & sent.keys() + dropped: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(dropped) == 1 + assert f"thinking.budget_tokens={budget!r}" in dropped[0] + assert "(drop_params=True)" in dropped[0] + assert str(litellm_params["model"]).split("/", 1)[1] in dropped[0] @pytest.mark.asyncio @pytest.mark.parametrize( diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 5ce6d03f477..af3108e5072 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -159,7 +159,7 @@ async def test_openai_model_does_not_forward_stream_options_to_responses_api(): await litellm.anthropic.messages.acreate( max_tokens=100, messages=[{"role": "user", "content": "Hello, how are you?"}], - model="openai/gpt-5.5", + model="openai/gpt-6.1-sol", api_key="test-api-key", stream_options={"include_usage": True}, ) @@ -669,7 +669,9 @@ class TestThinkingSummaryPreservation: ) thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( + thinking, model="openai/gpt-6.1-sol" + ) assert result == {"effort": "high", "summary": "concise"} def test_responses_adapter_no_summary_by_default(self): @@ -683,7 +685,9 @@ class TestThinkingSummaryPreservation: try: litellm.reasoning_auto_summary = False thinking = {"type": "enabled", "budget_tokens": 5000} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( + thinking, model="openai/gpt-6.1-sol" + ) assert result == {"effort": "high"} assert result is not None and "summary" not in result finally: diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py index 596cb968a0e..1d0d70d8e60 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import base64 import json +import logging import struct import time import uuid @@ -891,24 +892,32 @@ async def test_native_messages_send_no_effort_for_a_default_effort_beside_a_thin @pytest.mark.asyncio @pytest.mark.parametrize("model_info", [None, _DECLARING_LOW_AND_HIGH]) @pytest.mark.parametrize("drop_globally", [False, True]) -async def test_native_messages_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( +async def test_native_messages_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude_with_one_warning( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, model_info: Mapping[str, object] | None, drop_globally: bool, ) -> None: monkeypatch.setattr(litellm, "drop_params", drop_globally) request_drop: Final = {} if drop_globally else {"drop_params": True} - dropped: Final = await _native_opus_body_sent( - respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + dropped: Final = await _native_opus_body_sent( + respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] adaptive_alone: Final = await _native_opus_body_sent( respx_mock, model_info, thinking={"type": "adaptive"}, **request_drop ) assert "output_config" not in adaptive_alone assert dropped == adaptive_alone + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("thinking.budget_tokens='lots'", "claude-opus-5-5", "(drop_params=True)") + ) _MIXED_GROUP: Final = "gpt-5.6-mixed" diff --git a/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py b/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py index 50dc0deabfd..28abb348371 100644 --- a/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py +++ b/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py @@ -1,5 +1,8 @@ """Tests for ``reasoning_effort`` translation on the Anthropic /v1/messages route.""" +import logging +from typing import Final + import pytest from litellm.constants import ( @@ -46,6 +49,49 @@ def test_reasoning_effort_maps_to_output_config_for_adaptive_model( assert result.get("output_config") == {"effort": expected_effort} +def _reasoning_effort_drop_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + record.getMessage() for record in caplog.records if "Dropping reasoning_effort=" in record.getMessage() + ) + + +@pytest.mark.parametrize("reasoning_effort", [5, {"effort": "high"}]) +def test_reasoning_effort_that_is_not_a_string_is_dropped_with_one_warning( + reasoning_effort: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-4-7", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={"max_tokens": 1024, "reasoning_effort": reasoning_effort}, + litellm_params={}, + headers={}, + ) + + warnings: Final = _reasoning_effort_drop_warnings(caplog) + assert "reasoning_effort" not in result + assert "output_config" not in result + assert len(warnings) == 1 + assert f"reasoning_effort={reasoning_effort!r}" in warnings[0] + assert "claude-opus-4-7" in warnings[0] + + +@pytest.mark.parametrize("reasoning_effort", ["high", {"summary": "concise"}]) +def test_reasoning_effort_that_is_a_string_or_names_no_level_logs_no_drop( + reasoning_effort: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-4-7", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={"max_tokens": 1024, "reasoning_effort": reasoning_effort}, + litellm_params={}, + headers={}, + ) + + assert _reasoning_effort_drop_warnings(caplog) == () + + def test_reasoning_effort_none_clears_thinking_and_output_config(): config = AnthropicMessagesConfig() optional_params = { diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index f6fcb9e6f1e..ece0cf5d423 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -44,6 +44,7 @@ def _make_request(**overrides) -> AnthropicMessagesRequest: _ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() +_MODEL: Final = "openai/gpt-6.1-sol" # --------------------------------------------------------------------------- @@ -957,13 +958,13 @@ class TestTranslateThinkingToReasoning: """Anthropic thinking param -> Responses API reasoning param.""" def test_budget_high_effort(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}, model=_MODEL) # Default (reasoning_auto_summary=False): only effort, no summary assert result == {"effort": "high"} assert result is not None and "summary" not in result def test_budget_above_threshold_high_effort(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 50000}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 50000}, model=_MODEL) assert result is not None assert result["effort"] == "high" assert "summary" not in result @@ -973,7 +974,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result == {"effort": "medium"} assert result is not None and "summary" not in result @@ -983,13 +985,14 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result == {"effort": "low"} assert result is not None and "summary" not in result def test_budget_minimal_effort(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 500}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 500}, model=_MODEL) assert result == {"effort": "minimal"} assert result is not None and "summary" not in result @@ -998,7 +1001,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result_high is not None assert result_high["effort"] == "high" @@ -1006,7 +1010,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result_medium is not None assert result_medium["effort"] == "medium" @@ -1015,23 +1020,24 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result_low is not None assert result_low["effort"] == "low" assert "summary" not in result_low def test_disabled_type_returns_none(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}, model=_MODEL) assert result is None def test_non_dict_returns_none(self): - result = _ADAPTER.translate_thinking_to_reasoning("enabled") # type: ignore + result = _ADAPTER.translate_thinking_to_reasoning("enabled", model=_MODEL) # type: ignore assert result is None def test_missing_budget_defaults_to_minimal(self): """Missing budget_tokens defaults to 0, below the low threshold -> minimal.""" - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}, model=_MODEL) assert result == {"effort": "minimal"} assert result is not None and "summary" not in result @@ -1042,7 +1048,7 @@ class TestTranslateThinkingToReasoning: original = litellm.reasoning_auto_summary try: litellm.reasoning_auto_summary = True - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}, model=_MODEL) assert result == {"effort": "high", "summary": "detailed"} finally: litellm.reasoning_auto_summary = original @@ -1059,7 +1065,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result == {"effort": "medium", "summary": "detailed"} finally: diff --git a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index 3432c9503de..7128c04a056 100644 --- a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -162,7 +162,8 @@ class TestAdapterAdaptiveThinking: adapter = LiteLLMAnthropicMessagesAdapter() result = adapter.translate_anthropic_thinking_to_reasoning_effort( - {"type": "adaptive"} + {"type": "adaptive"}, + model="openai/gpt-6.1-sol", ) assert result == "medium" @@ -198,6 +199,7 @@ class TestAdapterAdaptiveThinking: result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( thinking={"type": "adaptive"}, output_config={"effort": "xhigh"}, + model="openai/gpt-6.1-sol", ) assert result is not None assert result["effort"] == "xhigh" @@ -210,6 +212,7 @@ class TestAdapterAdaptiveThinking: result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( thinking={"type": "adaptive"}, + model="openai/gpt-6.1-sol", ) assert result is not None assert result["effort"] == "medium" diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 3ec72f29d34..c1f28e33969 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -11,6 +11,7 @@ Verifies that: """ import json +import logging import os import sys import threading @@ -4530,16 +4531,25 @@ def test_shared_legacy_thinking_translation_rejects_a_non_numeric_budget(budget_ @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize("budget_tokens", _NON_NUMERIC_BUDGETS) -def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_drop_params(budget_tokens: object) -> None: +def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_drop_params( + budget_tokens: object, caplog: pytest.LogCaptureFixture +) -> None: from litellm.llms.anthropic.common_utils import AnthropicModelInfo optional_params: Final = {"thinking": {"type": "enabled", "budget_tokens": budget_tokens}} - AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( - model="claude-opus-5", optional_params=optional_params, custom_llm_provider="azure_ai", drop_params=True - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model="claude-opus-5", optional_params=optional_params, custom_llm_provider="azure_ai", drop_params=True + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert optional_params == {"thinking": {"type": "adaptive"}} + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (f"thinking.budget_tokens={budget_tokens!r}", "claude-opus-5", "(drop_params=True)") + ) @pytest.mark.parametrize( diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index 4d0ebef4f2d..d95f831df3e 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Mapping from datetime import datetime from types import SimpleNamespace @@ -597,6 +598,45 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( assert unexpected.call_count == 0 +@pytest.mark.usefixtures("local_model_cost_map") +def test_native_baseline_body_drops_a_non_numeric_budget_without_logging_a_drop( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + request: Final = { + "model": "claude-opus-5", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + "thinking": {"type": "enabled", "budget_tokens": "high"}, + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + body: Final = prepare_native_baseline_body(request, "anthropic/claude-opus-5") + + assert body is not None + assert body["thinking"] == {"type": "adaptive"} + assert [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] == [] + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_native_baseline_body_drops_a_non_string_reasoning_effort_without_logging_a_drop( + caplog: pytest.LogCaptureFixture, +) -> None: + request: Final = { + "model": "claude-opus-5", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": 5, + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + body: Final = prepare_native_baseline_body(request, "anthropic/claude-opus-5") + + assert body is not None + assert "reasoning_effort" not in body + assert [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] == [] + + def test_native_baseline_body_matches_the_sent_body_when_drop_params_meets_an_auto_added_beta( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 145d45317c3..9f68a4fb73f 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,4 +1,5 @@ import json +import logging import traceback from collections.abc import Callable from types import MappingProxyType @@ -758,6 +759,66 @@ def test_drop_tool_level_extra_fields_strips_copilot_mcp_server_name(): assert result["tools"][1]["function"]["name"] == "read_file" +def test_extra_parameters_error_drops_reasoning_effort_with_one_warning(caplog: pytest.LogCaptureFixture) -> None: + import httpx + + error_text: Final = json.dumps( + { + "error": { + "message": "Extra parameters ['reasoning_effort'] are not allowed when extra-parameters is not set " + "or set to be 'error'. Set extra-parameters to 'pass-through' to pass to the model." + } + } + ) + error: Final = httpx.HTTPStatusError( + message="422", + request=httpx.Request("POST", "https://example.services.ai.azure.com/models/chat/completions"), + response=httpx.Response(422, text=error_text), + ) + request_data: Final = { + "model": "Kimi-K2.6", + "messages": [{"role": "user", "content": "Say hi."}], + "reasoning_effort": "high", + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + retried: Final = AzureAIStudioConfig().transform_request_on_unprocessable_entity_error(error, request_data) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert retried == {"model": "Kimi-K2.6", "messages": [{"role": "user", "content": "Say hi."}]} + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='high'", "Kimi-K2.6")) + assert "drop_params" not in drop_warnings[0] + + +def test_extra_inputs_error_names_drop_params_in_the_reasoning_drop_warning(caplog: pytest.LogCaptureFixture) -> None: + import httpx + + detail: Final = [ + {"type": "extra_forbidden", "loc": ["body", "reasoning_effort"], "msg": "Extra inputs are not permitted"} + ] + error: Final = httpx.HTTPStatusError( + message="422", + request=httpx.Request("POST", "https://example.services.ai.azure.com/models/chat/completions"), + response=httpx.Response(422, text=json.dumps({"error": {"message": json.dumps({"detail": detail})}})), + ) + request_data: Final = { + "model": "Mistral-Large-3", + "messages": [{"role": "user", "content": "Say hi."}], + "reasoning_effort": "high", + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + retried: Final = AzureAIStudioConfig().transform_request_on_unprocessable_entity_error(error, request_data) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert retried == {"model": "Mistral-Large-3", "messages": [{"role": "user", "content": "Say hi."}]} + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] for fragment in ("reasoning_effort='high'", "Mistral-Large-3", "(drop_params=True)") + ) + + def _find_key_anywhere(obj, key: str) -> bool: if isinstance(obj, dict): if key in obj: diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 2b84a606492..1298bdb586e 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -1,6 +1,7 @@ """Bedrock Runtime Chat Completions: the default for GPT 5.6 and newer, ``bedrock/chat_completions/`` for the rest.""" import json +import logging import re from typing import Final @@ -707,18 +708,30 @@ def test_with_max_completion_tokens_leaves_other_params_alone(): @pytest.mark.parametrize( - "model", - ["us.xai.grok-4.6", "bedrock/us-gov-west-1/us.xai.grok-4.6"], + ("model", "logged_model"), + [ + ("us.xai.grok-4.6", "us.xai.grok-4.6"), + ("chat_completions/us.xai.grok-4.6", "us.xai.grok-4.6"), + ("bedrock/us-gov-west-1/us.xai.grok-4.6", "us-gov-west-1/us.xai.grok-4.6"), + ], ) -def test_map_openai_params_drops_reasoning_effort_none_for_grok(model): +def test_map_openai_params_drops_reasoning_effort_none_for_grok_with_one_warning( + model: str, logged_model: str, caplog: pytest.LogCaptureFixture +) -> None: cfg = AmazonBedrockRuntimeChatCompletionsConfig() - mapped = cfg.map_openai_params( - non_default_params={"reasoning_effort": "none", "max_tokens": 64}, - optional_params={}, - model=model, - drop_params=False, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped = cfg.map_openai_params( + non_default_params={"reasoning_effort": "none", "max_tokens": 64}, + optional_params={}, + model=model, + drop_params=False, + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning_effort" not in mapped + assert len(drop_warnings) == 1 + assert "reasoning_effort='none'" in drop_warnings[0] + assert f"model={logged_model!r}" in drop_warnings[0] + assert "drop_params" not in drop_warnings[0] def test_map_openai_params_keeps_reasoning_effort_low_for_grok(): @@ -751,17 +764,24 @@ def test_map_openai_params_refuses_a_non_string_reasoning_effort_without_drop_pa @pytest.mark.parametrize("reasoning_effort", [["low"], {"effort": "low"}, 5], ids=["list", "object", "int"]) @pytest.mark.parametrize("drop_params_via", ["request", "litellm.drop_params"]) def test_map_openai_params_drops_a_non_string_reasoning_effort_under_drop_params( - monkeypatch, model, reasoning_effort, drop_params_via + monkeypatch, caplog, model, reasoning_effort, drop_params_via ): monkeypatch.setattr(litellm, "drop_params", drop_params_via == "litellm.drop_params") - mapped = AmazonBedrockRuntimeChatCompletionsConfig().map_openai_params( - non_default_params={"reasoning_effort": reasoning_effort, "max_tokens": 64}, - optional_params={}, - model=model, - drop_params=drop_params_via == "request", - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped = AmazonBedrockRuntimeChatCompletionsConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort, "max_tokens": 64}, + optional_params={}, + model=model, + drop_params=drop_params_via == "request", + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning_effort" not in mapped assert mapped["max_completion_tokens"] == 64 + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (f"reasoning_effort={reasoning_effort!r}", repr(model), "(drop_params=True)") + ) def test_map_openai_params_keeps_reasoning_effort_none_for_gpt56(): diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index ae97820927b..90fa36b1b48 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -13,6 +13,7 @@ import pytest import respx import litellm +from litellm._logging import verbose_logger from litellm import ModelResponse, Router, completion from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, @@ -1221,6 +1222,97 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info assert properties == _LOOKAROUND_FREE_PROPERTIES +@pytest.mark.parametrize( + ("model", "litellm_params"), + [ + ("us.newvendor.model-1", None), + ("converse/us.newvendor.model-1", None), + ("us-east-1/us.newvendor.model-1", None), + ("us.newvendor.model-1", {"base_model": "bedrock/converse/us.newvendor.model-1"}), + ], +) +def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model, litellm_params): + assert "us.newvendor.model-1" not in litellm.model_cost + + properties: Final = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL, litellm_params) + + assert properties == _LOOKAROUND_FREE_PROPERTIES + + +@pytest.mark.parametrize( + "model", + [ + "arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123", + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", + "us.anthropic.claude-unreleased-v1:0", + "eu.openai.gpt-unreleased", + "apac.newvendor.listed-1", + ], +) +def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_litellm_still_recognizes(model, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "us.newvendor.listed-1", {"litellm_provider": "bedrock_converse"}) + + properties = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL) + + assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"] + + +def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_deployment_opts_out(): + properties = _tool_schema_properties( + "us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}} + ) + + assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"] + + +@pytest.mark.parametrize( + "deployment_params", + [ + {"model": "bedrock/us.newvendor.model-1"}, + {"model": "bedrock/converse/us.newvendor.model-1", "allowed_openai_params": ["tools"]}, + ], +) +def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex( + respx_mock, deployment_params +): + route: Final = respx_mock.post( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.newvendor.model-1/converse" + ).mock( + return_value=httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ) + ) + router: Final = Router( + model_list=[ + { + "model_name": "unlisted", + "litellm_params": { + **deployment_params, + "aws_access_key_id": "fake-access-key", + "aws_secret_access_key": "fake-secret-key", + "aws_region_name": "us-east-1", + }, + "model_info": {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + } + ] + ) + + router.completion( + model="unlisted", + messages=[{"role": "user", "content": "hi"}], + tools=[_ARTIFACT_DATA_OPENAI_TOOL], + client=HTTPHandler(), + ) + + tool: Final = json.loads(route.calls[0].request.content)["toolConfig"]["tools"][0] + assert tool["toolSpec"]["inputSchema"]["json"]["properties"] == _LOOKAROUND_FREE_PROPERTIES + + def test_transform_request_keeps_lookaround_regex_when_the_deployment_model_info_opts_out(): properties = _tool_schema_properties( "global.moonshotai.kimi-k3", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}} @@ -9189,18 +9281,223 @@ def test_bedrock_tools_transformation_valid_params(): assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"] -def test_not_found_error(): +def _invalid_model_identifier_reply(sent: list[httpx.Request]): + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response( + 400, + json={"message": "The provided model identifier is invalid."}, + headers={"x-amzn-ErrorType": "ValidationException", "x-amzn-RequestId": "req-123"}, + ) + + return respond + + +@pytest.mark.parametrize( + "model, path", + [ + ("bedrock/bad_model", b"/model/bad_model/converse"), + ("bedrock/us-east-1/bad_model", b"/model/bad_model/converse"), + ("bedrock/newvendor.listed-v1:0", b"/model/newvendor.listed-v1%3A0/converse"), + ("bedrock/us-east-1/newvendor.listed-v1:0", b"/model/newvendor.listed-v1%3A0/converse"), + ], +) +def test_not_found_error(model, path, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "newvendor.listed-v1:0", {"litellm_provider": "bedrock", "mode": "chat"}) + sent: Final[list[httpx.Request]] = [] + + with pytest.raises(litellm.NotFoundError) as exc_info: + completion( + model=model, + messages=[{"role": "user", "content": "What is the meaning of life"}], + client=HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) + ), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert exc_info.value.status_code == 404 + assert exc_info.value.response.status_code == 404 + assert exc_info.value.response.headers["x-amzn-requestid"] == "req-123" + assert [request.url.raw_path for request in sent] == [path] + + +@pytest.mark.parametrize( + "model", ["bedrock/", "bedrock/us-east-1/", "bedrock/../../converse", "bedrock/xx-fake-1/us.xai.grok-4.7"] +) +def test_a_model_id_that_is_empty_or_a_path_fails_locally_without_calling_aws(model): + sent: Final[list[httpx.Request]] = [] + with pytest.raises(litellm.NotFoundError): completion( - model="bedrock/bad_model", - messages=[ - { - "role": "user", - "content": "What is the meaning of life", - } - ], + model=model, + messages=[{"role": "user", "content": "What is the meaning of life"}], + client=HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) + ), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", ) + assert sent == [] + + +@pytest.mark.parametrize( + "model, path", + [ + ("bedrock/converse/bad_model", b"/model/bad_model/converse"), + ("bedrock/anthropic.claude-typo-v1:0", b"/model/anthropic.claude-typo-v1:0/invoke"), + ], +) +def test_invalid_model_identifier_stays_a_bad_request_for_a_model_litellm_recognizes(model, path): + sent: Final[list[httpx.Request]] = [] + + with pytest.raises(litellm.BadRequestError) as exc_info: + completion( + model=model, + messages=[{"role": "user", "content": "What is the meaning of life"}], + client=HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) + ), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert exc_info.value.status_code == 400 + assert [request.url.raw_path for request in sent] == [path] + + +_WEATHER_TOOL: Final = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} + + +def _converse_tool_call_reply(sent: list[httpx.Request]): + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response( + 200, + json={ + "output": { + "message": { + "role": "assistant", + "content": [ + {"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}} + ], + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ) + + return respond + + +@pytest.mark.parametrize("model", ["bedrock/us.newvendor.model-1", "bedrock/us-east-1/us.newvendor.model-1"]) +def test_model_missing_from_cost_map_sends_tools_to_converse(model): + assert "us.newvendor.model-1" not in litellm.model_cost + sent: Final[list[httpx.Request]] = [] + + response: Final = completion( + model=model, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + tool_choice="auto", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + assert [request.url.raw_path for request in sent] == [b"/model/us.newvendor.model-1/converse"] + tool_config: Final = json.loads(sent[0].content)["toolConfig"] + assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"] + assert tool_config["toolChoice"] == {"auto": {}} + + +def test_a_price_only_row_for_an_unlisted_model_still_sends_tools_to_converse(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, "us.newvendor.priced-1", {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} + ) + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/us.newvendor.priced-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + tool_config: Final = json.loads(sent[0].content)["toolConfig"] + assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"] + + +def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params(): + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/converse/us.newvendor.model-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + drop_params=True, + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert "toolConfig" not in json.loads(sent[0].content) + + +def test_unknown_model_gets_reasoning_effort_as_sent(): + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/us.newvendor.model-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + reasoning_effort="high", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert json.loads(sent[0].content)["additionalModelRequestFields"] == {"reasoning_effort": "high"} + + +def test_mapped_model_without_function_calling_still_rejects_tools(): + model: Final = "bedrock/converse/meta.llama3-8b-instruct-v1:0" + assert "meta.llama3-8b-instruct-v1:0" in litellm.model_cost + assert not litellm.supports_function_calling(model=model) + sent: Final[list[httpx.Request]] = [] + + with pytest.raises(litellm.UnsupportedParamsError): + completion( + model=model, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert sent == [] + @pytest.mark.parametrize( "model, expected_base_model", @@ -10865,6 +11162,52 @@ def test_converse_default_and_auto_service_tier_always_forwarded( assert result["serviceTier"] == {"type": "default"} +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +_DEEPSEEK_R1_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.deepseek.r1-v1:0" + + +@pytest.mark.parametrize( + "param, value", (("thinking", {"type": "enabled", "budget_tokens": 1024}), ("reasoning_effort", "high")) +) +def test_bedrock_deepseek_arn_reasoning_drop_warns_once( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + optional_params: Final = AmazonConverseConfig().map_openai_params( + non_default_params={param: value, "max_tokens": 100}, + optional_params={}, + model=_DEEPSEEK_R1_PROFILE_ARN, + drop_params=False, + ) + + warnings: Final = _warnings_naming(caplog, param) + assert param not in optional_params + assert len(warnings) == 1 + assert _DEEPSEEK_R1_PROFILE_ARN in warnings[0] + + +@pytest.mark.parametrize( + "param, value", (("thinking", {"type": "enabled", "budget_tokens": 1024}), ("reasoning_effort", "high")) +) +def test_bedrock_deepseek_r1_drop_of_an_unadvertised_param_does_not_warn( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + AmazonConverseConfig().map_openai_params( + non_default_params={param: value, "max_tokens": 100}, + optional_params={}, + model="us.deepseek.r1-v1:0", + drop_params=False, + ) + + assert _warnings_naming(caplog, param) == () + + def _converse_bound_thinking_request( headers: dict[str, str] | None, body_betas: tuple[str, ...] = (), drop_params: bool = True ) -> dict[str, object]: diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index bd1ef9cba00..496af9fb2d4 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -212,24 +212,17 @@ def test_system_defined_inference_profile_arn_still_routes_to_converse(): assert route == "converse" -def test_other_opaque_arn_types_still_route_to_invoke(): - """ - Only application-inference-profile ARNs are auto-routed to converse. Other - opaque ARNs (provisioned-model, imported-model, custom-model-deployment) - also yield no invoke provider, but they are frequently invoke-only with - provider-specific body formats, so routing them to converse could break - them. Guards the deliberate scope against an over-broad "any opaque ARN -> - converse" generalization. - """ - for arn_segment in ( - "provisioned-model/abcdefgh1234", - "imported-model/abcdefgh1234", - "custom-model-deployment/abcdefgh1234", - ): - route = BedrockModelInfo.get_bedrock_route( - model=f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" - ) - assert route == "invoke", f"{arn_segment} should stay on invoke route" +@pytest.mark.parametrize( + "arn_segment", + ["provisioned-model/abcdefgh1234", "imported-model/abcdefgh1234", "custom-model-deployment/abcdefgh1234"], +) +def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: str): + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" + assert BaseAWSLLM.get_bedrock_invoke_provider(model) is None + assert BedrockModelInfo.get_bedrock_route(model) == "converse" + assert BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") == "invoke" def test_govcloud_cross_region_inference_prefix(): @@ -1041,6 +1034,84 @@ def test_unmapped_openai_family_model_routes_to_converse(): assert BedrockModelInfo.get_bedrock_route(imported) == "openai" +@pytest.mark.parametrize( + "unmapped", + [ + "bedrock/minimax.minimax-m99-unmapped", + "bedrock/us.nvidia.nemotron-99b-unmapped", + "bedrock/writer.palmyra-x99-unmapped", + "bedrock/zai.glm-99-unmapped", + "bedrock/google.gemma-99-unmapped", + "bedrock/global.xai.grok-99-unmapped", + ], +) +def test_unmapped_converse_only_family_model_routes_to_converse(unmapped: str): + import litellm + + assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models + assert BedrockModelInfo.get_bedrock_route(unmapped) == "converse" + + +def test_unmapped_model_of_family_with_invoke_handler_still_routes_to_invoke(): + from typing import Final + + import litellm + + unmapped: Final = "bedrock/meta.llama99-unmapped-v1:0" + assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models + assert BedrockModelInfo.get_bedrock_route(unmapped) == "invoke" + + +@pytest.mark.parametrize( + ("model", "model_path", "response_json"), + [ + ( + "bedrock/us.newvendor.model-1", + b"/model/us.newvendor.model-1/converse", + { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ), + ( + "bedrock/mistral.mistral-99-unmapped-v1:0", + b"/model/mistral.mistral-99-unmapped-v1:0/invoke", + {"outputs": [{"text": "ok", "stop_reason": "stop"}]}, + ), + ], +) +def test_completion_on_unmapped_model_posts_converse_unless_its_id_names_an_invoke_provider( + model: str, model_path: bytes, response_json: dict[str, object] +): + import httpx + + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + assert BedrockModelInfo.get_base_model(model) not in litellm.bedrock_converse_models + sent: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response( + 200, + json=response_json, + headers={"x-amzn-bedrock-input-token-count": "1", "x-amzn-bedrock-output-token-count": "1"}, + ) + + response: Final = litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert response.choices[0].message.content == "ok" + assert [request.url.raw_path for request in sent] == [model_path] + + @pytest.mark.parametrize( ("model", "expected"), [ diff --git a/tests/unit/llms/bedrock/test_nova_imported_models.py b/tests/unit/llms/bedrock/test_nova_imported_models.py index e3677aaf9e6..231f3097dcd 100644 --- a/tests/unit/llms/bedrock/test_nova_imported_models.py +++ b/tests/unit/llms/bedrock/test_nova_imported_models.py @@ -24,10 +24,8 @@ class TestGetBedrockRoute: def test_nova2_prefix_routes_to_converse(self): assert BedrockModelInfo.get_bedrock_route(NOVA2_MODEL) == "converse" - def test_plain_arn_routes_to_invoke(self): - # Without spec prefix, ARN doesn't match converse models - result = BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}") - assert result == "invoke" + def test_plain_arn_without_an_invoke_provider_routes_to_converse(self): + assert BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}") == "converse" class TestGetBedrockBaseModel: diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 794ac2d26d5..23d7feb2557 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -14,6 +14,7 @@ import httpcore import httpx import pytest import respx +import websockets.exceptions from botocore.credentials import RefreshableCredentials from h2.events import ConnectionTerminated @@ -2787,6 +2788,194 @@ async def test_async_realtime_error_event_send_failure_still_closes(): assert client_ws.events == [("close", (1011, "Internal server error: vertex token refresh exploded"))] +_GEMINI_LIVE_SETUP_COMPLETE = json.dumps({"setupComplete": {}}) +_GEMINI_LIVE_TEXT_TURN = json.dumps({"serverContent": {"modelTurn": {"parts": [{"text": "Blue."}]}}}) +_GEMINI_LIVE_TURN_COMPLETE = json.dumps({"serverContent": {"turnComplete": True}}) + + +class _ScriptedRealtimeBackend: + """Backend socket that answers the first setup frame with the scripted replies + and closes once the client is done, so both bridge loops run to completion.""" + + def __init__(self, replies_after_setup): + self.sent = [] + self.closed = asyncio.Event() + self._replies_after_setup = list(replies_after_setup) + self._inbox = asyncio.Queue() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return None + + async def send(self, payload): + self.sent.append(payload) + if "setup" in json.loads(payload): + for reply in self._replies_after_setup: + self._inbox.put_nowait(reply) + + def close_after_replies(self): + self._inbox.put_nowait(None) + + async def recv(self, decode=True): + reply = await self._inbox.get() + if reply is None: + self.closed.set() + raise websockets.exceptions.ConnectionClosedOK(None, None) + return reply + + +class _ScriptedRealtimeClient: + def __init__(self, messages, backend): + self.scope = {"headers": []} + self.exceptions = websockets.exceptions + self.sent = [] + self._messages = list(messages) + self._backend = backend + + async def send_text(self, payload): + self.sent.append(payload) + + async def receive_text(self): + if self._messages: + return self._messages.pop(0) + self._backend.close_after_replies() + await self._backend.closed.wait() + raise websockets.exceptions.ConnectionClosedOK(None, None) + + async def close(self, code=None, reason=None): + return None + + +def _realtime_logging_obj(): + logging_obj = Mock() + logging_obj.litellm_trace_id = "trace-lit-6974" + logging_obj.async_success_handler = AsyncMock() + logging_obj.dispatch_failure_handlers = AsyncMock() + return logging_obj + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model, session, backend_modality, client_modality, backend_turn, drop_params", + [ + ( + "gemini-live-2.5-flash", + {"type": "realtime", "output_modalities": ["text"]}, + "TEXT", + "text", + [_GEMINI_LIVE_TEXT_TURN], + False, + ), + ("gemini-live-2.5-flash", {"modalities": ["text"]}, "TEXT", "text", [_GEMINI_LIVE_TEXT_TURN], False), + ( + "gemini-live-2.5-flash-native-audio", + {"type": "realtime", "output_modalities": ["text"]}, + "AUDIO", + "audio", + [], + True, + ), + ], +) +async def test_async_realtime_vertex_first_session_update_builds_the_only_setup( + model, session, backend_modality, client_modality, backend_turn, drop_params: bool, monkeypatch +): + """Regression for text-only Vertex Live sessions answered in audio: the + connect-time default setup (AUDIO) used to go out before the client's + session.update, and Gemini accepts only one setup per connection, so the + requested modality was dropped and billed at the audio rate. The setup now + goes out on the client's first message, built from session.update when that + comes first. Under drop_params an audio-only model still gets AUDIO, and the + client learns the downgrade from session.updated.""" + from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig + + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + backend = _ScriptedRealtimeBackend([_GEMINI_LIVE_SETUP_COMPLETE, *backend_turn, _GEMINI_LIVE_TURN_COMPLETE]) + client_ws = _ScriptedRealtimeClient( + [ + json.dumps({"type": "session.update", "session": session}), + json.dumps( + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "What color is the sky? One word."}], + }, + } + ), + json.dumps({"type": "response.create"}), + ], + backend, + ) + handler = BaseLLMHTTPHandler() + + with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=backend)): + await asyncio.wait_for( + handler.async_realtime( + model=model, + websocket=client_ws, + logging_obj=_realtime_logging_obj(), + provider_config=VertexAIRealtimeConfig( + access_token="fake-token", project="fake-project", location="us-central1", drop_params=drop_params + ), + headers={}, + ), + timeout=10, + ) + + backend_frames = [json.loads(frame) for frame in backend.sent] + setups = [frame["setup"] for frame in backend_frames if "setup" in frame] + assert len(setups) == 1 + assert "setup" in backend_frames[0] + assert setups[0]["generationConfig"]["responseModalities"] == [backend_modality] + assert any("clientContent" in frame for frame in backend_frames[1:]) + + client_events = [json.loads(frame) for frame in client_ws.sent] + event_types = [event["type"] for event in client_events] + assert event_types[:2] == ["session.created", "session.updated"] + assert client_events[1]["session"]["modalities"] == [client_modality] + if backend_turn: + assert "response.output_text.delta" in event_types + assert "response.output_audio_transcript.delta" not in event_types + + +@pytest.mark.asyncio +async def test_async_realtime_sends_setup_at_connect_when_provider_cannot_build_it_from_session_update(): + """Bedrock Nova Sonic style providers keep the connect-time setup and get no + synthetic session.created.""" + setup = json.dumps({"event": {"sessionStart": {}}}) + backend = _ScriptedRealtimeBackend([]) + client_ws = _ScriptedRealtimeClient([], backend) + provider_config = Mock() + provider_config.get_complete_url.return_value = "wss://backend.example/live" + provider_config.validate_environment.return_value = {} + provider_config.requires_session_configuration.return_value = True + provider_config.builds_setup_from_session_update.return_value = False + provider_config.session_configuration_request.return_value = setup + provider_config.unbilled_usage_on_session_close.return_value = None + provider_config.open_backend = AsyncMock(return_value=None) + provider_config.wrap_backend.side_effect = lambda opened: opened + handler = BaseLLMHTTPHandler() + + with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=backend)): + await asyncio.wait_for( + handler.async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=client_ws, + logging_obj=_realtime_logging_obj(), + provider_config=provider_config, + headers={}, + ), + timeout=10, + ) + + assert backend.sent == [setup] + assert client_ws.sent == [] + + class _JSONBodyAudioTranscriptionConfig(BaseAudioTranscriptionConfig): def get_supported_openai_params(self, model): return [] diff --git a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py index 3fbb12881da..56af47aeebe 100644 --- a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Iterator from typing import Final @@ -6,6 +7,7 @@ import httpx import litellm import pytest import respx +from litellm._logging import verbose_logger from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig @@ -951,3 +953,42 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "https://gateway.example.com/deepseek/chat/completions" ) + + +def _budget_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and "Dropping thinking.budget_tokens=" in r.getMessage() + ) + + +def test_enabled_thinking_budget_dropped_for_deepseek_warns_once(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = DeepSeekChatConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 4000}}, + optional_params={}, + model="deepseek-reasoner", + drop_params=False, + ) + + warnings: Final = _budget_warnings(caplog) + assert result["thinking"] == {"type": "enabled"} + assert len(warnings) == 1 + assert "budget_tokens=4000" in warnings[0] + assert "deepseek-reasoner" in warnings[0] + + +@pytest.mark.parametrize( + "thinking", ({"type": "enabled"}, {"type": "disabled", "budget_tokens": 4000}), ids=("no-budget", "disabled") +) +def test_deepseek_thinking_without_a_requested_budget_does_not_warn( + thinking: dict[str, object], caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = DeepSeekChatConfig().map_openai_params( + non_default_params={"thinking": thinking}, optional_params={}, model="deepseek-reasoner", drop_params=False + ) + + assert result["thinking"] == {"type": thinking["type"]} + assert _budget_warnings(caplog) == () diff --git a/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py b/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py index b7a79f03f8d..f4a7455ff27 100644 --- a/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py +++ b/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py @@ -1,3 +1,6 @@ +import logging +from typing import Final + import httpx import pytest @@ -242,11 +245,33 @@ def test_map_openai_params_maps_reasoning_effort(effort, expected): assert mapped["reasoning"] is expected -def test_map_openai_params_drops_unknown_reasoning_effort_when_dropping(): - mapped = FalAIChatConfig().map_openai_params( - non_default_params={"reasoning_effort": "extreme"}, optional_params={}, model=MODEL, drop_params=True - ) +@pytest.mark.parametrize("drop_globally", [False, True]) +def test_map_openai_params_drops_unknown_reasoning_effort_with_one_warning( + drop_globally: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_globally) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model=MODEL, + drop_params=not drop_globally, + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning" not in mapped + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='xhigh'", MODEL, "(drop_params=True)")) + + +def test_map_openai_params_rejects_unknown_reasoning_effort_without_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(FalAIError) as exc_info: + FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, optional_params={}, model=MODEL, drop_params=False + ) + assert exc_info.value.status_code == 400 @pytest.mark.parametrize("effort", [{"level": "low"}, ["low"], 1]) diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 28f4fdead9e..ed166d206c8 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -9,6 +10,7 @@ import respx from pydantic import ValidationError import litellm +from litellm._logging import verbose_logger from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -2361,6 +2363,39 @@ def test_global_disable_flag_with_transform_messages_helper( assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": content}] +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +def test_chat_template_kwargs_effort_dropped_for_non_reasoning_model_warns_once( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAIConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"reasoning_budget": 512}}}, + _NON_REASONING_MODEL, + ) + + warnings: Final = _warnings_naming(caplog, "chat_template_kwargs") + assert len(warnings) == 1 + assert "'reasoning_budget': 512" in warnings[0] + assert "llama-v3-70b-instruct" in warnings[0] + + +def test_chat_template_kwargs_thinking_off_for_non_reasoning_model_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAIConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": False}}}, + _NON_REASONING_MODEL, + ) + + assert _warnings_naming(caplog, "chat_template_kwargs") == () + + _CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN: Final = ( {"role": "system", "content": "You are Codex"}, {"role": "user", "content": "Hi there"}, diff --git a/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py b/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py index e5a77aa8d41..d35442d4095 100644 --- a/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py +++ b/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py @@ -1,7 +1,11 @@ +import logging +from typing import Final + import pytest import litellm +from litellm._logging import verbose_logger from litellm.llms.fireworks_ai.completion.transformation import ( @@ -204,3 +208,36 @@ def test_transform_text_completion_request_keeps_sdk_rejected_keys_in_extra_body assert "chat_template_kwargs" not in data["extra_body"] assert "best_of" not in data["extra_body"] assert "response_format" not in data + + +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +def test_chat_template_kwargs_effort_dropped_for_non_reasoning_model_warns_once( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAITextCompletionConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"reasoning_budget": 512}}}, + _NON_REASONING_MODEL, + ) + + warnings: Final = _warnings_naming(caplog, "chat_template_kwargs") + assert len(warnings) == 1 + assert "'reasoning_budget': 512" in warnings[0] + assert "llama-v3-70b-instruct" in warnings[0] + + +def test_chat_template_kwargs_thinking_off_for_non_reasoning_model_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAITextCompletionConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": False}}}, + _NON_REASONING_MODEL, + ) + + assert _warnings_naming(caplog, "chat_template_kwargs") == () diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index 017f279d6c4..460a50ed61a 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -1,3 +1,4 @@ +import base64 import json from collections.abc import Mapping from typing import Final, cast @@ -7,7 +8,7 @@ import pytest import litellm -from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig +from litellm.llms.gemini.realtime.transformation import GEMINI_LIVE_DOCUMENTED_VOICES, GeminiRealtimeConfig from litellm.types.llms.gemini import BidiGenerateContentServerMessage @@ -439,6 +440,51 @@ def test_gemini_realtime_session_update_with_tools(): assert "parameters" in function_decl +def _schema_nodes(schema: object): + if isinstance(schema, dict): + yield schema + for value in schema.values(): + yield from _schema_nodes(value) + elif isinstance(schema, list): + for value in schema: + yield from _schema_nodes(value) + + +def test_gemini_session_update_converts_flat_realtime_tool_schemas_to_gemini_schemas(): + session_update: Final = { + "type": "session.update", + "session": { + "tools": [ + { + "type": "function", + "name": "lookup_weather", + "description": "Look up the weather in a city.", + "parameters": { + "type": "object", + "$defs": { + "City": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]} + }, + "properties": {"city": {"$ref": "#/$defs/City"}, "note": {"type": ["string", "null"]}}, + "required": ["city"], + }, + } + ] + }, + } + + messages: Final = GeminiRealtimeConfig().transform_realtime_request( + json.dumps(session_update), "gemini-2.5-flash", session_configuration_request=None + ) + + declaration: Final = json.loads(messages[0])["setup"]["tools"][0]["function_declarations"][0] + assert declaration["name"] == "lookup_weather" + parameters: Final = declaration["parameters"] + assert parameters["properties"]["city"]["properties"]["name"]["type"] == "string" + nodes: Final = tuple(_schema_nodes(parameters)) + assert not any("$defs" in node or "$ref" in node for node in nodes) + assert all(isinstance(node["type"], str) for node in nodes if "type" in node) + + def test_gemini_session_update_defaults_to_audio_modality(): config = GeminiRealtimeConfig() @@ -613,6 +659,35 @@ def test_gemini_audio_only_live_models_drop_text_from_text_audio_combo(patch_gem assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] +@pytest.mark.parametrize("output_modalities", ["text", ["text"]]) +def test_gemini_session_update_accepts_a_single_string_output_modality(output_modalities): + """Regression: a bare string was iterated per character, so output_modalities + "text" reached Vertex as responseModalities ["T"], which it rejects with 1007.""" + config = GeminiRealtimeConfig() + session_update = {"type": "session.update", "session": {"type": "realtime", "output_modalities": output_modalities}} + + messages = config.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + + +@pytest.mark.parametrize( + ("output_modalities", "expected"), + [(["video", "text"], ["TEXT"]), (["video"], ["AUDIO"]), (None, ["AUDIO"])], +) +def test_gemini_session_update_drops_modalities_live_cannot_produce(output_modalities, expected): + config = GeminiRealtimeConfig() + session_update = {"type": "session.update", "session": {"type": "realtime", "output_modalities": output_modalities}} + + messages = config.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == expected + + def test_gemini_non_live_model_preserves_text_modality(): config = GeminiRealtimeConfig() session_update = { @@ -1396,22 +1471,71 @@ def test_gemini_input_audio_buffer_clear_is_local_noop(): assert messages == [] -def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad(): +_MANUAL_VAD_SETUP = json.dumps({"setup": {"realtimeInputConfig": {"automaticActivityDetection": {"disabled": True}}}}) +_APPEND = json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(b"\x00\x01" * 80).decode()}) + + +def _realtime_input_keys(config: GeminiRealtimeConfig, client_event: str, setup: str | None) -> list[str]: + frames = config.transform_realtime_request(client_event, "gemini-live-2.5-flash-native-audio", setup) + return [key for frame in frames for key in json.loads(frame)["realtimeInput"]] + + +@pytest.mark.parametrize("turn_end", ["input_audio_buffer.commit", "response.create"]) +def test_gemini_manual_vad_wraps_each_audio_turn_in_activity_start_and_end(turn_end): + """With activity detection off, Gemini Live answers only after activityEnd closes an + activityStart, and it closes the socket on any other shape of those markers.""" config = GeminiRealtimeConfig() - setup = { - "setup": { - "realtimeInputConfig": { - "automaticActivityDetection": {"disabled": True}, - } - } - } - messages = config.transform_realtime_request( - json.dumps({"type": "input_audio_buffer.commit"}), - "gemini-live-2.5-flash-native-audio", - session_configuration_request=json.dumps(setup), + end = json.dumps({"type": turn_end}) + + turns = [ + [_realtime_input_keys(config, event, _MANUAL_VAD_SETUP) for event in (_APPEND, _APPEND, end, end)] + for _ in range(2) + ] + + assert turns == [[["activityStart", "audio"], ["audio"], ["activityEnd"], []]] * 2 + + +def test_gemini_manual_vad_activity_markers_are_empty_messages(): + config = GeminiRealtimeConfig() + commit = json.dumps({"type": "input_audio_buffer.commit"}) + + first_append = config.transform_realtime_request(_APPEND, "gemini-live-2.5-flash", _MANUAL_VAD_SETUP) + turn_end = config.transform_realtime_request(commit, "gemini-live-2.5-flash", _MANUAL_VAD_SETUP) + + assert json.loads(first_append[0]) == {"realtimeInput": {"activityStart": {}}} + assert [json.loads(frame) for frame in turn_end] == [{"realtimeInput": {"activityEnd": {}}}] + + +@pytest.mark.parametrize( + ("session", "expected_keys"), + [ + ({"turn_detection": None}, [["activityStart", "audio"], ["activityEnd"]]), + ({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, [["activityStart", "audio"], ["activityEnd"]]), + ({"type": "realtime", "audio": {"input": {}}}, [["audio"], ["audioStreamEnd"]]), + ], +) +def test_gemini_null_turn_detection_makes_the_client_mark_each_turn(session, expected_keys): + """OpenAI's turn_detection null turns server VAD off, so the client commits each turn itself.""" + config = GeminiRealtimeConfig() + setup = config.transform_realtime_request( + json.dumps({"type": "session.update", "session": session}), "gemini-live-2.5-flash-native-audio", None ) - assert len(messages) == 1 - assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": True}} + commit = json.dumps({"type": "input_audio_buffer.commit"}) + + keys = [_realtime_input_keys(config, event, setup[0]) for event in (_APPEND, commit)] + + assert keys == expected_keys + + +def test_gemini_automatic_vad_sends_no_activity_markers(): + config = GeminiRealtimeConfig() + commit = json.dumps({"type": "input_audio_buffer.commit"}) + response_create = json.dumps({"type": "response.create"}) + + keys = [_realtime_input_keys(config, event, None) for event in (_APPEND, commit)] + + assert keys == [["audio"], ["audioStreamEnd"]] + assert config.transform_realtime_request(response_create, "gemini-live-2.5-flash", None) == [] def test_gemini_tool_call_id_to_name_evicts_oldest_when_capped(): @@ -1883,13 +2007,75 @@ def test_map_openai_params_drops_stock_voice_case_insensitively(): """ cfg = GeminiRealtimeConfig() - dropped = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Alloy"}) + dropped = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Alloy"}, model="gemini-3.8-live") assert "speechConfig" not in dropped.get("generationConfig", {}) - passthrough = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Kore"}) + passthrough = cfg.map_openai_params( + optional_params={}, non_default_params={"voice": "Kore"}, model="gemini-3.8-live" + ) assert passthrough["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" +_DOCUMENTED_VOICE: Final = sorted(GEMINI_LIVE_DOCUMENTED_VOICES)[0].capitalize() + + +@pytest.mark.parametrize("drop_params", [False, True]) +def test_google_ai_studio_forwards_a_documented_voice(drop_params): + config: Final = GeminiRealtimeConfig(drop_params=drop_params) + mapped: Final = config.map_openai_params( + optional_params={}, non_default_params={"voice": _DOCUMENTED_VOICE}, model="gemini-3.8-live" + ) + + speech_config: Final = mapped["generationConfig"]["speechConfig"] + assert speech_config["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == _DOCUMENTED_VOICE + assert config.setup_session_update_rejection("gemini-3.8-live", {"voice": _DOCUMENTED_VOICE}, setup_sent=False) is None + + +@pytest.mark.parametrize( + ("session", "param"), + [ + ({"voice": "Nova"}, "session.voice"), + ({"type": "realtime", "audio": {"output": {"voice": "Nova"}}}, "session.audio.output.voice"), + ], +) +def test_google_ai_studio_rejects_an_undocumented_voice_on_the_session_update(session, param): + rejection: Final = GeminiRealtimeConfig().setup_session_update_rejection( + "gemini-2.5-flash-native-audio-preview-12-2025", session, setup_sent=False + ) + + assert rejection is not None + assert (rejection.code, rejection.param) == ("invalid_value", param) + + +def _setup_speech_config(config: GeminiRealtimeConfig, model: str, session: dict[str, object]) -> object: + messages: Final = config.transform_realtime_request( + json.dumps({"type": "session.update", "session": session}), model, session_configuration_request=None + ) + return json.loads(messages[0])["setup"].get("generationConfig", {}).get("speechConfig") + + +@pytest.mark.parametrize( + "session", + [{"voice": "Nova"}, {"type": "realtime", "audio": {"output": {"voice": "Nova"}}}], +) +def test_google_ai_studio_drops_an_undocumented_voice_with_drop_params(session): + config: Final = GeminiRealtimeConfig(drop_params=True) + model: Final = "gemini-2.5-flash-native-audio-preview-12-2025" + + assert config.setup_session_update_rejection(model, session, setup_sent=False) is None + assert _setup_speech_config(config, model, session) is None + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("model", ["gemini-3.8-live", "gemini-3.1-flash-live-preview"]) +def test_google_ai_studio_forwards_any_voice_to_models_that_accept_it(model, drop_params): + config: Final = GeminiRealtimeConfig(drop_params=drop_params) + + assert config.setup_session_update_rejection(model, {"voice": "Nova"}, setup_sent=False) is None + speech_config: Final = _setup_speech_config(config, model, {"voice": "Nova"}) + assert speech_config == {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Nova"}}} + + @pytest.fixture(autouse=False) def patch_gemini_transcribe_live_cost_map_entry(monkeypatch): """Inject the gemini-3.5-transcribe-live registry entry locally. @@ -1954,6 +2140,55 @@ def test_gemini_transcribe_live_coerces_audio_modality_to_text(modalities, patch assert setup["generationConfig"]["responseModalities"] == ["TEXT"] +@pytest.mark.parametrize("model", ["gemini-3.5-transcribe-live", "gemini/gemini-3.5-transcribe-live"]) +def test_gemini_transcribe_live_synthetic_session_created_reports_text_output( + model, patch_gemini_transcribe_live_cost_map_entry +): + session = GeminiRealtimeConfig().transform_session_created_event(model, "sess_transcribe")["session"] + + assert session["modalities"] == ["text"] + + +@pytest.mark.parametrize( + ("model", "modalities", "producible"), + [ + ("gemini-3.5-transcribe-live", ["audio"], False), + ("gemini-3.5-transcribe-live", ["audio", "text"], True), + ("gemini-live-2.5-flash-native-audio", ["text"], False), + ("gemini-live-2.5-flash-native-audio", ["text", "audio"], True), + ], +) +@pytest.mark.parametrize("drop_params", [False, True]) +def test_gemini_live_rejects_only_output_modalities_the_model_cannot_produce_at_all( + model, + modalities, + producible, + drop_params, + patch_gemini_transcribe_live_cost_map_entry, + patch_gemini_audio_cost_map_entries, +): + rejection = GeminiRealtimeConfig(drop_params=drop_params).setup_session_update_rejection( + model, {"output_modalities": modalities}, setup_sent=False + ) + + assert (rejection is None) is (drop_params or producible) + assert rejection is None or rejection.param == "session.output_modalities" + + +@pytest.mark.parametrize("drop_params", [False, True]) +def test_gemini_provider_realtime_config_carries_drop_params(drop_params): + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + model = "gemini-live-2.5-flash-native-audio" + config = ProviderConfigManager.get_provider_realtime_config( + model=model, provider=LlmProviders.GEMINI, drop_params=drop_params + ) + + rejection = config.setup_session_update_rejection(model, {"modalities": ["text"]}, setup_sent=False) + assert (rejection is None) is drop_params + + def test_gemini_chat_model_with_text_output_modalities_keeps_audio_eager_setup( patch_gemini_transcribe_live_cost_map_entry, ): @@ -2169,6 +2404,47 @@ def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_tra assert config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") is None +def test_gemini_realtime_two_output_modalities_collapse_to_audio(): + """Gemini Live closes the socket with 1007 when the setup names two response + modalities, so a GA session.update asking for ["text", "audio"] must reach the + backend as AUDIO only, the same choice the beta shim makes.""" + config = GeminiRealtimeConfig() + session_update = { + "type": "session.update", + "session": {"output_modalities": ["text", "audio"], "instructions": "Be brief."}, + } + + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash", + session_configuration_request=None, + ) + + assert len(messages) == 1 + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"] + + +def test_gemini_realtime_inf_max_response_output_tokens_is_not_forwarded(): + """OpenAI's default max_response_output_tokens is the string "inf", which Gemini + rejects with 1007 (TYPE_INT32), so only integer limits map to maxOutputTokens.""" + config = GeminiRealtimeConfig() + + def generation_config_for(limit): + session_update = { + "type": "session.update", + "session": {"modalities": ["text"], "max_response_output_tokens": limit}, + } + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash", + session_configuration_request=None, + ) + return json.loads(messages[0])["setup"]["generationConfig"] + + assert "maxOutputTokens" not in generation_config_for("inf") + assert generation_config_for(200)["maxOutputTokens"] == 200 + + def _grounded_live_frame(grounding_metadata: Mapping[str, object] | None) -> Mapping[str, object]: """One Live server frame. Grounding metadata and usageMetadata arrive together, as Vertex sends them.""" from typing import Final diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index b76e1707805..720a1b4f458 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,8 +1,16 @@ +import json +import logging +from collections.abc import Mapping from typing import Final, List, cast from unittest.mock import MagicMock, patch +from urllib.parse import urlparse +import httpx import pytest +import respx +from openai import AsyncOpenAI +from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY from litellm.llms.mistral.chat.transformation import ( MistralChatResponseIterator, @@ -45,21 +53,15 @@ class TestMistralReasoningSupport: mistral_config = MistralConfig() # Test magistral model supports reasoning parameters - supported_params = mistral_config.get_supported_openai_params( - "mistral/magistral-medium-2506" - ) + supported_params = mistral_config.get_supported_openai_params("mistral/magistral-medium-2506") assert "reasoning_effort" in supported_params assert "thinking" in supported_params - supported_params_reasoning = mistral_config.get_supported_openai_params( - "mistral/mistral-medium-latest" - ) + supported_params_reasoning = mistral_config.get_supported_openai_params("mistral/mistral-medium-latest") assert "reasoning_effort" in supported_params_reasoning assert "thinking" not in supported_params_reasoning - supported_params_normal = mistral_config.get_supported_openai_params( - "mistral/mistral-large-latest" - ) + supported_params_normal = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") assert "reasoning_effort" not in supported_params_normal assert "thinking" not in supported_params_normal @@ -117,6 +119,36 @@ class TestMistralReasoningSupport: ) assert optional_params["reasoning_effort"] == sent + @pytest.mark.parametrize( + ("requested", "drop_params", "sent"), + [ + ({"effort": "medium"}, False, {"reasoning_effort": "high"}), + ({"effort": "high", "summary": "auto"}, True, {"reasoning_effort": "high"}), + ({"summary": "auto"}, True, {}), + ], + ) + def test_reasoning_effort_object_is_sent_as_its_level(self, local_model_cost_map, requested, drop_params, sent): + import litellm + + optional_params = litellm.get_optional_params( + model="mistral-small-latest", + custom_llm_provider="mistral", + reasoning_effort=requested, + drop_params=drop_params, + ) + assert {key: optional_params[key] for key in ("reasoning_effort",) if key in optional_params} == sent + + @pytest.mark.parametrize("requested", [{"effort": "high", "summary": "auto"}, {"summary": "auto"}]) + def test_reasoning_effort_object_mistral_cannot_take_is_rejected(self, local_model_cost_map, requested): + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['reasoning_effort'\]"): + litellm.get_optional_params( + model="devstral-latest", + custom_llm_provider="mistral", + reasoning_effort=requested, + ) + def test_reasoning_effort_is_forwarded_verbatim_when_the_map_declares_no_levels( self, local_model_cost_map, monkeypatch ): @@ -194,9 +226,7 @@ class TestMistralReasoningSupport: messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should add a new system message at the beginning assert len(result) == 2 @@ -218,9 +248,7 @@ class TestMistralReasoningSupport: ] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should modify existing system message assert len(result) == 2 @@ -251,9 +279,7 @@ class TestMistralReasoningSupport: ] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should modify existing system message preserving list format assert len(result) == 2 @@ -266,10 +292,7 @@ class TestMistralReasoningSupport: # Original content should be preserved assert "You are a helpful assistant." in result[0]["content"][1]["text"] - assert ( - "You always provide detailed explanations." - in result[0]["content"][2]["text"] - ) + assert "You always provide detailed explanations." in result[0]["content"][2]["text"] assert result[1]["role"] == "user" @@ -287,9 +310,7 @@ class TestMistralReasoningSupport: ] string_params = {"_add_reasoning_prompt": True} - string_result = mistral_config._add_reasoning_system_prompt_if_needed( - string_messages, string_params - ) + string_result = mistral_config._add_reasoning_system_prompt_if_needed(string_messages, string_params) assert isinstance(string_result[0]["content"], str) assert "" in string_result[0]["content"] assert "You are helpful." in string_result[0]["content"] @@ -304,9 +325,7 @@ class TestMistralReasoningSupport: ] list_params = {"_add_reasoning_prompt": True} - list_result = mistral_config._add_reasoning_system_prompt_if_needed( - list_messages, list_params - ) + list_result = mistral_config._add_reasoning_system_prompt_if_needed(list_messages, list_params) assert isinstance(list_result[0]["content"], list) assert list_result[0]["content"][0]["type"] == "text" assert "" in list_result[0]["content"][0]["text"] @@ -319,9 +338,7 @@ class TestMistralReasoningSupport: messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should return messages unchanged assert result == messages @@ -444,9 +461,7 @@ class TestMistralReasoningSupport: def test_mistral_streaming_chunk_preserves_thinking_blocks(): """Ensure streaming chunks keep magistral reasoning content.""" - iterator = MistralChatResponseIterator( - streaming_response=iter([]), sync_stream=True, json_mode=False - ) + iterator = MistralChatResponseIterator(streaming_response=iter([]), sync_stream=True, json_mode=False) streamed_chunk = { "id": "chunk-1", @@ -523,9 +538,7 @@ class TestMistralParallelToolCalls: def test_get_supported_openai_params_includes_parallel_tool_calls(self): """Test that parallel_tool_calls is in supported parameters.""" mistral_config = MistralConfig() - supported_params = mistral_config.get_supported_openai_params( - "mistral/mistral-large-latest" - ) + supported_params = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") assert "parallel_tool_calls" in supported_params def test_transform_request_preserves_parallel_tool_calls(self): @@ -691,9 +704,7 @@ class TestMistralEmptyContentHandling: result = MistralConfig._handle_empty_content_response(response_data) - assert ( - result["choices"][0]["message"]["content"] == "Hello, how can I help you?" - ) + assert result["choices"][0]["message"]["content"] == "Hello, how can I help you?" def test_handle_empty_content_response_handles_multiple_choices(self): """Test that only the first choice is processed for empty content.""" @@ -1431,9 +1442,7 @@ def test_mistral_transform_request_keeps_an_empty_tool_message_after_its_tool_ca { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, {"role": "tool", "tool_call_id": "call_1", "content": ""}, ], @@ -1457,9 +1466,7 @@ def test_mistral_transform_request_sends_a_null_tool_reply_as_empty_text(reply): { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, {"role": "tool", "tool_call_id": "call_1", **reply}, ], @@ -1514,9 +1521,7 @@ def test_mistral_transform_request_hoists_tool_message_image(): { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, { "role": "tool", @@ -1665,3 +1670,377 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_ ) assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] + + +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +@pytest.mark.parametrize( + "param, value", + ( + ("reasoning_effort", "none"), + ("reasoning_effort", "xhigh"), + ("thinking", {"type": "disabled"}), + ("thinking", {"type": ["enabled"]}), + ), +) +def test_magistral_reasoning_request_replaced_by_prompt_warns_once( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = MistralConfig().map_openai_params( + non_default_params={param: value}, + optional_params={}, + model="magistral-medium-latest", + drop_params=False, + ) + + warnings: Final = _warnings_naming(caplog, param) + assert result["_add_reasoning_prompt"] is True + assert len(warnings) == 1 + assert "magistral-medium-latest" in warnings[0] + + +@pytest.mark.parametrize( + "param, value", + ( + ("reasoning_effort", "low"), + ("reasoning_effort", "medium"), + ("reasoning_effort", "high"), + ("thinking", {"type": "enabled", "budget_tokens": 1024}), + ), +) +def test_magistral_reasoning_request_the_prompt_honors_does_not_warn( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = MistralConfig().map_openai_params( + non_default_params={param: value}, + optional_params={}, + model="magistral-medium-latest", + drop_params=False, + ) + + assert result["_add_reasoning_prompt"] is True + assert _warnings_naming(caplog, param) == () + + +def test_non_magistral_reasoning_effort_is_sent_without_warning(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = MistralConfig().map_openai_params( + non_default_params={"reasoning_effort": "high"}, + optional_params={}, + model="mistral-medium-latest", + drop_params=False, + ) + + assert result["reasoning_effort"] == "high" + assert _warnings_naming(caplog, "reasoning_effort") == () + + +@pytest.mark.parametrize("reasoning_effort", [5, {"effort": 5}, {"effort": 5, "summary": "auto"}]) +def test_mistral_warns_once_when_drop_params_drops_a_level_it_cannot_send( + caplog: pytest.LogCaptureFixture, reasoning_effort: object +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = MistralConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort}, + optional_params={}, + model="mistral-medium-latest", + drop_params=True, + ) + + assert "reasoning_effort" not in result + warnings: Final = _warnings_naming(caplog, "reasoning_effort") + assert len(warnings) == 1 + assert "model='mistral-medium-latest'" in warnings[0] + assert "(drop_params=True)" in warnings[0] + + +@pytest.mark.parametrize("reasoning_effort", [{"summary": "auto"}, {"effort": "high", "summary": "auto"}]) +def test_mistral_stays_quiet_under_drop_params_when_no_level_is_lost( + caplog: pytest.LogCaptureFixture, reasoning_effort: object +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + MistralConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort}, + optional_params={}, + model="mistral-medium-latest", + drop_params=True, + ) + + assert _warnings_naming(caplog, "reasoning_effort") == () + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [ + ("mistral-large-latest", "mistral"), + ("mistral-large-4", "mistral"), + ("a-mistral-model-missing-from-the-cost-map", "mistral"), + ("codestral-latest", "codestral"), + ], +) +def test_prompt_cache_key_reaches_the_mistral_request_body( + local_model_cost_map: None, model: str, custom_llm_provider: str +) -> None: + import litellm + + optional_params: Final = litellm.get_optional_params( + model=model, custom_llm_provider=custom_llm_provider, prompt_cache_key="session-1" + ) + request: Final = MistralConfig().transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request["prompt_cache_key"] == "session-1" + + +AZURE_HOSTED_MISTRAL_API_BASE: Final = "https://my-resource.services.ai.azure.com/openai/v1" + + +def _transform_with_prompt_cache_key( + litellm_params: dict[str, object], prompt_cache_key: object = "session-1" +) -> Mapping[str, object]: + return cast( + Mapping[str, object], + MistralConfig().transform_request( + model="mistral-small-latest", + messages=[{"role": "user", "content": "hi"}], + optional_params={"prompt_cache_key": prompt_cache_key, "temperature": 0.2}, + litellm_params=litellm_params, + headers={}, + ), + ) + + +@pytest.mark.parametrize("api_base", ["https://api.mistral.ai/v1", "https://codestral.mistral.ai/v1"]) +def test_prompt_cache_key_reaches_a_mistral_hosted_endpoint(api_base: str) -> None: + request: Final = _transform_with_prompt_cache_key({"api_base": api_base}) + + assert request["prompt_cache_key"] == "session-1" + + +@pytest.mark.parametrize("api_base", [AZURE_HOSTED_MISTRAL_API_BASE, "https://mistral.ai.example.com/v1"]) +def test_prompt_cache_key_for_another_host_is_rejected_without_drop_params(api_base: str) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="prompt_cache_key") as rejection: + _transform_with_prompt_cache_key({"api_base": api_base}) + + upstream_host: Final = urlparse(api_base).hostname or api_base + assert upstream_host not in str(rejection.value) + + +UNSENDABLE_PROMPT_CACHE_KEYS: Final = ("Büro-Sitzung-42", "日本", 123, ["session-1"]) + + +@pytest.mark.parametrize("prompt_cache_key", UNSENDABLE_PROMPT_CACHE_KEYS) +def test_prompt_cache_key_the_mistral_api_cannot_take_is_rejected_without_drop_params( + prompt_cache_key: object, +) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="ASCII"): + _transform_with_prompt_cache_key({"api_base": "https://api.mistral.ai/v1"}, prompt_cache_key) + + +@pytest.mark.parametrize("prompt_cache_key", UNSENDABLE_PROMPT_CACHE_KEYS) +def test_prompt_cache_key_the_mistral_api_cannot_take_is_dropped_under_drop_params(prompt_cache_key: object) -> None: + request: Final = _transform_with_prompt_cache_key( + {"api_base": "https://api.mistral.ai/v1", "drop_params": True}, prompt_cache_key + ) + + assert "prompt_cache_key" not in request + assert request["temperature"] == 0.2 + + +def test_prompt_cache_key_with_ascii_punctuation_reaches_the_mistral_api() -> None: + request: Final = _transform_with_prompt_cache_key({}, "team a/session:42") + + assert request["prompt_cache_key"] == "team a/session:42" + + +def test_prompt_cache_key_allowlisted_for_another_purpose_is_still_rejected_for_another_host() -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="allowed_openai_params"): + _transform_with_prompt_cache_key( + {"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "allowed_openai_params": ["reasoning_effort"]} + ) + + +def _chat_completion_reply() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "mistral-small-latest", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +def test_allowlisted_prompt_cache_key_reaches_a_gateway_in_front_of_mistral(respx_mock: respx.MockRouter) -> None: + import litellm + + gateway_route: Final = respx_mock.post("https://gateway.example.com/mistral/v1/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + litellm.completion( + model="mistral/mistral-small-latest", + messages=[{"role": "user", "content": "hi"}], + api_base="https://gateway.example.com/mistral/v1", + api_key="gateway-key", + prompt_cache_key="session-1", + allowed_openai_params=["prompt_cache_key"], + max_retries=0, + ) + + assert json.loads(gateway_route.calls.last.request.content)["prompt_cache_key"] == "session-1" + + +@pytest.mark.asyncio +async def test_prompt_cache_key_is_dropped_from_an_async_codestral_call_to_another_host( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + drop_params=True, + max_retries=0, + client=AsyncOpenAI( + api_key="azure-key", base_url=AZURE_HOSTED_MISTRAL_API_BASE, http_client=httpx.AsyncClient() + ), + ) + + assert "prompt_cache_key" not in json.loads(azure_route.calls.last.request.content) + + +@pytest.mark.asyncio +async def test_prompt_cache_key_rejection_for_an_async_codestral_call_names_codestral( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + with pytest.raises(litellm.BadRequestError, match=r"codestral does not support parameters: \['prompt_cache_key'\]"): + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + max_retries=0, + client=AsyncOpenAI( + api_key="azure-key", base_url=AZURE_HOSTED_MISTRAL_API_BASE, http_client=httpx.AsyncClient() + ), + ) + + assert not azure_route.called + + +@pytest.mark.asyncio +async def test_async_codestral_call_sends_an_uploaded_file_by_its_id(respx_mock: respx.MockRouter) -> None: + import litellm + + codestral_api_base: Final = "https://codestral.mistral.ai/v1" + codestral_route: Final = respx_mock.post(f"{codestral_api_base}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Review this file"}, + {"type": "file", "file": {"file_id": "file-12345"}}, + ], + } + ], + api_base=codestral_api_base, + api_key="codestral-key", + max_retries=0, + client=AsyncOpenAI(api_key="codestral-key", base_url=codestral_api_base, http_client=httpx.AsyncClient()), + ) + + sent_content: Final = json.loads(codestral_route.calls.last.request.content)["messages"][0]["content"] + assert sent_content[1] == {"type": "file", "file_id": "file-12345"} + + +def test_prompt_cache_key_rejection_for_a_sync_codestral_call_is_an_unsupported_params_error( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + with pytest.raises( + litellm.UnsupportedParamsError, match=r"^litellm.UnsupportedParamsError: codestral does not support" + ): + litellm.completion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + max_retries=0, + ) + + assert not azure_route.called + + +def test_prompt_cache_key_for_another_host_is_dropped_under_drop_params() -> None: + request: Final = _transform_with_prompt_cache_key({"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "drop_params": True}) + + assert "prompt_cache_key" not in request + assert request["temperature"] == 0.2 + + +def test_prompt_cache_key_for_the_azure_endpoint_set_in_the_environment_is_dropped_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISTRAL_AZURE_API_BASE", "https://my-resource.services.ai.azure.com/openai") + + request: Final = _transform_with_prompt_cache_key({"drop_params": True}) + + assert "prompt_cache_key" not in request + + +def test_prompt_cache_key_stays_unsupported_for_vertex_hosted_mistral(local_model_cost_map: None) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError): + litellm.get_optional_params(model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s") + + dropped: Final = litellm.get_optional_params( + model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s", drop_params=True + ) + assert "prompt_cache_key" not in dropped diff --git a/tests/unit/llms/oci/chat/test_oci_chat_transformation.py b/tests/unit/llms/oci/chat/test_oci_chat_transformation.py index 0535279d177..f22b5e32593 100644 --- a/tests/unit/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/unit/llms/oci/chat/test_oci_chat_transformation.py @@ -1,4 +1,5 @@ import datetime +import logging from typing import Final import httpx import pytest @@ -1103,19 +1104,25 @@ class TestOCIReasoningEffort: ) assert chat_request.get("reasoningEffort") == "HIGH" - def test_reasoning_effort_unsupported_on_cohere_dropped(self): - """drop_params=True → silently drop reasoning_effort for Cohere.""" + def test_reasoning_effort_unsupported_on_cohere_dropped_with_one_warning(self, caplog): from litellm.llms.oci.chat.transformation import OCIChatConfig config = OCIChatConfig() - result = config.map_openai_params( - non_default_params={"reasoning_effort": "low"}, - optional_params={}, - model="cohere.command-latest", - drop_params=True, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="cohere.command-latest", + drop_params=True, + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning_effort" not in result assert "reasoningEffort" not in result + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("reasoning_effort='low'", "cohere.command-latest", "(drop_params=True)") + ) def test_reasoning_effort_unsupported_on_cohere_raises(self): """drop_params=False → raise rather than ship a payload Cohere will reject.""" @@ -2110,16 +2117,20 @@ class TestReasoningEffortGating: assert "reasoning_effort" not in OCIChatConfig().get_supported_openai_params("meta.llama-3.3-70b-instruct") @pytest.mark.parametrize("model", SERVED_GROK_4_20_MODELS) - def test_reasoning_effort_dropped_under_drop_params(self, local_model_cost_map, model): - result = OCIChatConfig().map_openai_params( - non_default_params={"reasoning_effort": "low", "temperature": 0.1}, - optional_params={}, - model=model, - drop_params=True, - ) + def test_reasoning_effort_dropped_under_drop_params_with_one_warning(self, local_model_cost_map, model, caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result = OCIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "low", "temperature": 0.1}, + optional_params={}, + model=model, + drop_params=True, + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoningEffort" not in result assert "reasoning_effort" not in result assert result["temperature"] == 0.1 + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='low'", model, "(drop_params=True)")) def test_reasoning_effort_raises_without_drop_params_for_gated_grok(self, local_model_cost_map, monkeypatch): monkeypatch.setattr(litellm, "drop_params", False) diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index df9f5fd9d14..af1b85cf325 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -1,5 +1,6 @@ import asyncio import inspect +import logging import os import sys from typing import Final, cast @@ -23,6 +24,7 @@ import json from unittest.mock import ANY, MagicMock, patch import litellm +from litellm._logging import verbose_logger from litellm.types.utils import Choices, EmbeddingResponse, Message, ModelResponse, ModelResponseStream from unittest import mock @@ -1114,3 +1116,36 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "http://gateway.example.com/ollama/api/chat" ) + + +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +@pytest.mark.parametrize("effort", ("minimal", "xhigh")) +def test_reasoning_effort_mapped_to_think_off_warns_once(effort: str, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaChatConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model="qwen3", drop_params=False + ) + + warnings: Final = _warnings_naming(caplog, "reasoning_effort") + assert result["think"] is False + assert len(warnings) == 1 + assert f"reasoning_effort={effort!r}" in warnings[0] + assert "qwen3" in warnings[0] + + +@pytest.mark.parametrize("effort, think", (("none", False), ("low", True), ("high", True))) +def test_reasoning_effort_with_a_matching_think_value_does_not_warn( + effort: str, think: bool, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaChatConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model="qwen3", drop_params=False + ) + + assert result["think"] is think + assert _warnings_naming(caplog, "reasoning_effort") == () diff --git a/tests/unit/llms/ollama/test_ollama_completion_transformation.py b/tests/unit/llms/ollama/test_ollama_completion_transformation.py index 248e745ebff..edea39bb10c 100644 --- a/tests/unit/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_completion_transformation.py @@ -1,6 +1,7 @@ import base64 import io import json +import logging import sys from typing import Final from unittest.mock import MagicMock, patch @@ -10,6 +11,7 @@ import pytest import respx import litellm +from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.ollama.completion.transformation import ( @@ -990,3 +992,40 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "http://gateway.example.com/ollama/api/generate" ) + +def _reasoning_effort_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and "Dropping reasoning_effort=" in r.getMessage() + ) + + +@pytest.mark.parametrize("effort", ("minimal", "xhigh")) +def test_reasoning_effort_mapped_to_think_off_warns_once(effort: str, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model="qwen3:0.6b", drop_params=False + ) + + warnings: Final = _reasoning_effort_warnings(caplog) + assert result["think"] is False + assert len(warnings) == 1 + assert f"reasoning_effort={effort!r}" in warnings[0] + assert "qwen3:0.6b" in warnings[0] + + +@pytest.mark.parametrize( + "model, effort, think", + (("qwen3:0.6b", "none", False), ("qwen3:0.6b", "high", True), ("gpt-oss:20b", "xhigh", "xhigh")), +) +def test_reasoning_effort_ollama_honors_does_not_warn( + model: str, effort: str, think: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model=model, drop_params=False + ) + + assert result["think"] == think + assert _reasoning_effort_warnings(caplog) == () diff --git a/tests/unit/llms/openai/test_openai_common_utils.py b/tests/unit/llms/openai/test_openai_common_utils.py index 46c14fcfd8f..f0865cca7f9 100644 --- a/tests/unit/llms/openai/test_openai_common_utils.py +++ b/tests/unit/llms/openai/test_openai_common_utils.py @@ -1,4 +1,6 @@ import asyncio +import json +import logging from typing import Final from unittest.mock import MagicMock, patch @@ -9,7 +11,11 @@ import respx import litellm from litellm.litellm_core_utils.token_counter import token_counter -from litellm.llms.openai.common_utils import BaseOpenAILLM, is_openai_backed_api_base +from litellm.llms.openai.common_utils import ( + BaseOpenAILLM, + drop_params_from_unprocessable_entity_error, + is_openai_backed_api_base, +) # Test parameters for different API functions API_FUNCTION_PARAMS = [ @@ -319,6 +325,104 @@ def _completion_kwargs(provider: str, client, **overrides) -> dict: } +_REASONING_EFFORT_422: Final = { + "error": { + "message": json.dumps( + {"detail": [{"loc": ["body", "reasoning_effort"], "msg": "Extra inputs are not permitted"}]} + ) + } +} +_CHAT_COMPLETION: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-6.1-sol", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +_CHAT_COMPLETION_STREAM: Final = ( + 'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-6.1-sol",' + '"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}\n\n' + "data: [DONE]\n\n" +) + + +def _answer_422_while_reasoning_effort_is_sent(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + if "reasoning_effort" in body: + return httpx.Response(422, json=_REASONING_EFFORT_422) + if body.get("stream"): + return httpx.Response(200, text=_CHAT_COMPLETION_STREAM, headers={"content-type": "text/event-stream"}) + return httpx.Response(200, json=_CHAT_COMPLETION) + + +async def _complete_through_the_422_retry(call_shape: str) -> str: + transport: Final = httpx.MockTransport(_answer_422_while_reasoning_effort_is_sent) + kwargs: Final = { + "model": "gpt-6.1-sol", + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": "high", + } + if call_shape == "sync": + sync_client: Final = openai.OpenAI(api_key="sk-not-a-real-key", http_client=httpx.Client(transport=transport)) + return litellm.completion(**kwargs, client=sync_client).choices[0].message.content + async_client: Final = openai.AsyncOpenAI( + api_key="sk-not-a-real-key", http_client=httpx.AsyncClient(transport=transport) + ) + if call_shape == "async": + return (await litellm.acompletion(**kwargs, client=async_client)).choices[0].message.content + stream: Final = await litellm.acompletion(**kwargs, client=async_client, stream=True) + return "".join([chunk.choices[0].delta.content or "" async for chunk in stream]) + + +@pytest.mark.parametrize("call_shape", ["sync", "async", "async_stream"]) +def test_a_422_naming_reasoning_effort_drops_it_with_one_warning( + call_shape: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + content: Final = asyncio.run(_complete_through_the_422_retry(call_shape)) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert content == "ok" + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("reasoning_effort='high'", "gpt-6.1-sol", "(drop_params=True)", "422") + ) + + +def test_a_422_with_an_unhashable_loc_still_drops_reasoning_effort_with_one_warning( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + detail: Final = [ + {"loc": ["body", ["messages", 0]], "msg": "Invalid"}, + {"loc": ["body", "reasoning_effort"], "msg": "Extra inputs are not permitted"}, + ] + + def answer(request: httpx.Request) -> httpx.Response: + if "reasoning_effort" in json.loads(request.content): + return httpx.Response(422, json={"error": {"message": json.dumps({"detail": detail})}}) + return httpx.Response(200, json=_CHAT_COMPLETION) + + monkeypatch.setattr(litellm, "drop_params", True) + client: Final = openai.OpenAI( + api_key="sk-not-a-real-key", http_client=httpx.Client(transport=httpx.MockTransport(answer)) + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + response: Final = litellm.completion( + model="gpt-6.1-sol", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort="high", + client=client, + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert response.choices[0].message.content == "ok" + assert len(drop_warnings) == 1 + assert "reasoning_effort='high'" in drop_warnings[0] + + @pytest.mark.parametrize("provider", ["openai", "azure"]) def test_sync_output_limit_400_maps_to_length_truncated_response(provider): response = litellm.completion( @@ -588,3 +692,24 @@ async def test_azure_gateway_and_image_variations_use_the_callers_async_session( ) assert response.data[0].url == "https://example.com/image.png" assert not session.is_closed + + + +def test_reasoning_drop_warning_caps_a_large_caller_value(caplog: pytest.LogCaptureFixture) -> None: + error: Final = httpx.HTTPStatusError( + message="422", + request=httpx.Request("POST", "https://example.invalid/v1/chat/completions"), + response=httpx.Response(422, json=_REASONING_EFFORT_422), + ) + data: Final[dict[str, object]] = {"model": "gpt-6.1-sol", "messages": [], "reasoning_effort": "x" * 5000} + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + retried: Final = drop_params_from_unprocessable_entity_error( + error, data, model="gpt-6.1-sol", drop_params_gated=True + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert "reasoning_effort" not in retried + assert len(drop_warnings) == 1 + assert len(drop_warnings[0]) < 300 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='xxx", "'gpt-6.1-sol'", "422")) diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index 59355dfabcb..888c47505a4 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -1,6 +1,7 @@ import json from typing import Final +from unittest.mock import MagicMock import httpx import pytest @@ -464,6 +465,34 @@ def test_openrouter_cost_tracking_non_streaming(): ) + +@pytest.mark.parametrize("cost", [float("nan"), float("inf")]) +def test_openrouter_chat_leaves_a_non_finite_provider_cost_out_of_the_response_headers(cost: float) -> None: + body: Final = { + "id": "gen-123", + "object": "chat.completion", + "created": 1234567890, + "model": "openai/gpt-5.6-luna", + "choices": [{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop", "index": 0}], + "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11, "cost": cost}, + } + + result: Final = OpenrouterConfig().transform_response( + model="openrouter/openai/gpt-5.6-luna", + raw_response=httpx.Response( + 200, content=json.dumps(body).encode(), headers={"content-type": "application/json"} + ), + model_response=litellm.ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "Reply with ok"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert "llm_provider-x-litellm-response-cost" not in result._hidden_params.get("additional_headers", {}) + def test_openrouter_cost_tracking_streaming(): """ Test OpenRouter cost tracking for streaming completions. diff --git a/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py b/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py index 19517e58b31..78d7161c388 100644 --- a/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py +++ b/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py @@ -9,6 +9,8 @@ reasoning.encrypted_content for multi-turn stateless workflows. Related issue: https://github.com/BerriAI/litellm/issues/22189 """ +import json +from typing import Final from unittest.mock import MagicMock import httpx @@ -157,6 +159,29 @@ def test_openrouter_responses_returns_provider_reported_cost(cost): assert get_response_cost_from_hidden_params(response.hidden_params) == cost +@pytest.mark.parametrize("cost", [float("nan"), float("inf")]) +def test_openrouter_responses_drops_a_non_finite_provider_cost(cost: float) -> None: + body: Final = { + "id": "resp_test", + "created_at": 1, + "model": "unmapped-model", + "object": "response", + "output": [], + "status": "completed", + "usage": {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5, "cost": cost}, + } + response: Final = OpenRouterResponsesAPIConfig().transform_response_api_response( + model="openrouter/unmapped-model", + raw_response=httpx.Response( + 200, content=json.dumps(body).encode(), headers={"content-type": "application/json"} + ), + logging_obj=MagicMock(), + ) + + assert get_response_cost_from_hidden_params(response.hidden_params) is None + assert json.loads(json.dumps(response.model_dump(), allow_nan=False, default=str))["usage"]["cost"] is None + + @pytest.mark.parametrize( "event_type", ["response.completed", "response.incomplete", "response.failed"], diff --git a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py index 83c71479311..380f85445e5 100644 --- a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py @@ -6,9 +6,11 @@ search queries, and reasoning tokens. """ import json +import logging import math import os from datetime import datetime, timezone +from typing import Final from unittest.mock import patch import pytest @@ -140,6 +142,26 @@ class TestPerplexityCostCalculator: assert prompt_cost == 0.0 assert completion_cost == 0.008 + @pytest.mark.parametrize( + "reported_cost", + [float("nan"), float("inf"), {"total_cost": float("nan")}], + ids=["nan", "inf", "nan-total-cost-dict"], + ) + def test_a_non_finite_perplexity_cost_is_priced_like_no_reported_cost_with_a_warning( + self, reported_cost: float | dict, caplog: pytest.LogCaptureFixture + ): + unreported: Final = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + reported: Final = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + reported.cost = reported_cost + + token_priced: Final = perplexity_cost_per_token(model="sonar-pro", usage=unreported) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + reported_priced: Final = perplexity_cost_per_token(model="sonar-pro", usage=reported) + + assert sum(token_priced) > 0 + assert reported_priced == token_priced + assert [record for record in caplog.records if "non-finite provider-reported cost" in record.getMessage()] + OFF_PEAK_MODEL = "sonar-off-peak-test" OFF_PEAK_WINDOW = "14:00-00:00" INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) diff --git a/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py b/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py index f8caa8d5fed..356528d8909 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py @@ -343,6 +343,26 @@ def test_body_carries_no_aws_credentials( assert client.request_headers["X-Amz-Security-Token"] == "test-session-token" +@pytest.mark.parametrize("model", ["sagemaker_chat/my-endpoint", "sagemaker_nova/my-endpoint"]) +def test_inference_component_travels_only_in_the_header(monkeypatch: pytest.MonkeyPatch, model: str): + client = _invoke_sagemaker_chat(monkeypatch, model=model, model_id="my-inference-component") + + assert client.request_headers[_INFERENCE_COMPONENT_HEADER] == "my-inference-component" + assert "model_id" not in client.request_body + + +def test_inference_component_header_keeps_header_keys_that_are_not_strings(): + headers = SagemakerChatConfig().validate_environment( + headers={7: "other", "x-amzn-sagemaker-inference-component": "client-component"}, + model="my-endpoint", + messages=[], + optional_params={"model_id": "my-inference-component"}, + litellm_params={}, + ) + + assert headers == {7: "other", _INFERENCE_COMPONENT_HEADER: "my-inference-component"} + + def test_body_model_stays_the_endpoint_name_when_hf_model_name_is_unset(monkeypatch): """Without `hf_model_name` the body must keep the model it has today.""" client = _invoke_sagemaker_chat(monkeypatch) diff --git a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py index 0980c9f5c66..5e40c2ebb60 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py @@ -372,6 +372,27 @@ def test_completion_sagemaker_non_stream( assert response.choices[0].message.content == "SageMaker reply" +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.usefixtures("fake_provider_credentials") +def test_completion_sagemaker_sends_the_inference_component_only_as_a_header(respx_mock: respx.MockRouter) -> None: + url: Final = "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/my-endpoint/invocations" + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"}) + ) + litellm.completion( + model="sagemaker/my-endpoint", + messages=[{"role": "user", "content": "hi"}], + temperature=0.2, + model_id="my-inference-component", + aws_region_name="us-west-2", + num_retries=0, + ) + + request: Final = route.calls[0].request + assert request.headers["X-Amzn-SageMaker-Inference-Component"] == "my-inference-component" + assert json.loads(request.content) == {"inputs": "hi", "parameters": {"temperature": 0.2}} + + @pytest.mark.usefixtures("httpx_transport") @pytest.mark.usefixtures("fake_provider_credentials") def test_completion_sagemaker_non_stream_with_aws_params(respx_mock: respx.MockRouter) -> None: diff --git a/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py b/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py index 4d6d252ae6e..1bf646fc2ad 100644 --- a/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py +++ b/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py @@ -1,5 +1,10 @@ +import logging +from typing import Final from unittest.mock import patch +import pytest + +from litellm._logging import verbose_logger from litellm.llms.tencent.chat.transformation import TencentChatConfig @@ -208,6 +213,27 @@ class TestAdaptiveThinkingCoercion: assert result["extra_body"]["thinking"] == {"type": "adaptive", "budget_tokens": 4096} + @pytest.mark.parametrize("budget, warned", (("high", 1), (4096, 0)), ids=("non-integer", "integer")) + def test_enabled_thinking_on_adaptive_only_model_warns_only_when_the_budget_is_dropped( + self, local_model_cost_map, budget: object, warned: int, caplog: pytest.LogCaptureFixture + ): + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = TencentChatConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": budget}}, + optional_params={}, + model="tencent/minimax-m3", + drop_params=False, + ) + + warnings: Final = tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and f"Dropping thinking.budget_tokens={budget!r}" in r.getMessage() + ) + assert result["extra_body"]["thinking"]["type"] == "adaptive" + assert result["extra_body"]["thinking"].get("budget_tokens") == (None if warned else budget) + assert len(warnings) == warned + def test_disabled_thinking_kept_for_adaptive_only_model(self, local_model_cost_map): config = TencentChatConfig() result = config.map_openai_params( diff --git a/tests/unit/llms/test_subscription_accounts.py b/tests/unit/llms/test_subscription_accounts.py new file mode 100644 index 00000000000..6995ee5d4df --- /dev/null +++ b/tests/unit/llms/test_subscription_accounts.py @@ -0,0 +1,110 @@ +import base64 +import json +from pathlib import Path +from typing import Final + +import pytest + +import litellm +from litellm.constants import SUBSCRIPTION_BACKED_PROVIDERS +from litellm.llms.chatgpt.common_utils import OPENAI_AUTH_CLAIM_KEY +from litellm.llms.subscription_accounts import SUBSCRIPTION_ACCOUNT_ID_RESOLVERS, resolve_subscription_account_id +from litellm.types.utils import CredentialItem + + +def _write_login(path: Path, account_id: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"account_id": account_id})) + return path + + +def _access_token_for(account_id: str) -> str: + claims: Final = json.dumps({OPENAI_AUTH_CLAIM_KEY: {"chatgpt_account_id": account_id}}).encode() + payload: Final = base64.urlsafe_b64encode(claims).decode().rstrip("=") + return f"header.{payload}.signature" + + +@pytest.fixture +def signed_in_chatgpt_token_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + _write_login(tmp_path / "auth.json", "acct-from-token-file") + return tmp_path + + +def test_every_subscription_backed_provider_has_an_account_id_resolver() -> None: + assert frozenset(SUBSCRIPTION_ACCOUNT_ID_RESOLVERS) == SUBSCRIPTION_BACKED_PROVIDERS + + +def test_chatgpt_resolves_to_the_account_in_the_signed_in_token_file(signed_in_chatgpt_token_dir: Path) -> None: + assert resolve_subscription_account_id("chatgpt", {"model": "chatgpt/gpt-5.6-terra"}) == "acct-from-token-file" + + +def test_each_deployment_takes_the_account_of_its_own_auth_file(signed_in_chatgpt_token_dir: Path) -> None: + first: Final = _write_login(signed_in_chatgpt_token_dir / "team-a" / "auth.json", "acct-team-a") + second: Final = _write_login(signed_in_chatgpt_token_dir / "team-b" / "auth.json", "acct-team-b") + + assert resolve_subscription_account_id("chatgpt", {"chatgpt_auth_file": str(first)}) == "acct-team-a" + assert resolve_subscription_account_id("chatgpt", {"chatgpt_auth_file": str(second)}) == "acct-team-b" + + +def test_a_configured_access_token_resolves_to_its_own_account(signed_in_chatgpt_token_dir: Path) -> None: + litellm_params: Final = {"model": "chatgpt/gpt-5.6-terra", "api_key": _access_token_for("acct-from-api-key")} + + assert resolve_subscription_account_id("chatgpt", litellm_params) == "acct-from-api-key" + + +def test_a_metered_provider_resolves_to_no_account_even_with_a_chatgpt_login( + signed_in_chatgpt_token_dir: Path, +) -> None: + assert resolve_subscription_account_id("openai", {"model": "openai/gpt-5.6-terra"}) is None + + +def test_chatgpt_without_a_login_resolves_to_no_account(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + empty_dir: Final = tmp_path / "no-login" + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(empty_dir)) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + assert resolve_subscription_account_id("chatgpt", {"model": "chatgpt/gpt-5.6-terra"}) is None + + +def test_resolving_an_account_leaves_the_login_file_untouched(signed_in_chatgpt_token_dir: Path) -> None: + login: Final = signed_in_chatgpt_token_dir / "no-saved-account" / "auth.json" + login.parent.mkdir(parents=True) + saved: Final = json.dumps({"access_token": _access_token_for("acct-derived"), "refresh_token": "rt-current"}) + login.write_text(saved) + + assert resolve_subscription_account_id("chatgpt", {"chatgpt_auth_file": str(login)}) == "acct-derived" + assert login.read_text() == saved + + +@pytest.mark.parametrize("credential_field", ["chatgpt_auth_file", "api_key"]) +def test_a_named_credential_resolves_to_the_account_it_signs_in_as( + credential_field: str, signed_in_chatgpt_token_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + login: Final = _write_login(signed_in_chatgpt_token_dir / "named" / "auth.json", "acct-named-credential") + values: Final = { + "chatgpt_auth_file": {"chatgpt_auth_file": str(login)}, + "api_key": {"api_key": _access_token_for("acct-named-credential")}, + }[credential_field] + monkeypatch.setattr( + litellm, + "credential_list", + [CredentialItem(credential_name="chatgpt-team", credential_values=values, credential_info={})], + ) + litellm_params: Final = {"model": "chatgpt/gpt-5.6-terra", "litellm_credential_name": "chatgpt-team"} + + assert resolve_subscription_account_id("chatgpt", litellm_params) == "acct-named-credential" + + +@pytest.mark.parametrize( + ("api_key", "expected_account"), + [("oidc/env/CHATGPT_TEAM_TOKEN", "acct-from-oidc-env"), ("oidc/env/CHATGPT_UNSET_TOKEN", None)], +) +def test_an_oidc_env_api_key_resolves_to_the_account_of_the_token_it_points_to( + api_key: str, expected_account: str | None, signed_in_chatgpt_token_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TEAM_TOKEN", _access_token_for("acct-from-oidc-env")) + monkeypatch.delenv("CHATGPT_UNSET_TOKEN", raising=False) + litellm_params: Final = {"model": "chatgpt/gpt-5.6-terra", "api_key": api_key} + + assert resolve_subscription_account_id("chatgpt", litellm_params) == expected_account diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 67c78b62151..98d1b8f1300 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1693,6 +1693,78 @@ def test_vertex_ai_map_thinking_param_without_budget_tokens_for_gemini_3(): assert result["thinkingConfig"] == {"includeThoughts": True} +def _budget_drop_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and "thinking.budget_tokens" in r.getMessage() + ) + + +@pytest.mark.parametrize("budget", ("high", 1024.5, {"tokens": 1024})) +def test_gemini_non_integer_thinking_budget_drop_warns_once(budget: object, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": budget}}, + optional_params={}, + model="gemini-2.5-flash", + drop_params=False, + ) + + warnings: Final = _budget_drop_warnings(caplog) + assert "thinkingBudget" not in result["thinkingConfig"] + assert len(warnings) == 1 + assert f"thinking.budget_tokens={budget!r}" in warnings[0] + assert "gemini-2.5-flash" in warnings[0] + + +def test_gemini_integer_thinking_budget_is_sent_without_warning(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 2048}}, + optional_params={}, + model="gemini-2.5-flash", + drop_params=False, + ) + + assert result["thinkingConfig"]["thinkingBudget"] == 2048 + assert _budget_drop_warnings(caplog) == () + + +def test_gemini_3_thinking_budget_drop_warns_once(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 2048}}, + optional_params={}, + model="gemini-3-pro-preview", + drop_params=False, + ) + + warnings: Final = _budget_drop_warnings(caplog) + assert "thinkingBudget" not in result["thinkingConfig"] + assert len(warnings) == 1 + assert "thinking.budget_tokens=2048" in warnings[0] + assert "gemini-3-pro-preview" in warnings[0] + + +@pytest.mark.parametrize( + "thinking", + ({"type": "enabled"}, {"type": "enabled", "budget_tokens": 0}, {"type": "disabled", "budget_tokens": 2048}), +) +def test_gemini_3_thinking_without_a_dropped_budget_does_not_warn( + thinking: dict[str, object], caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": thinking}, + optional_params={}, + model="gemini-3-pro-preview", + drop_params=False, + ) + + assert _budget_drop_warnings(caplog) == () + + def test_vertex_ai_map_tools(): v = VertexGeminiConfig() optional_params = {} diff --git a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 0bfff870d15..7f0106616ce 100644 --- a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -18,6 +18,7 @@ import websockets.exceptions # registers websockets.exceptions on the websocket import litellm from litellm.llms.vertex_ai.common_utils import get_vertex_base_url +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig # --------------------------------------------------------------------------- @@ -179,6 +180,25 @@ def test_vertex_audio_only_live_model_coerces_text_modality_to_audio( assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] +@pytest.mark.parametrize("drop_params", [False, True]) +def test_vertex_realtime_config_carries_drop_params(drop_params, patch_native_audio_cost_map_entry): + from litellm.llms.vertex_ai.realtime.transformation import vertex_realtime_config + + cfg: Final = vertex_realtime_config( + _NATIVE_AUDIO_MODEL, + access_token="tok", + resolve_access_token=AsyncMock(return_value="tok"), + project="my-proj", + location="us-central1", + drop_params=drop_params, + ) + + rejection: Final = cfg.setup_session_update_rejection( + _NATIVE_AUDIO_MODEL, {"modalities": ["text"]}, setup_sent=False + ) + assert (rejection is None) is drop_params + + def test_vertex_session_update_normalizes_ga_remapped_fields( patch_native_audio_cost_map_entry, ): @@ -253,6 +273,26 @@ def test_vertex_keeps_turn_detection_tuning_when_auto_response_is_off(): } + +@pytest.mark.parametrize( + ("session", "disabled"), + [ + ({"turn_detection": None}, True), + ({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, True), + ({"type": "realtime", "audio": {"input": {}}}, False), + ], +) +def test_vertex_turns_activity_detection_off_for_null_turn_detection(session, disabled): + """OpenAI's turn_detection null is push-to-talk, so Vertex must wait for the client's activity markers.""" + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1") + messages: Final = cfg.transform_realtime_request( + json.dumps({"type": "session.update", "session": session}), + _NATIVE_AUDIO_MODEL, + session_configuration_request=None, + ) + setup: Final = json.loads(messages[0])["setup"] + assert setup["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] is disabled + # --------------------------------------------------------------------------- # Round-trip test: text-in / text-out via RealTimeStreaming # --------------------------------------------------------------------------- @@ -607,3 +647,73 @@ def test_vertex_native_audio_unmapped_voice_passes_through(patch_native_audio_co generation_config = json.loads(messages[0])["setup"]["generationConfig"] assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + + +def test_vertex_session_update_two_output_modalities_collapse_to_audio(): + """Vertex Live rejects a setup naming two response modalities, so a GA + ["text", "audio"] request must be sent as AUDIO only.""" + cfg = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="global") + session_update = { + "type": "session.update", + "session": {"output_modalities": ["text", "audio"], "instructions": "Be brief."}, + } + + messages = cfg.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash", + session_configuration_request=None, + ) + + assert len(messages) == 1 + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"] + + +@pytest.mark.parametrize( + ("session", "param"), + [ + ({"voice": "Nova"}, "session.voice"), + ({"type": "realtime", "audio": {"output": {"voice": "Nova"}}}, "session.audio.output.voice"), + ], +) +def test_vertex_rejects_an_undocumented_voice_on_the_session_update(session, param): + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1") + + rejection: Final = cfg.setup_session_update_rejection("gemini-live-2.5-flash", session, setup_sent=False) + + assert rejection is not None + assert (rejection.code, rejection.param) == ("invalid_value", param) + + +@pytest.mark.parametrize("voice", ["Kore", "alloy"]) +def test_vertex_accepts_documented_and_openai_stock_voices_on_the_session_update(voice): + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1") + + assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", {"voice": voice}, setup_sent=False) is None + + +def test_vertex_drops_an_undocumented_voice_with_drop_params(): + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1", drop_params=True) + session_update: Final = {"type": "session.update", "session": {"voice": "Nova", "instructions": "Be brief."}} + + assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None + messages: Final = cfg.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + assert "speechConfig" not in json.loads(messages[0])["setup"].get("generationConfig", {}) + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("voice", ["fable", "onyx"]) +def test_vertex_sends_the_documented_voice_an_openai_tts_voice_maps_to(voice, drop_params): + cfg: Final = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1", drop_params=drop_params + ) + session_update: Final = {"type": "session.update", "session": {"voice": voice}} + + assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None + messages: Final = cfg.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + assert json.loads(messages[0])["setup"]["generationConfig"]["speechConfig"] == { + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES[voice]}} + } diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 0bdc5f1910a..6b59f6d40dc 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -47,6 +47,7 @@ from litellm.proxy.auth.auth_checks import ( can_object_call_model, _can_object_call_vector_stores, _check_agent_access_group_model_access, + agent_model_grants, check_end_user_budget, _check_team_member_budget, personal_key_owner_teams, @@ -11294,6 +11295,15 @@ async def test_agent_access_groups_naming_no_model_deny_every_model(): assert exc_info.value.type == ProxyErrorTypes.agent_model_access_denied +@pytest.mark.asyncio +async def test_agent_model_grants_names_every_model_the_agents_access_groups_allow(): + agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=[]) + resolve, asked = _agent_model_ceiling_resolver(frozenset({"gpt-5", "openai/gpt-5.6"})) + + assert await agent_model_grants(agent_key, resolve) == frozenset({"gpt-5", "openai/gpt-5.6"}) + assert asked == ["agent-1"] + + @pytest.mark.asyncio async def test_agent_without_access_groups_adds_no_model_ceiling(): agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"]) diff --git a/tests/unit/proxy/auth/test_handle_jwt.py b/tests/unit/proxy/auth/test_handle_jwt.py index 51cf15996cd..4c223edc20d 100644 --- a/tests/unit/proxy/auth/test_handle_jwt.py +++ b/tests/unit/proxy/auth/test_handle_jwt.py @@ -913,6 +913,31 @@ async def test_sync_user_role_and_teams_cache_invalidation_on_team_change(): assert call_kwargs.kwargs["model_type"] == LiteLLM_UserTable +@pytest.mark.asyncio +async def test_sync_user_role_and_teams_keeps_record_order_and_adds_claim_teams_in_claim_order(): + jwt_handler: Final = JWTHandler() + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=AsyncMock(), + litellm_jwtauth=LiteLLM_JWTAuth(team_ids_jwt_field="groups", sync_user_role_and_teams=True), + ) + cache: Final = AsyncMock() + user: Final = LiteLLM_UserTable(user_id="u1", teams=["team-zeta", "team-kappa", "team-alpha"]) + token: Final = {"groups": ["team-mu", "team-alpha", "team-beta", "team-mu", "team-zeta", "team-omega"]} + + with patch( + "litellm.proxy.management_endpoints.scim.scim_v2.patch_team_membership", + new_callable=AsyncMock, + ) as mock_patch: + await JWTAuthManager.sync_user_role_and_teams(jwt_handler, token, user, AsyncMock(), user_api_key_cache=cache) + + assert mock_patch.await_args.kwargs["teams_ids_to_add_user_to"] == ["team-mu", "team-beta", "team-omega"] + assert mock_patch.await_args.kwargs["teams_ids_to_remove_user_from"] == ["team-kappa"] + expected_teams: Final = ["team-zeta", "team-alpha", "team-mu", "team-beta", "team-omega"] + assert user.teams == expected_teams + assert cache.async_set_cache.await_args.kwargs["value"].teams == expected_teams + + @pytest.mark.asyncio async def test_sync_user_role_and_teams_no_cache_write_when_nothing_changes(): """Test that cache is NOT written when role and teams already match.""" diff --git a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py index 92cfb6a71e5..cbb37c964c6 100644 --- a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py @@ -5,6 +5,7 @@ from typing import Callable, Coroutine, Final, Iterable, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch import pytest +from pydantic import BaseModel, ConfigDict from redis.asyncio import Redis import litellm @@ -102,8 +103,8 @@ class _ScriptedPubSubClient: class _QueuePubSub: - def __init__(self, initial_messages: Iterable[str] = ()) -> None: - self.queue: "asyncio.Queue[str]" = asyncio.Queue() + def __init__(self, initial_messages: Iterable[object] = ()) -> None: + self.queue: "asyncio.Queue[object]" = asyncio.Queue() for message in initial_messages: self.queue.put_nowait(message) self.subscribed_channels: List[str] = [] @@ -112,7 +113,7 @@ class _QueuePubSub: async def subscribe(self, *channels: str) -> None: self.subscribed_channels.extend(channels) - async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]: if timeout == 0: try: return self.queue.get_nowait() @@ -128,7 +129,7 @@ class _QueuePubSub: class _BrokenPubSub(_QueuePubSub): - async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]: raise ConnectionError("connection lost") @@ -138,11 +139,11 @@ class _CloseFailingBrokenPubSub(_BrokenPubSub): class _EmptyPollsThenMessagePubSub(_QueuePubSub): - def __init__(self, empty_polls: int, initial_messages: Iterable[str] = ()) -> None: + def __init__(self, empty_polls: int, initial_messages: Iterable[object] = ()) -> None: super().__init__(initial_messages=initial_messages) self.remaining_empty_polls = empty_polls - async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]: if timeout != 0 and self.remaining_empty_polls > 0: self.remaining_empty_polls -= 1 return None @@ -157,6 +158,38 @@ class _FakeClock: return self.now +class _VirtualTimePubSub(_QueuePubSub): + """Timed polls move the injected clock forward instead of waiting on the wall clock. + + A delivery scheduled at a clock time is returned by the first poll whose window + reaches it, with the clock set to that time, so a test asserts when each resync + and targeted apply ran in subscriber time. + """ + + def __init__(self, clock: _FakeClock, deliveries: Iterable[Tuple[float, object]] = ()) -> None: + super().__init__() + self._clock = clock + self.pending: List[Tuple[float, object]] = sorted(deliveries, key=lambda delivery: delivery[0]) + + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]: + await asyncio.sleep(0) + due_by = self._clock.now + timeout + if self.pending and self.pending[0][0] <= due_by: + delivered_at, message = self.pending.pop(0) + self._clock.now = max(self._clock.now, delivered_at) + return message + self._clock.now = due_by + return None + + +def _change(*model_ids: str) -> str: + return json.dumps({"object_type": "litellm_proxymodeltable", "model_ids": list(model_ids)}) + + +def _redis_delivery(message: str) -> dict: + return {"type": "message", "pattern": None, "channel": b"litellm_proxy.config_change", "data": message.encode()} + + class _ScriptedPubSubRedisClient(Redis): def __init__(self, pubsubs: Iterable[_QueuePubSub]) -> None: self._scripted_pubsubs = iter(pubsubs) @@ -204,7 +237,16 @@ async def test_publish_sends_object_type_json_on_channel() -> None: assert len(client.published) == 1 channel, message = client.published[0] assert channel == "litellm_proxy.config_change" - assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} + assert json.loads(message) == {"object_type": "litellm_proxymodeltable", "model_ids": []} + + +async def test_publish_carries_the_written_model_ids() -> None: + client = _RecordingRedisClient() + cache = _FakeRedisCache(client) + + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable", model_ids=("m-1", "m-2")) + + assert json.loads(client.published[0][1]) == {"object_type": "litellm_proxymodeltable", "model_ids": ["m-1", "m-2"]} async def test_publish_uses_namespaced_channel() -> None: @@ -235,7 +277,7 @@ async def test_publish_reaches_cluster_derived_pubsub_clients() -> None: await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable") assert client.published == [ - (CONFIG_SYNC_CHANNEL, json.dumps({"object_type": "litellm_proxymodeltable"})) + (CONFIG_SYNC_CHANNEL, json.dumps({"object_type": "litellm_proxymodeltable", "model_ids": []})) ] @@ -312,29 +354,210 @@ class _MaxJitterRandom(random.Random): return b -async def test_debounce_sleep_adds_jitter_from_injected_rng() -> None: - pubsub = _QueuePubSub(initial_messages=[json.dumps({"object_type": "litellm_proxymodeltable"})]) - cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) - sleeps: List[float] = [] - fired = asyncio.Event() +def _virtual_time_subscriber( + pubsub: _VirtualTimePubSub, + clock: _FakeClock, + events: List[str], + resynced: asyncio.Event, + *, + debounce_seconds: float = 0.0, + jitter_max_seconds: float = 0.0, + min_resync_interval_seconds: float = 10.0, + rng: Optional[random.Random] = None, + expected_resyncs: int = 1, + apply_error: Optional[Exception] = None, +) -> ConfigSyncSubscriber: + async def resync() -> None: + events.append(f"resync@{clock.now}") + if sum(event.startswith("resync@") for event in events) >= expected_resyncs: + resynced.set() - async def recording_sleep(seconds: float) -> None: - sleeps.append(seconds) + async def apply(model_ids: Tuple[str, ...]) -> None: + events.append(f"apply:{','.join(model_ids)}@{clock.now}") + if apply_error is not None: + raise apply_error - subscriber = ConfigSyncSubscriber( - redis_cache=cache, - resync_callbacks=(_recording_callback([], "resync", fired),), - debounce_seconds=1.0, - jitter_max_seconds=4.0, - rng=_MaxJitterRandom(), - sleep=recording_sleep, + return ConfigSyncSubscriber( + redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])), + resync_callbacks=(resync,), + apply_model_ids=apply, + debounce_seconds=debounce_seconds, + jitter_max_seconds=jitter_max_seconds, + min_resync_interval_seconds=min_resync_interval_seconds, + rng=rng, + monotonic=clock, ) - subscriber.start() - await asyncio.wait_for(fired.wait(), timeout=5) - await subscriber.stop() - assert sleeps == [5.0] +async def _run_until_resynced(subscriber: ConfigSyncSubscriber, resynced: asyncio.Event) -> None: + subscriber.start() + try: + await asyncio.wait_for(resynced.wait(), timeout=5) + finally: + await subscriber.stop() + + +async def test_debounce_waits_out_the_jitter_from_injected_rng() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change())]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber( + pubsub, clock, events, resynced, debounce_seconds=1.0, jitter_max_seconds=4.0, rng=_MaxJitterRandom() + ) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["resync@1005.0"] + + +async def test_published_model_ids_apply_on_receipt_while_the_full_resync_still_debounces() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change("m-1"))]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber( + pubsub, clock, events, resynced, debounce_seconds=1.0, jitter_max_seconds=4.0, rng=_MaxJitterRandom() + ) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["apply:m-1@1000.0", "resync@1005.0"] + + +async def test_model_ids_published_inside_the_throttle_wait_apply_without_waiting() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub( + clock, deliveries=[(1000.0, _change()), (1004.0, _change("m-2")), (1006.0, _change("m-3", "m-4"))] + ) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["resync@1000.0", "apply:m-2@1004.0", "apply:m-3,m-4@1006.0", "resync@1010.0"] + + +async def test_model_ids_already_waiting_apply_together_in_one_call() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub( + clock, deliveries=[(1000.0, _change("m-1")), (1000.0, _change("m-2", "m-1")), (1000.0, _change("m-3"))] + ) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["apply:m-1,m-2,m-3@1000.0", "resync@1000.0"] + + +async def test_redis_shaped_message_applies_its_model_ids() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _redis_delivery(_change("m-1")))]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["apply:m-1@1000.0", "resync@1000.0"] + + +async def test_messages_without_usable_model_ids_only_schedule_the_full_resync() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub( + clock, + deliveries=[ + (1000.0, "change"), + (1004.0, json.dumps({"object_type": "litellm_proxymodeltable"})), + (1005.0, json.dumps({"object_type": "litellm_proxymodeltable", "model_ids": "m-1"})), + (1006.0, {"type": "message", "pattern": None, "channel": b"c", "data": 7}), + (1007.0, None), + ], + ) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["resync@1000.0", "resync@1010.0"] + + +async def test_failing_targeted_apply_still_runs_the_full_resync() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change("m-1"))]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, apply_error=RuntimeError("db down")) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["apply:m-1@1000.0", "resync@1000.0"] + + +async def test_subscriber_without_an_apply_callback_only_resyncs() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change("m-1"))]) + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])), + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.0, + jitter_max_seconds=0.0, + monotonic=clock, + ) + + await _run_until_resynced(subscriber, fired) + + assert resyncs == ["resync"] + + +async def test_peer_holds_every_published_row_before_its_next_full_resync() -> None: + publisher = _RecordingRedisClient() + cache = _FakeRedisCache(publisher) + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable", model_ids=("m-order-1",)) + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable", model_ids=("m-order-2",)) + on_the_wire = [_redis_delivery(message) for _, message in publisher.published] + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, on_the_wire[0]), (1007.0, on_the_wire[1])]) + db_rows_written_at = {"m-order-1": 1000.0, "m-order-2": 1007.0} + registry: set = set() + snapshots: List[Tuple[str, float, frozenset]] = [] + resynced = asyncio.Event() + + async def apply(model_ids: Tuple[str, ...]) -> None: + registry.update(model_ids) + snapshots.append(("apply", clock.now, frozenset(registry))) + + async def resync() -> None: + registry.update(row for row, written_at in db_rows_written_at.items() if written_at <= clock.now) + snapshots.append(("resync", clock.now, frozenset(registry))) + if sum(kind == "resync" for kind, _, _ in snapshots) == 2: + resynced.set() + + subscriber = ConfigSyncSubscriber( + redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])), + resync_callbacks=(resync,), + apply_model_ids=apply, + debounce_seconds=1.0, + jitter_max_seconds=5.0, + min_resync_interval_seconds=10.0, + rng=_MaxJitterRandom(), + monotonic=clock, + ) + + await _run_until_resynced(subscriber, resynced) + + assert snapshots == [ + ("apply", 1000.0, frozenset({"m-order-1"})), + ("resync", 1006.0, frozenset({"m-order-1"})), + ("apply", 1007.0, frozenset({"m-order-1", "m-order-2"})), + ("resync", 1016.0, frozenset({"m-order-1", "m-order-2"})), + ] def test_default_jitter_window_is_nonzero() -> None: @@ -345,92 +568,43 @@ def test_default_min_resync_interval_caps_reload_rate() -> None: assert CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS > CONFIG_SYNC_JITTER_MAX_SECONDS -def _throttled_subscriber( - cache: object, - events: List[str], - fired: asyncio.Event, - clock: _FakeClock, - min_resync_interval_seconds: float = 10.0, -) -> ConfigSyncSubscriber: - async def recording_sleep(seconds: float) -> None: - events.append(f"sleep:{seconds}") - await asyncio.sleep(0) - - async def resync() -> None: - events.append("resync") - fired.set() - - return ConfigSyncSubscriber( - redis_cache=cache, - resync_callbacks=(resync,), - debounce_seconds=0.0, - jitter_max_seconds=0.0, - min_resync_interval_seconds=min_resync_interval_seconds, - sleep=recording_sleep, - monotonic=clock, - ) - - async def test_resync_arriving_inside_min_interval_waits_out_the_remainder() -> None: - pubsub = _QueuePubSub() - cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) - events: List[str] = [] - fired = asyncio.Event() clock = _FakeClock() - subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, "change"), (1004.0, "change")]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2) - subscriber.start() - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - fired.clear() - clock.now += 4.0 - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - await subscriber.stop() + await _run_until_resynced(subscriber, resynced) - assert events == ["sleep:0.0", "resync", "sleep:0.0", "sleep:6.0", "resync"] + assert events == ["resync@1000.0", "resync@1010.0"] async def test_resync_after_min_interval_elapsed_is_not_throttled() -> None: - pubsub = _QueuePubSub() - cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) - events: List[str] = [] - fired = asyncio.Event() clock = _FakeClock() - subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, "change"), (1030.0, "change")]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2) - subscriber.start() - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - fired.clear() - clock.now += 30.0 - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - await subscriber.stop() + await _run_until_resynced(subscriber, resynced) - assert events == ["sleep:0.0", "resync", "sleep:0.0", "resync"] + assert events == ["resync@1000.0", "resync@1030.0"] async def test_writes_during_the_throttle_wait_collapse_into_the_next_resync() -> None: - pubsub = _QueuePubSub() - cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) - events: List[str] = [] - fired = asyncio.Event() clock = _FakeClock() - subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + pubsub = _VirtualTimePubSub( + clock, deliveries=[(1000.0, "change")] + [(1004.0 + offset, "change") for offset in range(5)] + ) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2) - subscriber.start() - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - fired.clear() - for _ in range(5): - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - await asyncio.sleep(0.1) - await subscriber.stop() + await _run_until_resynced(subscriber, resynced) - assert events.count("resync") == 2 - assert pubsub.queue.empty() + assert events == ["resync@1000.0", "resync@1010.0"] + assert pubsub.pending == [] async def test_polls_without_messages_do_not_trigger_resyncs() -> None: @@ -646,13 +820,39 @@ class _AllWritesTableActions: return action -def _recording_publish(calls: List[Tuple[str, str]]) -> Callable[[str], Coroutine[None, None, None]]: - async def publish(object_type: str) -> None: +def _recording_publish(calls: List[Tuple[str, str]]) -> Callable[[str, Tuple[str, ...]], Coroutine[None, None, None]]: + async def publish(object_type: str, model_ids: Tuple[str, ...]) -> None: calls.append(("publish", object_type)) return publish +def _recording_publish_with_ids( + calls: List[Tuple[str, Tuple[str, ...]]], +) -> Callable[[str, Tuple[str, ...]], Coroutine[None, None, None]]: + async def publish(object_type: str, model_ids: Tuple[str, ...]) -> None: + calls.append((object_type, model_ids)) + + return publish + + +class _ModelRow(BaseModel): + model_config = ConfigDict(frozen=True) + + model_id: str + model_name: str + + +class _ModelRowTableActions: + def __getattr__(self, name: str) -> Callable[..., Coroutine[None, None, object]]: + async def action(*args: object, **kwargs: object) -> object: + if name.endswith("_many"): + return 2 + return _ModelRow(model_id="m-1", model_name="gpt-5.2") + + return action + + def test_wrapper_passes_through_unsynced_tables() -> None: actions = object() @@ -722,6 +922,46 @@ async def test_wrapper_publishes_for_every_write_action(action_name: str) -> Non assert publish_calls == [("publish", "litellm_guardrailstable")] +@pytest.mark.parametrize( + "action_name, expected_ids", + [ + ("create", ("m-1",)), + ("update", ("m-1",)), + ("upsert", ("m-1",)), + ("delete", ()), + ("create_many", ()), + ("update_many", ()), + ("delete_many", ()), + ], +) +async def test_wrapper_publishes_the_written_model_id_for_row_writes( + action_name: str, expected_ids: Tuple[str, ...] +) -> None: + publish_calls: List[Tuple[str, Tuple[str, ...]]] = [] + wrapped = wrap_table_actions_for_config_sync( + actions=_ModelRowTableActions(), + table_name="litellm_proxymodeltable", + publish=_recording_publish_with_ids(publish_calls), + ) + + await getattr(wrapped, action_name)(data={}) + + assert publish_calls == [("litellm_proxymodeltable", expected_ids)] + + +async def test_wrapper_publishes_no_model_ids_for_other_tables() -> None: + publish_calls: List[Tuple[str, Tuple[str, ...]]] = [] + wrapped = wrap_table_actions_for_config_sync( + actions=_ModelRowTableActions(), + table_name="litellm_guardrailstable", + publish=_recording_publish_with_ids(publish_calls), + ) + + await wrapped.create(data={}) + + assert publish_calls == [("litellm_guardrailstable", ())] + + async def test_model_repository_write_publishes_via_live_coordination_cache() -> None: from litellm.proxy import proxy_server from litellm.proxy.proxy_server import _set_redis_usage_cache @@ -747,7 +987,7 @@ async def test_model_repository_write_publishes_via_live_coordination_cache() -> assert len(client.published) == 1 channel, message = client.published[0] assert channel == CONFIG_SYNC_CHANNEL - assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} + assert json.loads(message) == {"object_type": "litellm_proxymodeltable", "model_ids": ["m-1"]} async def test_ui_settings_write_publishes_via_live_coordination_cache() -> None: @@ -774,7 +1014,7 @@ async def test_ui_settings_write_publishes_via_live_coordination_cache() -> None assert len(client.published) == 1 channel, message = client.published[0] assert channel == CONFIG_SYNC_CHANNEL - assert json.loads(message) == {"object_type": "litellm_uisettings"} + assert json.loads(message) == {"object_type": "litellm_uisettings", "model_ids": []} async def _publish_calls_for_invalidated_param(param_name: str) -> List[Tuple[str, str]]: @@ -799,7 +1039,7 @@ async def test_invalidate_config_param_publishes_params_a_resync_applies(param_n assert len(published) == 1 channel, message = published[0] assert channel == CONFIG_SYNC_CHANNEL - assert json.loads(message) == {"object_type": param_name} + assert json.loads(message) == {"object_type": param_name, "model_ids": []} @pytest.mark.parametrize("param_name", _STARTUP_ONLY_CONFIG_PARAM_NAMES) @@ -932,6 +1172,46 @@ async def test_proxy_config_subscriber_resyncs_deployments_only() -> None: assert subscriber._task is None +async def test_proxy_config_subscriber_applies_published_model_ids_from_the_db( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import ProxyConfig + + rows = [MagicMock()] + prisma_client = MagicMock() + prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=rows) + router = MagicMock() + router.registry.return_value.deployments.return_value = ["deployment"] + installed: List[object] = [] + + def install(db_models: object) -> None: + installed.append(db_models) + + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", None) + monkeypatch.setattr(proxy_server.proxy_config, "get_credentials", AsyncMock()) + monkeypatch.setattr(proxy_server.proxy_config, "_add_deployment", install) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([_QueuePubSub()])) + config = ProxyConfig() + + config.start_config_sync_subscriber(prisma_client=prisma_client, proxy_logging_obj=MagicMock(), redis_cache=cache) + subscriber = config.config_sync_subscriber + assert subscriber is not None + apply = subscriber._apply_model_ids + assert apply is not None + await apply(("m-1", "m-2")) + await config.stop_config_sync_subscriber() + + prisma_client.db.litellm_proxymodeltable.find_many.assert_awaited_once_with( + where={"model_id": {"in": ["m-1", "m-2"]}} + ) + assert installed == [rows] + assert proxy_server.llm_model_list == ["deployment"] + + async def test_proxy_config_does_not_start_subscriber_without_coordination_redis() -> None: from litellm.proxy.proxy_server import ProxyConfig diff --git a/tests/unit/proxy/common_utils/test_openai_error_payload.py b/tests/unit/proxy/common_utils/test_openai_error_payload.py index acefc4c1e2c..d4ab811d505 100644 --- a/tests/unit/proxy/common_utils/test_openai_error_payload.py +++ b/tests/unit/proxy/common_utils/test_openai_error_payload.py @@ -111,10 +111,13 @@ def test_the_default_status_stands_when_the_exception_carries_none(default: int) assert error_status_code(ValueError("boom"), default) == default -@pytest.mark.parametrize("carried_status", [True, False, "429", None, 429.0]) -def test_a_non_int_carried_status_falls_back_to_the_default(carried_status: object): - """True is an int in Python but not an HTTP status, and a stringified one would break - every caller that compares the code numerically.""" +@pytest.mark.parametrize( + "carried_status", + [True, False, "slow down", "", None, 429.0, 200, 1301, 0, "200", "1301", "0", "0429", " 429", "\u0664\u0662\u0669", "4" * 5000], +) +def test_a_carried_status_that_is_not_an_error_status_falls_back_to_the_default(carried_status: object): + """True is an int in Python but not an HTTP status, and answering a 200 or a 1301 either reads + as success or makes uvicorn drop the connection.""" class _Carrier(Exception): status_code = carried_status @@ -122,6 +125,22 @@ def test_a_non_int_carried_status_falls_back_to_the_default(carried_status: obje assert error_status_code(_Carrier("boom"), 500) == 500 +def test_a_status_carried_as_a_decimal_string_is_answered_as_that_status(): + """Provider stream-error chunks pass the vendor's code through as a string ``status_code``, + and the shared handler used to answer it as is, so dropping it turned a rate limit into a 500.""" + + class _Carrier(Exception): + status_code = "429" + + assert error_status_code(_Carrier("slow down"), 500) == 429 + assert proxy_exception_for(_Carrier("slow down"), default_status_code=500).to_dict() == { + "message": "slow down", + "type": "rate_limit_error", + "param": None, + "code": "429", + } + + def test_a_proxy_exception_keeps_the_status_it_was_raised_with(): """ProxyException stores its status as the string ``code`` rather than ``status_code``, so a route tail that rewraps one used to answer a 4xx rejection as a 500.""" @@ -130,8 +149,8 @@ def test_a_proxy_exception_keeps_the_status_it_was_raised_with(): assert error_status_code(rejection, 500) == 400 -@pytest.mark.parametrize("carried_code", [None, "None", "", "rate_limited", "4xx", 404]) -def test_a_code_that_is_not_a_decimal_string_falls_back_to_the_default(carried_code: object): +@pytest.mark.parametrize("carried_code", [None, "None", "", "rate_limited", "4xx", 404, "200", "1301"]) +def test_a_code_that_is_not_a_stringified_error_status_falls_back_to_the_default(carried_code: object): """Only ProxyException's stringified status is a status; ``code`` on anything else (OpenAI's ``invalid_api_key``, a stray int) says nothing about the HTTP answer.""" diff --git a/tests/unit/proxy/common_utils/test_registry_read_through.py b/tests/unit/proxy/common_utils/test_registry_read_through.py index 04729167707..da7b1f497c9 100644 --- a/tests/unit/proxy/common_utils/test_registry_read_through.py +++ b/tests/unit/proxy/common_utils/test_registry_read_through.py @@ -1,4 +1,5 @@ import asyncio +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final import pytest @@ -6,7 +7,10 @@ import pytest from litellm.proxy.common_utils.registry_read_through import RegistryReadThrough if TYPE_CHECKING: + from litellm import Router + from litellm.proxy._types import LiteLLM_ProxyModelTable from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry + from litellm.types.utils import CredentialItem def nothing_loaded(_key: str) -> bool: @@ -487,6 +491,178 @@ async def test_resync_model_deployments_loads_db_credentials_before_reconciling_ installed.assert_called_once_with(db_models=rows, credential={"api_key": "sk-from-db"}) +@pytest.fixture +def stored_params_salt(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-salt-stored-model-rows") + + +def _stored_model_row(model_id: str, stored_params: Mapping[str, str]) -> "LiteLLM_ProxyModelTable": + from litellm.proxy._types import LiteLLM_ProxyModelTable + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + + return LiteLLM_ProxyModelTable( + model_id=model_id, + model_name="gpt-failover-group", + litellm_params={key: encrypt_value_helper(value) for key, value in stored_params.items()}, + model_info={"id": model_id}, + ) + + +def _backup_row(model_id: str, model: str = "openai/gpt-backup") -> "LiteLLM_ProxyModelTable": + return _stored_model_row(model_id, {"model": model, "api_key": "sk-backup"}) + + +def _router_serving(*model_ids: str) -> "Router": + from litellm import Router + + return Router( + model_list=[ + { + "model_name": "gpt-failover-group", + "litellm_params": {"model": "openai/gpt-primary", "api_key": "sk-primary"}, + "model_info": {"id": model_id}, + } + for model_id in model_ids + ] + ) + + +def _served_ids(deployments: "Sequence[Mapping[str, Mapping[str, object]]] | None") -> frozenset[object]: + return frozenset(deployment["model_info"]["id"] for deployment in deployments or ()) + + +class _ProxyModelTable: + def __init__(self, rows: "tuple[LiteLLM_ProxyModelTable, ...]") -> None: + self.rows = rows + self.reads = 0 + + async def find_many( + self, where: Mapping[str, Mapping[str, Sequence[str]]] + ) -> "tuple[LiteLLM_ProxyModelTable, ...]": + self.reads += 1 + return tuple(row for row in self.rows if row.model_id in where["model_id"]["in"]) + + async def create(self, data: Mapping[str, object]) -> None: + raise AssertionError(f"a model load wrote a row: {data}") + + +class _CredentialsTable: + def __init__(self, rows: "tuple[CredentialItem, ...]") -> None: + self.rows = rows + + async def find_many(self) -> "tuple[CredentialItem, ...]": + return self.rows + + +class _Db: + def __init__( + self, litellm_proxymodeltable: _ProxyModelTable, litellm_credentialstable: _CredentialsTable | None = None + ) -> None: + self.litellm_proxymodeltable = litellm_proxymodeltable + self.litellm_credentialstable = litellm_credentialstable or _CredentialsTable(()) + self.other_tables_read: list[str] = [] + + def __getattr__(self, table_name: str) -> object: + self.other_tables_read.append(table_name) + raise AttributeError(table_name) + + +class _PrismaClientWithoutConfigRows: + def __init__(self, db: object) -> None: + self.db = db + + async def get_generic_data(self, key: str, value: str, table_name: str) -> None: + return None + + +def _serve_db_models(monkeypatch: pytest.MonkeyPatch, db: object, router: "Router | None") -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "prisma_client", _PrismaClientWithoutConfigRows(db)) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", None) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_installs_only_the_published_rows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + router: Final = _router_serving("primary-id") + _serve_db_models( + monkeypatch, _Db(_ProxyModelTable((_backup_row("m-1"), _backup_row("m-2"), _backup_row("m-3")))), router + ) + + await resync_model_deployments_by_ids(("m-1", "m-2")) + + assert _served_ids(router.registry().deployments()) == {"primary-id", "m-1", "m-2"} + assert _served_ids(proxy_server.llm_model_list) == {"primary-id", "m-1", "m-2"} + assert not proxy_server.MODEL_RECONCILE_LOCK.locked() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_leaves_the_router_alone_for_deleted_rows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + router: Final = _router_serving("primary-id") + _serve_db_models(monkeypatch, _Db(_ProxyModelTable(())), router) + + await resync_model_deployments_by_ids(("deleted-id",)) + + assert _served_ids(router.registry().deployments()) == {"primary-id"} + assert proxy_server.llm_model_list is None + assert not proxy_server.MODEL_RECONCILE_LOCK.locked() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_without_a_router_leaves_the_reload_to_the_debounced_resync( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + table: Final = _ProxyModelTable((_backup_row("m-1"),)) + db: Final = _Db(table) + _serve_db_models(monkeypatch, db, None) + + await resync_model_deployments_by_ids(("m-1",)) + + assert proxy_server.llm_router is None + assert table.reads == 1 + assert db.other_tables_read == [] + + +@pytest.mark.asyncio +async def test_resync_model_deployments_by_ids_respects_supported_db_objects( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock( + side_effect=AssertionError("db hit for an object type this replica does not load") + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["guardrails"]}) + + await resync_model_deployments_by_ids(("gated-out-id",)) + + prisma_client.db.litellm_proxymodeltable.find_many.assert_not_awaited() + + @pytest.mark.asyncio async def test_resync_model_deployments_respects_supported_db_objects(monkeypatch): from unittest.mock import AsyncMock, MagicMock @@ -786,3 +962,174 @@ async def test_resync_model_deployments_reconciles_every_db_row_when_the_missed_ assert await _resync_model_deployments(router_name) is router_holds_the_name_after_reconcile reconciled.assert_called_once_with(prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_reads_the_writer_not_a_lagging_replica( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from unittest.mock import MagicMock + + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer: Final = MagicMock(name="writer_prisma") + writer.litellm_proxymodeltable = _ProxyModelTable((_backup_row("backup-id"),)) + replica_table: Final = _ProxyModelTable(()) + replica: Final = MagicMock(name="lagging_replica_prisma") + replica.litellm_proxymodeltable = replica_table + router: Final = _router_serving("primary-id") + _serve_db_models( + monkeypatch, + RoutingPrismaWrapper( + writer=PrismaWrapper(original_prisma=writer, iam_token_db_auth=False), + reader=PrismaWrapper(original_prisma=replica, iam_token_db_auth=False), + ), + router, + ) + + await resync_model_deployments_by_ids(("backup-id",)) + + assert router.registry().has_deployment("backup-id") + assert replica_table.reads == 0 + + +@pytest.mark.asyncio +async def test_unknown_model_name_read_through_reads_the_replica_without_waiting_for_a_reload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer: Final = MagicMock(name="writer_prisma") + writer.litellm_proxymodeltable = SimpleNamespace(find_many=AsyncMock(return_value=[]), create=AsyncMock()) + replica: Final = MagicMock(name="replica_prisma") + replica.litellm_proxymodeltable = SimpleNamespace(find_many=AsyncMock(return_value=[]), create=AsyncMock()) + prisma_client: Final = MagicMock() + prisma_client.db = RoutingPrismaWrapper( + writer=PrismaWrapper(original_prisma=writer, iam_token_db_auth=False), + reader=PrismaWrapper(original_prisma=replica, iam_token_db_auth=False), + ) + reload_in_progress: Final = asyncio.Lock() + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", MagicMock()) + monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", reload_in_progress) + + async with reload_in_progress: + found: Final = await asyncio.wait_for(_resync_model_deployments("no-such-model"), timeout=2) + + assert found is False + writer.litellm_proxymodeltable.find_many.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_installs_the_row_a_concurrent_reload_left_current( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + table: Final = _ProxyModelTable((_backup_row("m-1", model="openai/gpt-before-update"),)) + router: Final = _router_serving("primary-id") + _serve_db_models(monkeypatch, _Db(table), router) + reload_in_progress: Final = asyncio.Lock() + monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", reload_in_progress) + + async with reload_in_progress: + targeted_load: Final = asyncio.create_task(resync_model_deployments_by_ids(("m-1",))) + for _ in range(5): + await asyncio.sleep(0) + assert not router.registry().has_deployment("m-1") + table.rows = (_backup_row("m-1", model="openai/gpt-after-update"),) + await targeted_load + + installed: Final = router.registry().deployment("m-1") + assert installed is not None + assert installed.litellm_params.model == "openai/gpt-after-update" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +@pytest.mark.parametrize( + ("stored_credential_name", "credentials_on_this_worker", "api_key_this_worker_uses"), + [ + ("shared-openai", (), "sk-from-db"), + ("shared-openai", ("another-openai",), "sk-from-db"), + ("shared-openai", ("shared-openai",), "sk-on-this-worker"), + (None, (), None), + ], +) +async def test_resync_model_deployments_by_ids_reloads_credentials_only_for_one_this_worker_lacks( + monkeypatch: pytest.MonkeyPatch, + stored_credential_name: str | None, + credentials_on_this_worker: tuple[str, ...], + api_key_this_worker_uses: str | None, +) -> None: + import litellm + from litellm.litellm_core_utils.credential_accessor import CredentialAccessor + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + from litellm.types.utils import CredentialItem + + model_params: Final = {"model": "openai/gpt-backup"} + row: Final = _stored_model_row( + "backup-id", + model_params if stored_credential_name is None else model_params | {"litellm_credential_name": stored_credential_name}, + ) + stored_credential: Final = CredentialItem( + credential_name="shared-openai", + credential_values={"api_key": encrypt_value_helper("sk-from-db")}, + credential_info={}, + ) + router: Final = _router_serving("primary-id") + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem(credential_name=name, credential_values={"api_key": "sk-on-this-worker"}, credential_info={}) + for name in credentials_on_this_worker + ], + ) + _serve_db_models(monkeypatch, _Db(_ProxyModelTable((row,)), _CredentialsTable((stored_credential,))), router) + + await resync_model_deployments_by_ids(("backup-id",)) + + credential: Final = CredentialAccessor.find_credential("shared-openai") + assert (credential.credential_values.get("api_key") if credential else None) == api_key_this_worker_uses + assert router.registry().has_deployment("backup-id") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_installs_the_ordinary_rows_queued_with_an_auto_router( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + table: Final = _ProxyModelTable( + ( + _stored_model_row("auto-router-id", {"model": "auto_router/complexity-router"}), + _backup_row("backup-id"), + ) + ) + db: Final = _Db(table) + router: Final = _router_serving("primary-id") + _serve_db_models(monkeypatch, db, router) + + await resync_model_deployments_by_ids(("auto-router-id", "backup-id")) + + assert _served_ids(router.registry().deployments()) == {"primary-id", "backup-id"} + assert _served_ids(proxy_server.llm_model_list) == {"primary-id", "backup-id"} + assert table.reads == 1 + assert db.other_tables_read == [] + assert not proxy_server.MODEL_RECONCILE_LOCK.locked() diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index 47dd6d561b3..3d8e92988d9 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -3277,6 +3277,23 @@ class TestCredentialDisplayName: assert listed["openai-prod"]["credential_values"]["api_key"] != "sk-old" +def test_looking_up_a_missing_credential_is_labelled_a_client_error_not_a_gateway_failure( + restore_credential_list: None, +): + """Naming a credential that does not exist is the caller's own mistake, and the shared + handler used to label every one of these internal_server_error, so an SDK branching on + the type read a 404 it can never retry past as a gateway outage and backed off forever.""" + response: Final = _call_as("GET", "/credentials/by_name/definitely-not-there") + + assert response.status_code == 404, f"lookup answered {response.status_code}: {response.text}" + assert response.json()["error"] == { + "message": "Credential not found. Got credential name: definitely-not-there", + "type": "invalid_request_error", + "param": None, + "code": "404", + } + + def test_get_credential_by_model_masks_the_deployment_credentials(monkeypatch): import asyncio diff --git a/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py index 24c12e06c9c..65da5630f33 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py @@ -1,10 +1,12 @@ import json +import logging from datetime import datetime, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm._logging import verbose_proxy_logger from litellm.constants import REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import DailyTagSpendTransaction @@ -379,6 +381,104 @@ async def test_org_member_spend_is_summed_across_pods_and_restored_on_rpush_fail assert restored_spend["org_member_list_transactions"] == {member_key: 1.5} +@pytest.mark.asyncio +async def test_a_single_parked_batch_with_a_nan_spend_drains_as_zero( + redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock +): + mock_redis_cache.async_lpop = AsyncMock( + return_value=json.dumps({"key_list_transactions": {"nan-key": float("nan"), "ok-key": 0.5}}, allow_nan=True) + ) + + drained: Final = await redis_update_buffer.get_all_update_transactions_from_redis_buffer() + + assert drained is not None + assert drained["key_list_transactions"] == {"nan-key": 0.0, "ok-key": 0.5} + + +PARKED_DAILY_KEY: Final = "user-1_2025-01-15_api-key-123_gpt-4o_openai_/chat/completions" + + +def _parked_daily_batch(spend: float) -> str: + return json.dumps( + { + PARKED_DAILY_KEY: { + "spend": spend, + "prompt_tokens": 12, + "completion_tokens": 7, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + }, + allow_nan=True, + ) + + +@pytest.mark.asyncio +async def test_a_nan_spend_parked_by_an_older_version_drains_from_the_pipeline_as_zero( + redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock +): + entity_batch: Final = json.dumps({"key_list_transactions": {"hashed-key": float("nan")}}, allow_nan=True) + window_batches: Final = tuple( + json.dumps( + [ + { + "entity_type": "key", + "entity_id": "hashed-key", + "window_duration": "30d", + "window_start": "2026-08-01T00:00:00.000000", + "spend": spend, + "started_at": None, + } + ], + allow_nan=True, + ) + for spend in (float("nan"), 3.0) + ) + mock_redis_cache.async_lpop_pipeline = AsyncMock( + return_value=[ + [entity_batch], + [_parked_daily_batch(float("nan")), _parked_daily_batch(0.0042)], + None, + None, + None, + None, + list(window_batches), + ] + ) + + ( + db_spend, + daily_user, + *_rest, + window_spend, + ) = await redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline() + + assert db_spend is not None + assert db_spend["key_list_transactions"] == {"hashed-key": 0} + assert daily_user is not None + assert daily_user[PARKED_DAILY_KEY]["spend"] == 0.0042 + assert daily_user[PARKED_DAILY_KEY]["api_requests"] == 2 + assert window_spend is not None + assert [transaction["spend"] for transaction in window_spend] == [3.0] + + +@pytest.mark.asyncio +async def test_a_nan_spend_parked_by_an_older_version_drains_from_a_daily_key_as_zero( + redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock, caplog: pytest.LogCaptureFixture +): + mock_redis_cache.async_lpop = AsyncMock( + return_value=[_parked_daily_batch(float("inf")), _parked_daily_batch(0.0042)] + ) + + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + daily_tag: Final = await redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() + + assert daily_tag is not None + assert daily_tag[PARKED_DAILY_KEY]["spend"] == 0.0042 + assert [record.getMessage() for record in caplog.records if "Infinity" in record.getMessage()] != [] + + @pytest.mark.asyncio async def test_get_all_transactions_from_redis_buffer_pipeline_no_redis(): """When redis_cache is None, should return all Nones""" @@ -724,6 +824,10 @@ class _ListRedis: def __init__(self) -> None: self.rows: list[str] = [] + async def async_rpush(self, key: str, values: list[str]) -> int: + self.rows.extend(values) + return len(self.rows) + async def async_rpush_and_trim(self, key: str, values: list[str], max_len: int) -> int: self.rows.extend(values) pushed_len = len(self.rows) @@ -752,6 +856,32 @@ async def test_store_spend_logs_in_redis_drops_oldest_rows_past_the_cap(): assert await buffer.get_spend_logs_from_redis_buffer(limit=10) == () +@pytest.mark.asyncio +@pytest.mark.parametrize("spend", [float("nan"), float("inf")]) +async def test_a_non_finite_daily_spend_buffered_in_redis_drains_as_zero(spend: float): + buffer: Final = RedisUpdateBuffer(redis_cache=_ListRedis()) + queue: Final = DailySpendUpdateQueue() + await queue.add_update(json.loads(_parked_daily_batch(spend))) + await queue.add_update(json.loads(_parked_daily_batch(0.0042))) + + await buffer.store_in_memory_daily_tag_spend_updates_in_redis(queue) + drained: Final = await buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() + + assert drained is not None + assert drained[PARKED_DAILY_KEY]["spend"] == 0.0 + assert drained[PARKED_DAILY_KEY]["api_requests"] == 2 + + +@pytest.mark.asyncio +async def test_a_spend_log_row_parked_with_a_nan_spend_is_read_back_with_zero_spend(): + buffer: Final = RedisUpdateBuffer(redis_cache=_ListRedis()) + buffer.should_commit_spend_updates_to_redis = MagicMock(return_value=True) + + assert await buffer.store_spend_logs_in_redis([{"request_id": "nan-row", "spend": float("nan")}]) is True + + assert await buffer.get_spend_logs_from_redis_buffer(limit=10) == ({"request_id": "nan-row", "spend": 0.0},) + + @pytest.mark.asyncio async def test_store_spend_logs_in_redis_reports_failure_without_redis(): buffer = RedisUpdateBuffer(redis_cache=None) diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index c161aa5fe50..298b153f621 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -6,6 +6,7 @@ import re from collections.abc import AsyncIterator, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager from datetime import datetime, timedelta, timezone +from pathlib import Path from types import SimpleNamespace from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, call, patch @@ -2225,6 +2226,60 @@ async def test_shutdown_drain_finishes_the_rollups_update_database_started(): assert (team_row["api_requests"], team_row["spend"]) == (1, 0.1) +@pytest.mark.asyncio +@pytest.mark.parametrize("response_cost", [float("nan"), float("inf"), float("-inf")]) +async def test_update_database_records_a_non_finite_response_cost_as_zero(response_cost: float) -> None: + db_writer: Final = DBSpendUpdateWriter() + prisma: Final = _tool_usage_prisma() + prisma.get_request_status = MagicMock(return_value="success") + response: Final = litellm.ModelResponse( + model="gpt-5.6-luna", + usage=litellm.Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + with ( + patch("litellm.proxy.proxy_server.disable_spend_logs", False), + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), + patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), + ): + await db_writer.update_database( + token="test-token", + user_id="test-user", + end_user_id=None, + team_id="test-team", + org_id=None, + kwargs={ + "model": "gpt-5.6-luna", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "req-non-finite-cost", + "response_cost": response_cost, + "litellm_params": {"metadata": {"user_api_key": "hashed-key", "user_api_key_team_id": "test-team"}}, + }, + completion_response=response, + start_time=datetime.now(timezone.utc), + end_time=datetime.now(timezone.utc), + response_cost=response_cost, + ) + await drain_logging_tasks() + + assert [row["spend"] for row in prisma.spend_log_transactions] == [0] + spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() + assert spend["key_list_transactions"] == {"test-token": 0} + assert spend["team_list_transactions"] == {"test-team": 0} + daily_user: Final = ( + await db_writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) + daily_team: Final = ( + await db_writer.daily_team_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) + assert daily_user is not None and daily_team is not None + (user_row,) = daily_user.values() + (team_row,) = daily_team.values() + assert (user_row["api_requests"], user_row["spend"]) == (1, 0) + assert (team_row["api_requests"], team_row["spend"]) == (1, 0) + + @pytest.mark.asyncio async def test_shutdown_drain_waits_for_an_in_flight_rollup_enqueue(): """ @@ -4325,6 +4380,36 @@ async def test_insert_spend_log_asks_for_an_immediate_flush_only_on_rows_the_bri assert prisma.spend_log_flush_requested.is_set() is expects_flush + +@pytest.mark.asyncio +@pytest.mark.timeout(30) +async def test_a_chatgpt_responses_row_queues_without_a_default_chatgpt_login( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "no-default-login")) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + own_login: Final = tmp_path / "team" / "auth.json" + own_login.parent.mkdir() + own_login.write_text(json.dumps({"access_token": "at-team", "account_id": "acct-team"})) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "chatgpt-team", + "litellm_params": {"model": "chatgpt/gpt-5.6-terra", "chatgpt_auth_file": str(own_login)}, + "model_info": {"id": "dep-chatgpt"}, + } + ] + ) + prisma: Final = _tool_usage_prisma() + prisma.spend_log_flush_requested = asyncio.Event() + payload: Final = {"request_id": "req-1", "call_type": "aresponses", "model_id": "dep-chatgpt"} + + await DBSpendUpdateWriter()._insert_spend_log_to_db(payload=payload, prisma_client=prisma, llm_router=router) + + assert prisma.spend_log_transactions == [payload] + assert prisma.spend_log_flush_requested.is_set() is False + + def _batch_cost_payload() -> dict: return { **_minimal_spend_payload(), diff --git a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py index 6807a4fb569..7d912478877 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py @@ -13,7 +13,7 @@ import yaml from fastapi import HTTPException from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_endpoints import ( CreateGuardrailRequest, ListGuardrailsResponse, @@ -2198,6 +2198,39 @@ def _patch_apply_guardrail_env(mocker, guardrail_result): return mock_guardrail +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("vendor_status", "expected_type"), + [(401, "authentication_error"), (403, "permission_error")], +) +async def test_a_guardrail_vendors_rejection_is_labelled_the_way_chat_labels_it( + mocker, vendor_status: int, expected_type: str +): + """A vendor refusing the guardrail's own credentials is not the caller's key being rejected, + so it must not read as the proxy's auth_error the way a 401 the proxy raises does.""" + mock_guardrail: Final = _patch_apply_guardrail_env(mocker, None) + mock_guardrail.apply_guardrail.side_effect = HTTPException( + status_code=vendor_status, detail="vendor refused the request" + ) + mocker.patch( + "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", AsyncMock(return_value=None) + ) + + with pytest.raises(ProxyException) as raised: + await apply_guardrail( + fastapi_request=mocker.Mock(), + request=ApplyGuardrailRequest(guardrail_name="test-guardrail", text="hello"), + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert raised.value.to_dict() == { + "message": "vendor refused the request", + "type": expected_type, + "param": None, + "code": str(vendor_status), + } + + @pytest.mark.asyncio async def test_apply_guardrail_forwards_metadata_to_guardrail(mocker): """Client-supplied metadata must reach apply_guardrail via request_data so diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 9847787efce..2258eb95fb8 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -1,8 +1,9 @@ import asyncio import copy +import itertools import json import time -from collections.abc import Awaitable, Iterator, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from contextlib import contextmanager from datetime import datetime, timedelta from importlib import import_module @@ -22,7 +23,7 @@ import litellm import litellm.proxy.health_endpoints._health_endpoints as _health_endpoints_module from litellm.litellm_core_utils.health_check_helpers import TEST_IMAGE_BASE64 from litellm.models.credentials import CredentialItem -from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.health_endpoints._health_endpoints import ( _db_health_readiness_check, @@ -35,6 +36,7 @@ from litellm.proxy.health_endpoints._health_endpoints import ( test_model_connection as health_test_model_connection, ) from litellm.router import Router +from litellm.types.router import RouterModelGroupAliasItem from litellm.types.utils import oauth_token_exchange_litellm_params from litellm.types.workload_identity import ( ANTHROPIC_WIF_KWARGS_KEYS, @@ -925,7 +927,11 @@ async def test_test_model_connection_stored_audio_base_model_follows_the_stored_ deployment: Final = MappingProxyType( { "model_name": "custom-name-audio", - "litellm_params": {**deployment_params, "api_key": "fake-azure-key", "api_base": "https://x.openai.azure.com"}, + "litellm_params": { + **deployment_params, + "api_key": "fake-azure-key", + "api_base": "https://x.openai.azure.com", + }, "model_info": deployment_model_info, } ) @@ -1948,7 +1954,6 @@ def test_health_liveness_endpoint(proxy_client): assert duration_ms < 100, f"Health check took {duration_ms:.2f}ms, expected < 100ms for a simple endpoint" - def test_proxy_client_serves_full_proxy_routes_after_gateway_import(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy.proxy_server import app @@ -2472,6 +2477,7 @@ def _proxy_health_globals( llm_router: object, use_background_health_checks: bool = False, health_check_results: Mapping[str, object] | None = None, + prisma_client: object = None, ) -> Iterator[None]: with ( patch( # test-quality-ok: proxy module global, no injection seam @@ -2481,7 +2487,7 @@ def _proxy_health_globals( "litellm.proxy.proxy_server.llm_router", llm_router ), patch( # test-quality-ok: proxy module global, no injection seam - "litellm.proxy.proxy_server.prisma_client", None + "litellm.proxy.proxy_server.prisma_client", prisma_client ), patch( # test-quality-ok: proxy module global, no injection seam "litellm.proxy.proxy_server.use_background_health_checks", use_background_health_checks @@ -3812,7 +3818,11 @@ def test_clean_endpoint_data_never_displays_credential_fields(credential_field, async def _live_probed_model_ids( - model_list: Sequence[Mapping[str, object]], user_api_key_dict: UserAPIKeyAuth, model: str | None = None + model_list: Sequence[Mapping[str, object]], + user_api_key_dict: UserAPIKeyAuth, + model: str | None = None, + llm_router: Router | None = None, + prisma_client: object = None, ) -> set[str]: from fastapi import Response @@ -3825,7 +3835,7 @@ async def _live_probed_model_ids( return {"healthy_endpoints": [], "unhealthy_endpoints": [], "healthy_count": 0, "unhealthy_count": 0} with ( - _proxy_health_globals(model_list, _router_for(model_list)), + _proxy_health_globals(model_list, llm_router or _router_for(model_list), prisma_client=prisma_client), patch( # test-quality-ok: the model list handed to the probe is the assertion; no injection seam "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", side_effect=fake_perform, @@ -4120,6 +4130,7 @@ async def _live_narrowed_model_ids( user_api_key_dict: UserAPIKeyAuth, model: str | None = None, model_id: str | None = None, + llm_router: Router | None = None, ) -> set[str]: from fastapi import Response @@ -4130,7 +4141,7 @@ async def _live_narrowed_model_ids( return probed, [], {} with ( - _proxy_health_globals(model_list, _router_for(model_list)), + _proxy_health_globals(model_list, llm_router or _router_for(model_list)), patch( # test-quality-ok: the probe is the provider edge; which deployments reach it is the assertion "litellm.proxy.health_check._perform_health_check", side_effect=fake_probe ), @@ -4289,6 +4300,699 @@ async def test_health_endpoint_keeps_a_team_only_public_name_off_a_team_less_key probe.assert_not_awaited() +_PROVIDER_PREFIXED_MODEL_LIST: Final = [ + { + "model_name": "openai/gpt-5.4-nano", + "litellm_params": {"model": "openai/gpt-5.4-nano"}, + "model_info": {"id": "id-prefixed-nano"}, + }, + { + "model_name": "gpt-5.4-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-mini"}, + }, + { + "model_name": "claude-haiku-5-5", + "litellm_params": {"model": "anthropic/claude-haiku-5-5"}, + "model_info": {"id": "id-haiku"}, + }, +] +_PROVIDER_PREFIXED_CACHED_RESULTS: Final = { + "healthy_endpoints": [ + {"model": "openai/gpt-5.4-nano", "model_id": "id-prefixed-nano"}, + {"model": "openai/gpt-5.4-mini", "model_id": "id-mini"}, + {"model": "anthropic/claude-haiku-5-5", "model_id": "id-haiku"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 3, + "unhealthy_count": 0, +} +_PROVIDER_WILDCARD_KEY: Final = UserAPIKeyAuth(api_key="hashed-test-key", models=["openai/*"]) +_RESTRICTED_TEAM_KEY: Final = UserAPIKeyAuth( + api_key="hashed-test-key", models=[], team_id="team-a", team_models=["gpt-5.4-mini"] +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("caller", "expected_ids"), + [(_PROVIDER_WILDCARD_KEY, {"id-prefixed-nano", "id-mini"}), (_RESTRICTED_TEAM_KEY, {"id-mini"})], +) +async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request( + caller: UserAPIKeyAuth, expected_ids: set[str] +): + probed: Final = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller) + + assert probed == expected_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("caller", "expected_ids"), + [(_PROVIDER_WILDCARD_KEY, ["id-prefixed-nano", "id-mini"]), (_RESTRICTED_TEAM_KEY, ["id-mini"])], +) +async def test_health_endpoint_returns_the_deployments_auth_lets_the_key_request_on_background_cache_path( + caller: UserAPIKeyAuth, expected_ids: list[str] +): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, + _router_for(_PROVIDER_PREFIXED_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_PROVIDER_PREFIXED_CACHED_RESULTS, + ): + result: Final = await health_endpoint(response=Response(), user_api_key_dict=caller, model=None, model_id=None) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == expected_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +async def test_health_endpoint_refuses_a_deployment_the_team_allowlist_forbids(use_background_health_checks: bool): + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, + _router_for(_PROVIDER_PREFIXED_MODEL_LIST), + use_background_health_checks=use_background_health_checks, + health_check_results=_PROVIDER_PREFIXED_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), user_api_key_dict=_RESTRICTED_TEAM_KEY, model=None, model_id="id-prefixed-nano" + ) + + assert refused.value.status_code == 403 + probe.assert_not_awaited() + + +_KEY_ALIASED_AWAY_FROM_HAIKU: Final = UserAPIKeyAuth( + api_key="hashed-test-key", models=["gpt-5.4-mini"], aliases={"claude-haiku-5-5": "gpt-5.4-mini"} +) +_TEAM_ALIASED_AWAY_FROM_HAIKU: Final = UserAPIKeyAuth( + api_key="hashed-test-key", + models=["claude-haiku-5-5"], + team_id="team-a", + team_model_aliases={"claude-haiku-5-5": "gpt-5.4-mini"}, +) +_MINI_ONLY_KEY: Final = UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-mini"]) +_ALIAS_SHADOW_CASES: Final = [ + pytest.param(_KEY_ALIASED_AWAY_FROM_HAIKU, {}, id="key-alias"), + pytest.param(_TEAM_ALIASED_AWAY_FROM_HAIKU, {}, id="team-alias"), + pytest.param(_MINI_ONLY_KEY, {"claude-haiku-5-5": "gpt-5.4-mini"}, id="router-alias"), +] + + +def _router_with_aliases(model_group_alias: Mapping[str, str]) -> Router: + return Router(model_list=copy.deepcopy(_PROVIDER_PREFIXED_MODEL_LIST), model_group_alias=dict(model_group_alias)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) +async def test_health_endpoint_probes_the_deployment_an_alias_sends_the_key_to_not_the_one_it_shadows( + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] +): + probed: Final = await _live_probed_model_ids( + _PROVIDER_PREFIXED_MODEL_LIST, caller, llm_router=_router_with_aliases(model_group_alias) + ) + + assert probed == {"id-mini"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) +async def test_health_endpoint_returns_the_deployment_an_alias_sends_the_key_to_on_background_cache_path( + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] +): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, + _router_with_aliases(model_group_alias), + use_background_health_checks=True, + health_check_results=_PROVIDER_PREFIXED_CACHED_RESULTS, + ): + result: Final = await health_endpoint(response=Response(), user_api_key_dict=caller, model=None, model_id=None) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-mini"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +async def test_health_endpoint_refuses_the_deployment_a_key_alias_shadows(use_background_health_checks: bool): + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, + _router_for(_PROVIDER_PREFIXED_MODEL_LIST), + use_background_health_checks=use_background_health_checks, + health_check_results=_PROVIDER_PREFIXED_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), user_api_key_dict=_KEY_ALIASED_AWAY_FROM_HAIKU, model=None, model_id="id-haiku" + ) + + assert refused.value.status_code == 403 + probe.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) +async def test_health_endpoint_targets_the_deployment_an_alias_name_sends_the_key_to( + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] +): + probed: Final = await _live_narrowed_model_ids( + _PROVIDER_PREFIXED_MODEL_LIST, + caller, + model="claude-haiku-5-5", + llm_router=_router_with_aliases(model_group_alias), + ) + + assert probed == {"id-mini"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) +async def test_health_endpoint_records_an_alias_check_under_the_name_the_caller_asked_for( + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] +): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + async def fake_probe(model_list, details=True, max_concurrency=None, instrumentation_context=None): + return [{"model": m["litellm_params"]["model"], "model_id": m["model_info"]["id"]} for m in model_list], [], {} + + with ( + _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, _router_with_aliases(model_group_alias), prisma_client=MagicMock() + ), + patch( # test-quality-ok: the probe is the provider edge; the endpoint has no injection seam for it + "litellm.proxy.health_check._perform_health_check", side_effect=fake_probe + ), + patch( # test-quality-ok: the history row is the assertion; the endpoint has no injection seam for its writer + "litellm.proxy.health_endpoints._health_endpoints._save_health_check_to_db", new_callable=AsyncMock + ) as save, + ): + result: Final = await health_endpoint( + response=Response(), user_api_key_dict=caller, model="claude-haiku-5-5", model_id=None + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-mini"] + assert save.call_args.args[1] == "claude-haiku-5-5" + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) +async def test_health_endpoint_targets_the_deployment_an_alias_name_sends_the_key_to_on_background_cache_path( + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] +): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, + _router_with_aliases(model_group_alias), + use_background_health_checks=True, + health_check_results=_PROVIDER_PREFIXED_CACHED_RESULTS, + ): + result: Final = await health_endpoint( + response=Response(), user_api_key_dict=caller, model="claude-haiku-5-5", model_id=None + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-mini"] + + +@pytest.mark.asyncio +async def test_health_endpoint_never_probes_a_deployment_without_a_model_name_for_a_restricted_key(): + nameless_deployment: Final = { + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-nameless"}, + } + + probed: Final = await _live_probed_model_ids( + [*_PROVIDER_PREFIXED_MODEL_LIST, nameless_deployment], + _MINI_ONLY_KEY, + llm_router=_router_for(_PROVIDER_PREFIXED_MODEL_LIST), + ) + + assert probed == {"id-mini"} + + +@pytest.mark.asyncio +async def test_health_endpoint_never_probes_the_deployment_a_keys_router_alias_sends_it_to_but_forbids(): + caller: Final = UserAPIKeyAuth( + api_key="hashed-test-key", + models=["gpt-5.4-mini", "openai/gpt-5.4-nano"], + router_settings={"model_group_alias": {"gpt-5.4-mini": "claude-haiku-5-5"}}, + ) + + probed: Final = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller) + + assert probed == {"id-prefixed-nano"} + + +_WILDCARD_DEPLOYMENT: Final = { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "id-wild"}, +} +_CATCH_ALL_DEPLOYMENT: Final = {"model_name": "*", "litellm_params": {"model": "*"}, "model_info": {"id": "id-star"}} +_TEAM_WILDCARD_DEPLOYMENT: Final = { + "model_name": "openai-wild_team-a_7c1e", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "id-team-wild", "team_id": "team-a", "team_public_model_name": "openai/*"}, +} + + +def _team_key(team_models: list[str], team_model_aliases: Mapping[str, str] | None = None) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="hashed-test-key", + models=[], + team_id="team-a", + team_models=team_models, + team_model_aliases=dict(team_model_aliases) if team_model_aliases is not None else None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_list", "model_group_alias", "caller", "expected_ids"), + [ + pytest.param( + [_WILDCARD_DEPLOYMENT, _PROVIDER_PREFIXED_MODEL_LIST[2]], + {}, + _team_key(["openai/gpt-5.4-nano"]), + {"id-wild"}, + id="team-name-served-by-provider-wildcard", + ), + pytest.param( + [_TEAM_WILDCARD_DEPLOYMENT, _PROVIDER_PREFIXED_MODEL_LIST[2]], + {}, + _team_key(["openai/gpt-5.4-nano"]), + {"id-team-wild"}, + id="team-name-served-by-teams-own-wildcard", + ), + pytest.param( + [_CATCH_ALL_DEPLOYMENT], + {}, + _team_key(["gpt-5.4-mini"]), + {"id-star"}, + id="team-name-served-by-catch-all", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {"gpt-4": "gpt-5.4-mini"}, + _team_key(["gpt-4"]), + {"id-mini"}, + id="team-allows-router-alias", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {}, + _team_key(["my-alias"], {"my-alias": "gpt-5.4-mini"}), + {"id-mini"}, + id="team-allows-team-alias", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {"friendly-mini": RouterModelGroupAliasItem(model="gpt-5.4-mini", hidden=True)}, + UserAPIKeyAuth(api_key="hashed-test-key", models=["friendly-*"]), + {"id-mini"}, + id="wildcard-key-allows-hidden-router-alias", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {}, + _team_key(["friendly-*"], {"friendly-mini": "gpt-5.4-mini"}), + {"id-mini"}, + id="wildcard-team-allows-team-alias", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {"claude-haiku-5-5": "gpt-5.4-mini"}, + UserAPIKeyAuth(api_key="hashed-test-key", models=[]), + {"id-prefixed-nano", "id-mini", "id-haiku"}, + id="unrestricted-key-calls-shadowed-deployment-by-id", + ), + ], +) +async def test_health_endpoint_probes_every_deployment_the_callers_allowed_names_route_to( + model_list: Sequence[Mapping[str, object]], + model_group_alias: Mapping[str, str | RouterModelGroupAliasItem], + caller: UserAPIKeyAuth, + expected_ids: set[str], +): + llm_router: Final = Router( + model_list=copy.deepcopy([dict(deployment) for deployment in model_list]), + model_group_alias=dict(model_group_alias), + ) + + probed: Final = await _live_probed_model_ids(llm_router.get_model_list() or [], caller, llm_router=llm_router) + + assert probed == expected_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "caller", + [ + pytest.param( + UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-mini", "gpt-4-retired"]), id="key-grant" + ), + pytest.param(_team_key(["gpt-5.4-mini", "gpt-4-retired"]), id="team-grant"), + ], +) +async def test_health_endpoint_never_probes_the_default_fallback_for_a_granted_name_no_deployment_serves( + caller: UserAPIKeyAuth, +) -> None: + llm_router: Final = Router( + model_list=copy.deepcopy(_PROVIDER_PREFIXED_MODEL_LIST), default_fallbacks=["claude-haiku-5-5"] + ) + + probed: Final = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller, llm_router=llm_router) + + assert probed == {"id-mini"} + + +def _personal_key_whose_owner_may_call(models: list[str]) -> UserAPIKeyAuth: + from litellm.proxy.proxy_server import user_api_key_cache + + owner_id: Final = "health-scope-owner-" + "-".join(models) + user_api_key_cache.set_cache(key=owner_id, value=LiteLLM_UserTable(user_id=owner_id, models=models)) + return UserAPIKeyAuth(api_key="hashed-test-key", models=[], user_id=owner_id) + + +def _managed_agent_key_granted(models: list[str]) -> UserAPIKeyAuth: + from litellm.types.agents import AgentResponse + + policy: Final = AgentResponse( + agent_id="managed", agent_name="Managed", agent_card_params={}, object_permission={"models": models} + ) + return UserAPIKeyAuth(api_key="hashed-test-key", models=[]).model_copy(update={"managed_agent_policy": policy}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("make_caller", "models", "expected_ids"), + [ + pytest.param(_personal_key_whose_owner_may_call, ["gpt-5.4-mini"], {"id-mini"}, id="key-owners-models"), + pytest.param(_managed_agent_key_granted, ["gpt-5.4-mini"], {"id-mini"}, id="managed-agents-models"), + pytest.param(_managed_agent_key_granted, [], set(), id="managed-agent-without-model-grants"), + ], +) +async def test_health_endpoint_probes_only_the_models_the_keys_user_or_agent_may_call( + make_caller: Callable[[list[str]], UserAPIKeyAuth], models: list[str], expected_ids: set[str] +): + probed: Final = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, make_caller(models)) + + assert probed == expected_ids + + +@pytest.mark.asyncio +async def test_health_endpoint_refuses_a_model_id_the_keys_owner_may_not_call(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals(_PROVIDER_PREFIXED_MODEL_LIST, _router_for(_PROVIDER_PREFIXED_MODEL_LIST)), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=_personal_key_whose_owner_may_call(["gpt-5.4-mini"]), + model=None, + model_id="id-prefixed-nano", + ) + + assert refused.value.status_code == 403 + probe.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "make_caller", + [ + pytest.param(_personal_key_whose_owner_may_call, id="key-owner"), + pytest.param(_managed_agent_key_granted, id="managed-agent"), + ], +) +async def test_health_endpoint_keeps_the_wildcard_serving_a_name_the_keys_user_or_agent_may_call( + make_caller: Callable[[list[str]], UserAPIKeyAuth], +): + model_list: Final = [_WILDCARD_DEPLOYMENT, *_PROVIDER_PREFIXED_MODEL_LIST] + + probed: Final = await _live_probed_model_ids(model_list, make_caller(["openai/gpt-5.6"])) + + assert probed == {"id-wild"} + + +@pytest.mark.asyncio +async def test_health_endpoint_answers_an_agent_key_acting_for_an_unknown_user_the_way_chat_does(): + from litellm.types.agents import AgentCaller + + users_table: Final = MagicMock() + users_table.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + agent_key: Final = UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-mini"]).model_copy( + update={"agent_caller": AgentCaller(user_id="user-deleted-before-this-request")} + ) + + with pytest.raises(ProxyException) as failed: + await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, agent_key, prisma_client=users_table) + + assert (failed.value.code, failed.value.type) == ("401", ProxyErrorTypes.auth_error) + assert "user-deleted-before-this-request" in failed.value.message + + +@pytest.mark.asyncio +async def test_health_endpoint_lets_other_requests_run_while_it_scopes_a_large_model_list(): + model_list: Final = [ + {"model_name": f"model-{n}", "litellm_params": {"model": f"openai/model-{n}"}, "model_info": {"id": f"id-{n}"}} + for n in range(50) + ] + other_request_turns: Final = itertools.count() + scoping_done: Final = asyncio.Event() + + async def scope() -> set[str]: + try: + return await _live_probed_model_ids( + model_list, UserAPIKeyAuth(api_key="hashed-test-key", models=["model-0"]) + ) + finally: + scoping_done.set() + + async def other_request() -> None: + while not scoping_done.is_set(): + next(other_request_turns) + await asyncio.sleep(0) + + probed, _ = await asyncio.gather(scope(), other_request()) + + assert probed == {"id-0"} + assert next(other_request_turns) >= len(model_list) + + +_WILDCARD_CACHED_RESULTS: Final = { + "healthy_endpoints": [ + {"model": "openai/gpt-5.6", "model_id": "id-wild"}, + {"model": "openai/gpt-5.6", "model_id": "id-team-wild"}, + {"model": "anthropic/claude-haiku-5-5", "model_id": "id-haiku"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 3, + "unhealthy_count": 0, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +@pytest.mark.parametrize( + ("wildcard_deployment", "expected_id"), + [ + pytest.param(_WILDCARD_DEPLOYMENT, "id-wild", id="proxy-wildcard"), + pytest.param(_TEAM_WILDCARD_DEPLOYMENT, "id-team-wild", id="teams-own-wildcard"), + ], +) +async def test_health_endpoint_model_param_reaches_the_wildcard_serving_a_team_allowed_name( + wildcard_deployment: Mapping[str, object], expected_id: str, use_background_health_checks: bool +): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + model_list: Final = [wildcard_deployment, _PROVIDER_PREFIXED_MODEL_LIST[2]] + + async def fake_probe(model_list, details=True, max_concurrency=None, instrumentation_context=None): + return [{"model": m["litellm_params"]["model"], "model_id": m["model_info"]["id"]} for m in model_list], [], {} + + with ( + _proxy_health_globals( + model_list, + _router_for(model_list), + use_background_health_checks=use_background_health_checks, + health_check_results=_WILDCARD_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe is the provider edge; which deployments reach it is the assertion + "litellm.proxy.health_check._perform_health_check", side_effect=fake_probe + ), + ): + result: Final = await health_endpoint( + response=Response(), user_api_key_dict=_team_key(["openai/gpt-5.6"]), model="openai/gpt-5.6", model_id=None + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == [expected_id] + + +_ACCESS_GROUP_SPLIT_MODEL_LIST: Final = [ + { + "model_name": "gpt-5.4-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-mini-premium", "access_groups": ["premium"]}, + }, + { + "model_name": "gpt-5.4-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-mini-standard"}, + }, +] + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_only_the_access_groups_deployments_of_a_shared_name() -> None: + caller: Final = UserAPIKeyAuth(api_key="hashed-test-key", models=["premium"]) + + probed: Final = await _live_probed_model_ids(_ACCESS_GROUP_SPLIT_MODEL_LIST, caller) + + assert probed == {"id-mini-premium"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("direct_grant", ["gpt-5.4-mini", "*", "all-proxy-models"]) +async def test_health_endpoint_probes_every_deployment_of_a_name_the_key_also_holds_directly(direct_grant: str) -> None: + caller: Final = UserAPIKeyAuth(api_key="hashed-test-key", models=["premium", direct_grant]) + + probed: Final = await _live_probed_model_ids(_ACCESS_GROUP_SPLIT_MODEL_LIST, caller) + + assert probed == {"id-mini-premium", "id-mini-standard"} + + +_ACCESS_GROUP_SPLIT_WILDCARD_MODEL_LIST: Final = [ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "id-wild-premium", "access_groups": ["premium"]}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "id-wild-standard", "access_groups": ["standard"]}, + }, +] + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_only_the_access_groups_wildcard_deployments_serving_a_name() -> None: + caller: Final = _personal_key_whose_owner_may_call(["openai/gpt-5.4-nano"]).model_copy( + update={"models": ["premium"]} + ) + + probed: Final = await _live_probed_model_ids(_ACCESS_GROUP_SPLIT_WILDCARD_MODEL_LIST, caller) + + assert probed == {"id-wild-premium"} + + +def _team_deployment( + internal_name: str, public_name: str, model: str, model_id: str, access_groups: list[str] +) -> Mapping[str, object]: + return { + "model_name": internal_name, + "litellm_params": {"model": model}, + "model_info": { + "id": model_id, + "team_id": "team-a", + "team_public_model_name": public_name, + "access_groups": access_groups, + }, + } + + +def _team_split_model_list(public_name: str, model: str) -> list[Mapping[str, object]]: + return [ + _team_deployment("model_name_team-a_7c1e4b2a", public_name, model, "id-team-premium", ["premium"]), + _team_deployment("model_name_team-a_9d2f6e01", public_name, model, "id-team-standard", ["standard"]), + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("public_name", "model"), + [ + pytest.param("openai/*", "openai/*", id="team-wildcard"), + pytest.param("gpt-5.4-mini", "openai/gpt-5.4-mini", id="team-model"), + ], +) +async def test_health_endpoint_probes_only_the_access_groups_team_deployments_serving_a_name( + public_name: str, model: str +) -> None: + probed: Final = await _live_probed_model_ids(_team_split_model_list(public_name, model), _team_key(["premium"])) + + assert probed == {"id-team-premium"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("internal_name_grant", ["model_name_team-a_9d2f6e01", "model_name_team-a_*"]) +async def test_health_endpoint_probes_a_team_deployment_whose_internal_name_the_key_holds( + internal_name_grant: str, +) -> None: + caller: Final = _team_key(["premium", internal_name_grant]) + + probed: Final = await _live_probed_model_ids(_team_split_model_list("gpt-5.4-mini", "openai/gpt-5.4-mini"), caller) + + assert probed == {"id-team-premium", "id-team-standard"} + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_only_the_team_deployment_whose_internal_name_an_agent_holds() -> None: + from litellm.types.agents import AgentResponse + + policy: Final = AgentResponse( + agent_id="managed", + agent_name="Managed", + agent_card_params={}, + object_permission={"models": ["model_name_team-a_9d2f6e01"]}, + ) + caller: Final = _team_key(["gpt-5.4-mini"]).model_copy(update={"managed_agent_policy": policy}) + + probed: Final = await _live_probed_model_ids(_team_split_model_list("gpt-5.4-mini", "openai/gpt-5.4-mini"), caller) + + assert probed == {"id-team-standard"} + + def test_resolve_targeted_model_ids_lets_model_id_win_over_model(): resolve = _health_endpoints_module._resolve_targeted_model_ids diff --git a/tests/unit/proxy/management_endpoints/scim/test_scim_patch_user.py b/tests/unit/proxy/management_endpoints/scim/test_scim_patch_user.py index 31c5a54a753..1901e0efd4b 100644 --- a/tests/unit/proxy/management_endpoints/scim/test_scim_patch_user.py +++ b/tests/unit/proxy/management_endpoints/scim/test_scim_patch_user.py @@ -502,9 +502,9 @@ def test_apply_patch_ops_remove_group_filtered_path_without_value(): Operations=[SCIMPatchOperation(op="remove", path='groups[value eq "team-1"]')] ) - _, final_team_set = _apply_patch_ops(existing_user=user, patch_ops=patch_ops) + _, final_teams = _apply_patch_ops(existing_user=user, patch_ops=patch_ops) - assert final_team_set == {"team-2"} + assert final_teams == ("team-2",) def test_apply_patch_ops_add_group_filtered_path_without_value(): @@ -519,9 +519,9 @@ def test_apply_patch_ops_add_group_filtered_path_without_value(): Operations=[SCIMPatchOperation(op="add", path="groups[value eq 'team-3']")] ) - _, final_team_set = _apply_patch_ops(existing_user=user, patch_ops=patch_ops) + _, final_teams = _apply_patch_ops(existing_user=user, patch_ops=patch_ops) - assert final_team_set == {"team-1", "team-3"} + assert final_teams == ("team-1", "team-3") def test_apply_patch_ops_replace_groups_empty_value_does_not_use_path_filter(): @@ -539,9 +539,70 @@ def test_apply_patch_ops_replace_groups_empty_value_does_not_use_path_filter(): ] ) - _, final_team_set = _apply_patch_ops(existing_user=user, patch_ops=patch_ops) + _, final_teams = _apply_patch_ops(existing_user=user, patch_ops=patch_ops) - assert final_team_set == set() + assert final_teams == () + + +@pytest.mark.parametrize( + "operations,expected_teams", + [ + pytest.param( + [ + SCIMPatchOperation(op="add", path="groups", value=[{"value": "team-mu"}, {"value": "team-beta"}]), + SCIMPatchOperation(op="remove", path='groups[value eq "team-kappa"]'), + SCIMPatchOperation(op="add", path="groups", value=[{"value": "team-alpha"}, {"value": "team-omega"}]), + ], + ("team-zeta", "team-alpha", "team-delta", "team-mu", "team-beta", "team-omega"), + id="adds_follow_the_record_in_op_order", + ), + pytest.param( + [ + SCIMPatchOperation( + op="replace", + path="groups", + value=[{"value": "team-omega"}, {"value": "team-delta"}, {"value": "team-beta"}], + ) + ], + ("team-delta", "team-omega", "team-beta"), + id="replace_keeps_teams_the_user_has_in_record_order_and_adds_new_ones_in_request_order", + ), + pytest.param( + [ + SCIMPatchOperation( + op="replace", + path="groups", + value=[{"value": "team-omega"}, {"value": "team-delta"}, {"value": "team-beta"}], + ), + SCIMPatchOperation(op="add", path="groups", value=[{"value": "team-mu"}]), + SCIMPatchOperation(op="remove", path='groups[value eq "team-delta"]'), + ], + ("team-delta", "team-omega", "team-beta"), + id="a_replace_wins_over_later_group_ops", + ), + pytest.param( + [ + SCIMPatchOperation(op="remove", path='groups[value eq "team-zeta"]'), + SCIMPatchOperation(op="add", path="groups", value=[{"value": "team-zeta"}]), + ], + ("team-zeta", "team-alpha", "team-kappa", "team-delta"), + id="removing_and_re_adding_a_team_keeps_its_place", + ), + ], +) +def test_apply_patch_ops_keeps_team_order_so_the_first_team_stays_first( + operations: list[SCIMPatchOperation], expected_teams: tuple[str, ...] +): + user: Final = LiteLLM_UserTable( + user_id="user-order", + user_email="order@example.com", + teams=["team-zeta", "team-alpha", "team-kappa", "team-delta"], + metadata={}, + ) + + _, final_teams = _apply_patch_ops(existing_user=user, patch_ops=SCIMPatchOp(Operations=operations)) + + assert final_teams == expected_teams @pytest.mark.asyncio diff --git a/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index 6014c9229f6..180362f45f1 100644 --- a/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -25,6 +25,7 @@ from litellm.proxy._types import ( NewUserResponse, ProxyErrorTypes, ProxyException, + TeamMemberAddRequest, UserAPIKeyAuth, ) from litellm.proxy.management_endpoints.scim.scim_v2 import ( @@ -941,6 +942,57 @@ async def test_handle_existing_user_by_email_syncs_roster_and_dedups_teams(mocke assert update_calls[0].kwargs["data"]["teams"] == ["team-a", "team-b"] +class _EmailMatchedUserRow: + def __init__(self, user: LiteLLM_UserTable) -> None: + self.user = user + + async def find_first(self, *, where: dict[str, object]) -> LiteLLM_UserTable: + return self.user + + async def update(self, *, where: dict[str, object], data: dict[str, object]) -> LiteLLM_UserTable: + self.user = self.user.model_copy(update=data) + return self.user + + +@pytest.mark.asyncio +async def test_handle_existing_user_by_email_keeps_record_team_order(mocker: MockerFixture): + user_rows: Final = _EmailMatchedUserRow( + LiteLLM_UserTable( + user_id="same-id", + user_email="member@example.com", + user_alias="Member", + teams=["team-zeta", "team-alpha", "team-kappa"], + metadata={}, + ) + ) + mock_prisma_client: Final = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = user_rows + + mocker.patch( # test-quality-ok: roster helpers are module-level, not injectable into the helper + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=None), + ) + mocker.patch( # test-quality-ok: roster helpers are module-level, not injectable into the helper + "litellm.proxy.management_endpoints.scim.scim_v2._handle_team_membership_changes", + AsyncMock(), + ) + + await UserProvisionerHelpers.handle_existing_user_by_email( + prisma_client=mock_prisma_client, + new_user_request=NewUserRequest( + user_id="same-id", + user_email="member@example.com", + user_alias="Member", + teams=["team-mu", "team-kappa", "team-beta", "team-zeta"], + metadata={}, + auto_create_key=False, + ), + ) + + assert user_rows.user.teams == ["team-zeta", "team-kappa", "team-mu", "team-beta"] + + @pytest.mark.asyncio async def test_handle_existing_user_by_email_without_teams_preserves_memberships(mocker): """Adoption via POST /Users without ``groups`` must keep the user's existing teams. @@ -1290,6 +1342,49 @@ async def test_handle_team_membership_changes_add_and_remove(mocker): assert call_args[1]["teams_ids_to_remove_user_from"] == ["team1"] +class _TeamRosterRecord: + def __init__(self, missing_team: str) -> None: + self.missing_team = missing_team + self.teams: tuple[str, ...] = () + + async def team_member_add(self, *, data: TeamMemberAddRequest, user_api_key_dict: UserAPIKeyAuth) -> None: + if data.team_id == self.missing_team: + raise HTTPException(status_code=404, detail={"error": "Team not found"}) + self.teams = (*self.teams, data.team_id) + + +@pytest.mark.asyncio +async def test_handle_team_membership_changes_adds_teams_in_request_order_when_one_fails(mocker: MockerFixture): + requested_teams: Final = ( + "team-zeta", + "team-alpha", + "team-kappa", + "team-kept", + "team-mu", + "team-missing", + "team-beta", + "team-omega", + "team-delta", + "team-sigma", + "team-gamma", + "team-tau", + ) + roster: Final = _TeamRosterRecord(missing_team="team-missing") + mocker.patch( # test-quality-ok: team_member_add is module-level, not injectable into the helper + "litellm.proxy.management_endpoints.scim.scim_v2.team_member_add", + roster.team_member_add, + ) + + with pytest.raises(SCIMRosterSyncError): + await _handle_team_membership_changes( + user_id="test-user", + existing_teams=["team-kept"], + new_teams=list(requested_teams), + ) + + assert roster.teams == tuple(team for team in requested_teams if team not in {"team-kept", "team-missing"}) + + @pytest.mark.asyncio async def test_update_user_success(mocker): """Should successfully update user with PUT request""" @@ -1492,6 +1587,57 @@ async def test_update_user_without_groups_preserves_memberships_and_role(mocker, assert "user_role" not in update_data +@pytest.mark.asyncio +async def test_update_user_put_keeps_the_record_order_of_teams_the_user_already_has( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.proxy_server import proxy_config + + async def no_scim_admin_group() -> dict[str, object]: + return {} + + monkeypatch.setattr(proxy_config, "get_config", no_scim_admin_group) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + existing_user: Final = mocker.MagicMock() + existing_user.teams = ["team-zeta", "team-alpha", "team-kappa"] + existing_user.metadata = {} + scim_user: Final = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="order-user", + emails=[SCIMUserEmail(value="order@example.com")], + groups=[ + SCIMUserGroup(value="team-mu"), + SCIMUserGroup(value="team-kappa"), + SCIMUserGroup(value="team-beta"), + SCIMUserGroup(value="team-zeta"), + ], + ) + mock_prisma_client: Final = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.update = AsyncMock(return_value={"user_id": "order-user"}) + mocker.patch( # test-quality-ok: update_user's collaborators are module-level, not injectable + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( # test-quality-ok: update_user's collaborators are module-level, not injectable + "litellm.proxy.management_endpoints.scim.scim_v2._check_user_exists", + AsyncMock(return_value=existing_user), + ) + mocker.patch( # test-quality-ok: update_user's collaborators are module-level, not injectable + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + mocker.patch( # test-quality-ok: roster writes are module-level, not injectable + "litellm.proxy.management_endpoints.scim.scim_v2.patch_team_membership", + AsyncMock(), + ) + + await update_user(user_id="order-user", user=scim_user) + + update_data: Final = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs["data"] + assert update_data["teams"] == ["team-zeta", "team-kappa", "team-mu", "team-beta"] + + @pytest.mark.asyncio async def test_update_user_not_found(mocker): """Should raise 404 when user doesn't exist""" diff --git a/tests/unit/proxy/management_endpoints/test_access_group_management.py b/tests/unit/proxy/management_endpoints/test_access_group_management.py index c36576beaa9..d79af0af519 100644 --- a/tests/unit/proxy/management_endpoints/test_access_group_management.py +++ b/tests/unit/proxy/management_endpoints/test_access_group_management.py @@ -687,33 +687,152 @@ class _FakeAccessGroupBudgetTable: return self.rows.pop(access_group_name, None) +class _FakeClock: + def __init__(self) -> None: + self.now = 0.0 + self.slept: list[float] = [] + + def __call__(self) -> float: + return self.now + + async def sleep(self, seconds: float) -> None: + self.slept.append(seconds) + self.now += seconds + + class _FakeModelTable: - def __init__(self, journal: list[str], deployments) -> None: + def __init__(self, journal: list[str], deployments, failing_model_ids=frozenset()) -> None: self.journal = journal self.deployments = list(deployments) + self.failing_model_ids = failing_model_ids self.updates: list[tuple[dict, dict]] = [] + self.rows_locked_by_another_session: frozenset[str] = frozenset() + self.group_locked_by_another_session = False + self.edits_the_other_session_commits: dict[str, dict] = {} + self.other_session_commits_after_refusals: int | None = None + self.lock_refusals = 0 async def find_many(self, where=None, **kwargs): - return list(self.deployments) + model_names = (where or {}).get("model_name", {}).get("in") + return [d for d in self.deployments if model_names is None or d.model_name in model_names] + + def lock_group(self, wait): + if not self.group_locked_by_another_session: + return True + if wait: + raise TimeoutError("lock_timeout expired waiting for the access group lock") + self._refuse_lock() + return False + + def lock_rows(self, model_ids, skip_locked): + held = self.rows_locked_by_another_session.intersection(model_ids) + if not held: + return list(model_ids) + if not skip_locked: + raise TimeoutError(f"timed out waiting for the row lock on {sorted(held)}") + self._refuse_lock() + return [model_id for model_id in model_ids if model_id not in held] + + def _refuse_lock(self): + self.lock_refusals += 1 + if self.lock_refusals == self.other_session_commits_after_refusals: + self._other_session_commits() + + def _other_session_commits(self): + self.deployments = [ + _FakeDeployment( + model_id=d.model_id, + model_name=d.model_name, + model_info=self.edits_the_other_session_commits.get(d.model_id, d.model_info), + ) + for d in self.deployments + ] + self.rows_locked_by_another_session = frozenset() + self.group_locked_by_another_session = False async def find_unique(self, where, include=None): return next((d for d in self.deployments if d.model_id == where["model_id"]), None) async def update(self, where, data, include=None): + if where["model_id"] in self.failing_model_ids: + raise RuntimeError(f"database write to {where['model_id']} failed") self.journal.append(f"model_table.update:{where['model_id']}") self.updates.append((dict(where), dict(data))) return None + async def create(self, data, **kwargs): + raise AssertionError("access group endpoints never create deployments") + + +class _FakeModelTableBatch: + """Queues model-table updates and applies them all or none when the batch closes, like a Prisma batch.""" + + def __init__(self, model_table: _FakeModelTable) -> None: + self.model_table = model_table + self.queued: list[tuple[dict, dict]] = [] + self.litellm_proxymodeltable = SimpleNamespace(update=self._queue_update) + + def _queue_update(self, where, data): + self.queued.append((dict(where), dict(data))) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + if exc_type is not None: + return False + failing = [where["model_id"] for where, _ in self.queued if where["model_id"] in self.model_table.failing_model_ids] + if failing: + raise RuntimeError(f"batch write to {failing} failed, nothing applied") + for where, data in self.queued: + await self.model_table.update(where=where, data=data) + return False + + +class _FakeModelTableTx: + """An interactive transaction on the primary that journals the row lock and the read it guards.""" + + def __init__(self, journal: list[str], model_table: _FakeModelTable) -> None: + self.journal = journal + self.model_table = model_table + self.litellm_proxymodeltable = SimpleNamespace(find_many=self._find_many) + + async def query_raw(self, query, *args): + if "advisory_xact_lock" in query: + self.journal.append(f"access_group.lock:{args[0]}") + return [{"locked": self.model_table.lock_group(wait="pg_try_" not in query)}] + if "FOR UPDATE" in query: + model_ids = sorted(args[0] if args else (d.model_id for d in self.model_table.deployments)) + self.journal.append(f"model_table.lock:{','.join(model_ids)}") + locked = self.model_table.lock_rows(model_ids, skip_locked="SKIP LOCKED" in query) + return [{"model_id": model_id} for model_id in locked] + self.journal.append(f"model_table.query:{query}") + return [] + + async def _find_many(self): + self.journal.append("model_table.read") + return await self.model_table.find_many() + + def batch_(self): + return _FakeModelTableBatch(self.model_table) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + class _FakePrismaClient: - def __init__(self, journal: list[str], deployments=()) -> None: + def __init__(self, journal: list[str], deployments=(), failing_model_ids=frozenset()) -> None: self.budget_table = _FakeBudgetTable(journal) self.access_group_budget_table = _FakeAccessGroupBudgetTable(journal, self.budget_table) - self.model_table = _FakeModelTable(journal, deployments) + self.model_table = _FakeModelTable(journal, deployments, failing_model_ids) self.db = SimpleNamespace( litellm_budgettable=self.budget_table, litellm_modelaccessgroupbudgettable=self.access_group_budget_table, litellm_proxymodeltable=self.model_table, + tx=lambda timeout=None: _FakeModelTableTx(journal, self.model_table), ) def jsonify_object(self, data): @@ -784,6 +903,12 @@ def _proxy_with_stubbed_reload(prisma): yield +def _serve_model_groups(router, model_groups): + router.registry.return_value.names.side_effect = lambda model_name=None, *, kind="groups", **_: ( + list(model_groups) if kind == "groups" else [] + ) + + def _eviction_journal(access_group): """Both auth cache keys, in the order a write path has to evict them.""" from litellm.proxy.common_utils.user_api_key_cache import ( @@ -1290,3 +1415,605 @@ async def test_deleting_an_access_group_that_never_had_a_budget_still_evicts(): assert response.models_updated == 1 assert prisma.access_group_budget_table.rows == {} _assert_evicted_after_write(journal, "prod-models", "access_group_budget.delete:prod-models") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "selection", + [{"model_ids": ["stays", "joins"]}, {"model_names": ["gpt-stays", "gpt-joins"]}], +) +async def test_updating_an_access_group_writes_only_the_deployments_that_join_or_leave_it(selection): + import json + + import litellm.proxy.proxy_server as proxy_server + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + + with _proxy_with_stubbed_reload(prisma): + _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-leaves", "gpt-joins"]) + response = await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(**selection), + user_api_key_dict=_admin(), + ) + + written_access_groups = { + where["model_id"]: json.loads(data["model_info"])["access_groups"] + for where, data in prisma.model_table.updates + } + assert written_access_groups == {"joins": ["prod-models"], "leaves": []} + assert response.models_updated == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "selection", + [{"model_ids": ["stays", "joins"]}, {"model_names": ["gpt-stays", "gpt-joins"]}], +) +async def test_updating_an_access_group_reads_the_primary_not_a_lagging_replica(selection): + import json + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + primary = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + replica_before_the_last_edit = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves", access_groups=()), + _deployment(model_id="joins", model_name="gpt-joins"), + ], + ) + primary.db = RoutingPrismaWrapper( + writer=PrismaWrapper(original_prisma=primary.db, iam_token_db_auth=False), + reader=PrismaWrapper(original_prisma=replica_before_the_last_edit.db, iam_token_db_auth=False), + ) + + with _proxy_with_stubbed_reload(primary): + _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-leaves", "gpt-joins"]) + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(**selection), + user_api_key_dict=_admin(), + ) + + written_access_groups = { + where["model_id"]: json.loads(data["model_info"])["access_groups"] + for where, data in primary.model_table.updates + } + assert written_access_groups == {"joins": ["prod-models"], "leaves": []} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failing_model_id", ["leaves", "joins"]) +async def test_a_failed_access_group_update_leaves_every_deployment_unchanged(failing_model_id): + from fastapi import HTTPException + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + deployments = [ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ] + prisma = _FakePrismaClient([], deployments=deployments, failing_model_ids=frozenset({failing_model_id})) + + with _proxy_with_stubbed_reload(prisma): + _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-leaves", "gpt-joins"]) + with pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays", "joins"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 500 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +async def test_an_access_group_update_keeps_a_model_info_field_another_admin_set_while_names_were_validated(): + import json + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + + def another_admin_edits_joins_while_names_are_checked(model_name=None, *, kind="groups", **_): + if kind != "groups": + return [] + prisma.model_table.deployments = [ + deployment + if deployment.model_id != "joins" + else _FakeDeployment( + model_id="joins", + model_name="gpt-joins", + model_info={"access_groups": [], "description": "set by another admin"}, + ) + for deployment in prisma.model_table.deployments + ] + return ["gpt-stays", "gpt-joins"] + + with _proxy_with_stubbed_reload(prisma): + proxy_server.llm_router.registry.return_value.names.side_effect = ( + another_admin_edits_joins_while_names_are_checked + ) + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_names=["gpt-stays", "gpt-joins"]), + user_api_key_dict=_admin(), + ) + + written = {where["model_id"]: json.loads(data["model_info"]) for where, data in prisma.model_table.updates} + assert written["joins"]["description"] == "set by another admin" + assert written["joins"]["access_groups"] == ["prod-models"] + + +@pytest.mark.asyncio +async def test_an_access_group_update_locks_the_rows_it_reads_and_tells_other_workers_after_the_commit(): + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + journal: list[str] = [] + prisma = _FakePrismaClient( + journal, + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + + async def record_publish(model_ids): + journal.append(f"publish:{','.join(sorted(model_ids))}") + + with ( + _proxy_with_stubbed_reload(prisma), + patch( # test-quality-ok: the endpoint publishes through its module-level import; Redis is the real seam + "litellm.proxy.management_endpoints.model_access_group_management_endpoints.publish_model_rows_written", + new=record_publish, + ), + ): + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays", "joins"]), + user_api_key_dict=_admin(), + ) + + assert journal == [ + "access_group.lock:prod-models", + "model_table.read", + "model_table.lock:joins,leaves,stays", + "model_table.read", + "model_table.update:leaves", + "model_table.update:joins", + "publish:joins,leaves", + ] + + +@pytest.mark.asyncio +async def test_an_access_group_update_that_changes_no_membership_writes_and_announces_nothing(): + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + journal: list[str] = [] + prisma = _FakePrismaClient( + journal, + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="also-stays", model_name="gpt-also-stays"), + _deployment(model_id="outside", model_name="gpt-outside", access_groups=()), + ], + ) + + async def record_publish(model_ids): + journal.append(f"publish:{','.join(sorted(model_ids))}") + + with ( + _proxy_with_stubbed_reload(prisma), + patch( # test-quality-ok: the endpoint publishes through its module-level import; Redis is the real seam + "litellm.proxy.management_endpoints.model_access_group_management_endpoints.publish_model_rows_written", + new=record_publish, + ), + ): + response = await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays", "also-stays"]), + user_api_key_dict=_admin(), + ) + + assert response.models_updated == 2 + assert journal == [ + "access_group.lock:prod-models", + "model_table.read", + "model_table.lock:also-stays,stays", + "model_table.read", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ignored_model_names", [None, ["gpt-unrelated"]]) +async def test_an_access_group_update_goes_through_while_another_session_holds_a_model_row_outside_the_group( + ignored_model_names, +): + import json + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + _deployment(model_id="unrelated", model_name="gpt-unrelated", access_groups=("other-group",)), + ], + ) + prisma.model_table.rows_locked_by_another_session = frozenset({"unrelated"}) + + with _proxy_with_stubbed_reload(prisma): + response = await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays", "joins"], model_names=ignored_model_names), + user_api_key_dict=_admin(), + ) + + written_access_groups = { + where["model_id"]: json.loads(data["model_info"])["access_groups"] + for where, data in prisma.model_table.updates + } + assert written_access_groups == {"joins": ["prod-models"], "leaves": []} + assert response.models_updated == 2 + + +@pytest.mark.asyncio +async def test_an_access_group_update_retries_a_locked_row_and_keeps_the_edit_committed_there(): + import json + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + prisma.model_table.rows_locked_by_another_session = frozenset({"joins"}) + prisma.model_table.edits_the_other_session_commits = { + "joins": {"access_groups": [], "description": "saved while the update waited"} + } + prisma.model_table.other_session_commits_after_refusals = 1 + + with _proxy_with_stubbed_reload(prisma): + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays", "joins"]), + user_api_key_dict=_admin(), + ) + + written = {where["model_id"]: json.loads(data["model_info"]) for where, data in prisma.model_table.updates} + assert written == {"joins": {"access_groups": ["prod-models"], "description": "saved while the update waited"}} + + +@pytest.mark.asyncio +async def test_an_access_group_emptied_while_the_update_retried_a_locked_row_answers_404_and_writes_nothing(): + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"}) + prisma.model_table.edits_the_other_session_commits = {"leaves": {"access_groups": []}} + prisma.model_table.other_session_commits_after_refusals = 1 + + with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["joins"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 404 + assert prisma.model_table.updates == [] + + +def _another_session_holds_a_member_row(model_table): + model_table.rows_locked_by_another_session = frozenset({"leaves"}) + + +def _another_edit_holds_the_group(model_table): + model_table.group_locked_by_another_session = True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"] +) +async def test_an_access_group_update_answers_409_and_writes_nothing_while_another_session_keeps_a_lock_it_needs( + hold, +): + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + _rewrite_membership_once_unlocked, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + ], + ) + hold(prisma.model_table) + clock = _FakeClock() + + with pytest.raises(HTTPException) as raised: + await _rewrite_membership_once_unlocked( + prisma_client=prisma, + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays"]), + retry_budget_seconds=0.5, + sleep=clock.sleep, + clock=clock, + ) + + assert raised.value.status_code == 409 + assert 0 < clock.now <= 0.5 + assert prisma.model_table.lock_refusals == len(clock.slept) + 1 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"] +) +async def test_an_access_group_update_waits_out_a_lock_held_for_over_ten_seconds_and_writes_the_edit(hold): + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + _rewrite_membership_once_unlocked, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + ], + ) + hold(prisma.model_table) + prisma.model_table.other_session_commits_after_refusals = 20 + clock = _FakeClock() + + _, rewrites = await _rewrite_membership_once_unlocked( + prisma_client=prisma, + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays"]), + sleep=clock.sleep, + clock=clock, + ) + + assert clock.now > 10 + assert [model_id for model_id, _ in rewrites] == ["leaves"] + assert [where["model_id"] for where, _ in prisma.model_table.updates] == ["leaves"] + + +@pytest.mark.asyncio +async def test_updating_a_missing_access_group_answers_404_before_checking_the_model_names(): + from fastapi import HTTPException + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], deployments=[_deployment(model_id="other", model_name="gpt-other", access_groups=("other-group",))] + ) + + with _proxy_with_stubbed_reload(prisma): + _serve_model_groups(proxy_server.llm_router, ["gpt-other"]) + with pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_names=["no-such-model"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 404 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +async def test_an_access_group_update_answers_500_when_the_deployments_cannot_be_read(): + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient([], deployments=[_deployment(model_id="stays")]) + prisma.model_table.find_many = AsyncMock(side_effect=RuntimeError("database unreachable")) + + with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 500 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_ids", [["no-such-deployment"], ["joins", "no-such-deployment"]]) +async def test_updating_an_access_group_with_an_unknown_model_id_writes_no_deployment(model_ids): + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[_deployment(model_id="stays"), _deployment(model_id="joins", access_groups=())], + ) + + with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=model_ids), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 400 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +async def test_updating_an_access_group_no_deployment_carries_answers_404_and_writes_nothing(): + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient([], deployments=[_deployment(model_id="other", access_groups=("other-group",))]) + + with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["other"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 404 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +async def test_updating_an_access_group_with_a_model_name_only_the_config_serves_writes_no_deployment(): + from fastapi import HTTPException + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + + with _proxy_with_stubbed_reload(prisma): + _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-joins", "config-only-model"]) + with pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_names=["gpt-joins", "config-only-model"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 400 + assert prisma.model_table.updates == [] diff --git a/tests/unit/proxy/management_endpoints/test_common_daily_activity.py b/tests/unit/proxy/management_endpoints/test_common_daily_activity.py index c80fa3572fd..5fe4cd2edcd 100644 --- a/tests/unit/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/unit/proxy/management_endpoints/test_common_daily_activity.py @@ -2553,6 +2553,71 @@ async def test_get_daily_activity_rejects_non_canonical_dates_before_querying(st mock_table.find_many.assert_not_awaited() +def _provider_record(provider: str, *, model: str, spend: float): + base = vars(_spend_record("real-key", model=model, spend=spend)) + return SimpleNamespace(**{**base, "custom_llm_provider": provider, "api_requests": 1, "successful_requests": 1}) + + +def test_per_row_provider_bucket_marks_subscription_backed_providers_only(): + from litellm.proxy.management_endpoints.common_daily_activity import update_breakdown_metrics + from litellm.types.proxy.management_endpoints.common_daily_activity import BreakdownMetrics + + breakdown = BreakdownMetrics() + update_breakdown_metrics(breakdown, _provider_record("chatgpt", model="gpt-5.6-terra", spend=0.0), {}, {}, {}) + update_breakdown_metrics(breakdown, _provider_record("openai", model="gpt-5.6", spend=0.0), {}, {}, {}) + update_breakdown_metrics(breakdown, _provider_record("chatgpt", model="gpt-5.6-sol", spend=0.0), {}, {}, {}) + + assert breakdown.providers["chatgpt"].metadata == {"subscription_covered": True} + assert "subscription_covered" not in breakdown.providers["openai"].metadata + assert breakdown.providers["chatgpt"].metrics.api_requests == 2 + assert "subscription_covered" not in breakdown.models["gpt-5.6-terra"].metadata + + +def test_per_row_provider_marker_keeps_existing_provider_metadata(): + from litellm.proxy.management_endpoints.common_daily_activity import update_breakdown_metrics + from litellm.types.proxy.management_endpoints.common_daily_activity import BreakdownMetrics + + breakdown = BreakdownMetrics() + update_breakdown_metrics( + breakdown, + _provider_record("chatgpt", model="gpt-5.6-terra", spend=0.0), + {}, + {"chatgpt": {"region": "us"}}, + {}, + ) + + assert breakdown.providers["chatgpt"].metadata == {"region": "us", "subscription_covered": True} + + +@pytest.mark.parametrize("provider_row_first", [True, False]) +def test_grouping_sets_provider_bucket_marks_subscription_backed_providers_in_any_row_order(provider_row_first): + from litellm.proxy.management_endpoints.common_daily_activity import ( + _GROUP_DATE_PROVIDER, + _GROUP_DATE_PROVIDER_API_KEY, + _GROUP_GRAND_TOTAL, + _aggregate_grouping_sets_records_sync, + ) + + provider_rows = [ + _grouping_row(_GROUP_DATE_PROVIDER, custom_llm_provider="chatgpt", spend=0.0), + _grouping_row(_GROUP_DATE_PROVIDER_API_KEY, custom_llm_provider="chatgpt", api_key="real-key", spend=0.0), + ] + ordered_rows = provider_rows if provider_row_first else list(reversed(provider_rows)) + records = [ + *ordered_rows, + _grouping_row(_GROUP_DATE_PROVIDER, custom_llm_provider="openai", spend=0.0), + _grouping_row(_GROUP_DATE_PROVIDER_API_KEY, custom_llm_provider="openai", api_key="real-key", spend=0.0), + _grouping_row(_GROUP_GRAND_TOTAL, spend=0.0), + ] + + aggregated = _aggregate_grouping_sets_records_sync(records=records, api_key_metadata={}) + + providers = aggregated["results"][0].breakdown.providers + assert providers["chatgpt"].metadata == {"subscription_covered": True} + assert "real-key" in providers["chatgpt"].api_key_breakdown + assert "subscription_covered" not in providers["openai"].metadata + + @pytest.mark.parametrize("missing", [None, ""]) def test_missing_dimensions_preserve_all_paginated_metrics(missing: str | None) -> None: records: Final = ( diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index 42162087c9b..39586a1a204 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4195,6 +4195,65 @@ async def test_add_new_user_to_default_team_string_teams_have_no_member_budget(m assert mock_add.call_args.kwargs["team_id"] == "string-team" +@pytest.mark.asyncio +async def test_add_new_user_to_default_team_joins_teams_in_the_order_given_and_past_a_failing_team( + mocker: MockerFixture, +) -> None: + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + add_new_user_to_default_team, + ) + + requested: Final = ("team-zeta", "team-broken", "team-mu", "team-alpha") + seconds_per_team: Final = {"team-zeta": 0.03, "team-broken": 0.0, "team-mu": 0.02, "team-alpha": 0.0} + finished: Final = mocker.MagicMock() + + async def join_team(*, team_id: str, **_: object) -> None: + await asyncio.sleep(seconds_per_team[team_id]) + if team_id == "team-broken": + raise RuntimeError("team row locked") + finished(team_id) + + mocker.patch( + "litellm.proxy.management_endpoints.internal_user_endpoints._add_user_to_team", + side_effect=join_team, + ) + + await add_new_user_to_default_team( + user_id="scim-user", + user_email=None, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + teams=list(requested), + prisma_client=mocker.MagicMock(), + ) + + assert [call.args[0] for call in finished.call_args_list] == ["team-zeta", "team-mu", "team-alpha"] + + +@pytest.mark.asyncio +async def test_add_new_user_to_default_team_rejects_an_invalid_team_before_joining_any( + mocker: MockerFixture, +) -> None: + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + add_new_user_to_default_team, + ) + + join_team: Final = mocker.patch( + "litellm.proxy.management_endpoints.internal_user_endpoints._add_user_to_team", + new_callable=mocker.AsyncMock, + ) + + with pytest.raises(ValueError, match="Invalid team type"): + await add_new_user_to_default_team( + user_id="new-user", + user_email=None, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + teams=["team-zeta", {"team_id": "team-mu"}], + prisma_client=mocker.MagicMock(), + ) + + join_team.assert_not_awaited() + + @pytest.mark.asyncio async def test_add_user_to_team_logs_unknown_team_at_error(mocker, caplog): """A default team that no longer exists makes every membership write 404. diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index b0ef7e64e60..62dd367b8e5 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -13,7 +13,7 @@ import pytest import yaml from fastapi import HTTPException from fastapi.testclient import TestClient -from pydantic import JsonValue +from pydantic import JsonValue, ValidationError import litellm from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS @@ -7597,6 +7597,7 @@ async def test_generate_key_with_router_settings(monkeypatch): "num_retries": 3, "model_group_retry_policy": {"gpt-4": {"RateLimitErrorRetries": 5}}, "weights": {"gpt-4": {"weighted-id": 1}}, + "content_policy_fallbacks": [{"gpt-4": ["gpt-4o", "gpt-4o-mini"]}], } request_data = GenerateKeyRequest( @@ -7736,6 +7737,19 @@ async def test_update_key_with_router_settings( assert json.loads(cleared["router_settings"]) == {} +@pytest.mark.parametrize("request_type", [GenerateKeyRequest, UpdateKeyRequest]) +@pytest.mark.parametrize("dict_target", [{"model": "gpt-4o-mini"}, {}]) +def test_a_key_content_policy_rule_can_only_name_fallback_models( + request_type: type[GenerateKeyRequest] | type[UpdateKeyRequest], + dict_target: dict[str, str], +): + with pytest.raises(ValidationError, match="must be model names"): + request_type( + key="sk-key", + router_settings={"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", dict_target]}]}, + ) + + @pytest.mark.asyncio async def test_validate_max_budget(): """ @@ -18736,7 +18750,7 @@ async def test_list_keys_rejects_invalid_expires(): expires="expred", ) - assert exc_info.value.code == "400" + assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error") assert "Invalid expires value" in str(exc_info.value.message) diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 14d93f368c1..872d1607351 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -6508,7 +6508,11 @@ class TestStrategyRouterWriteValidation: assert fake.tx_obj.raw_calls == [] return assert handle is fake.tx_obj.litellm_proxymodeltable - published.assert_awaited_once_with(redis_cache=None, object_type="litellm_proxymodeltable") + published.assert_awaited_once_with( + redis_cache=None, + object_type="litellm_proxymodeltable", + model_ids=(model_id,) if model_id else (), + ) (lock_sql, lock_params), (count_sql, count_params) = fake.tx_obj.raw_calls assert "pg_advisory_xact_lock($1)" in lock_sql and "count" not in lock_sql assert lock_params == (AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY,) diff --git a/tests/unit/proxy/management_endpoints/test_subscription_accounts.py b/tests/unit/proxy/management_endpoints/test_subscription_accounts.py new file mode 100644 index 00000000000..bcd31efd4b7 --- /dev/null +++ b/tests/unit/proxy/management_endpoints/test_subscription_accounts.py @@ -0,0 +1,438 @@ +from collections.abc import Iterator +from datetime import datetime, timezone +from typing import Final +from uuid import uuid4 + +from unittest.mock import MagicMock + +import pytest +from fastapi import FastAPI, HTTPException, Request +from fastapi.testclient import TestClient +from pydantic import ValidationError + +from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable +from litellm.proxy._types import LiteLLM_JWTAuth, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import allowed_routes_check +from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.management_endpoints.subscription_accounts import ( + SubscriptionDeployment, + get_subscription_account_repository, + get_subscription_deployments, + router, + subscription_deployments_from_model_list, + subscription_deployments_of_router, +) +from litellm.router import Router +from litellm.types.proxy.management_endpoints.subscription_accounts import ( + SubscriptionAccountFeeRequest, + UpdateSubscriptionAccountFeeRequest, +) +from litellm.repositories.subscription_account_repository import ( + DuplicateSubscriptionAccount, + SubscriptionAccountFeeChanges, + SubscriptionAccountFeeFields, +) + +_ACCOUNT: Final = "org-team-workspace" +_TEST_FEE: Final = { + "custom_llm_provider": "chatgpt", + "account_id": _ACCOUNT, + "monthly_fee": 25.0, + "currency": "USD", + "billing_period_start": "2026-10-01", + "label": "Team plan (test value)", +} +_TWO_DEPLOYMENTS_ONE_ACCOUNT: Final = ( + SubscriptionDeployment(model_name="gpt-5.6-terra", custom_llm_provider="chatgpt", account_id=_ACCOUNT), + SubscriptionDeployment(model_name="gpt-5.6-sol", custom_llm_provider="chatgpt", account_id=_ACCOUNT), +) + + +class _FakeStore: + def __init__(self) -> None: + self.rows: dict[str, LiteLLM_SubscriptionAccountTable] = {} + self.written_columns: list[dict[str, float | str | None]] = [] + + async def list_all(self) -> list[LiteLLM_SubscriptionAccountTable]: + return sorted(self.rows.values(), key=lambda row: row.created_at) + + async def find_by_id(self, id_value: str) -> LiteLLM_SubscriptionAccountTable | None: + return self.rows.get(id_value) + + async def create_account( + self, custom_llm_provider: str, account_id: str, fee: SubscriptionAccountFeeFields, created_by: str + ) -> LiteLLM_SubscriptionAccountTable | DuplicateSubscriptionAccount: + for row in self.rows.values(): + if (row.custom_llm_provider, row.account_id) == (custom_llm_provider, account_id): + return DuplicateSubscriptionAccount(custom_llm_provider=custom_llm_provider, account_id=account_id) + now = datetime.now(timezone.utc) + row = LiteLLM_SubscriptionAccountTable( + subscription_account_id=str(uuid4()), + custom_llm_provider=custom_llm_provider, + account_id=account_id, + label=fee.label, + monthly_fee=fee.monthly_fee, + currency=fee.currency, + billing_period_start=fee.billing_period_start, + created_at=now, + created_by=created_by, + updated_at=now, + updated_by=created_by, + ) + self.rows[row.subscription_account_id] = row + return row + + async def update_fee( + self, subscription_account_id: str, changes: SubscriptionAccountFeeChanges, updated_by: str + ) -> LiteLLM_SubscriptionAccountTable | None: + columns = dict(changes.columns()) + self.written_columns.append(columns) + existing = self.rows.get(subscription_account_id) + if existing is None: + return None + updated = existing.model_copy( + update={**columns, "updated_by": updated_by, "updated_at": datetime.now(timezone.utc)} + ) + self.rows[subscription_account_id] = updated + return updated + + async def delete_account(self, subscription_account_id: str) -> LiteLLM_SubscriptionAccountTable | None: + return self.rows.pop(subscription_account_id, None) + + +def _resolve_auth(request: Request) -> UserAPIKeyAuth: + role: Final = LitellmUserRoles(request.headers.get("x-user-role", LitellmUserRoles.PROXY_ADMIN.value)) + return UserAPIKeyAuth(user_id="admin", user_role=role, api_key="sk-test") + + +@pytest.fixture +def deployments() -> tuple[SubscriptionDeployment, ...]: + return _TWO_DEPLOYMENTS_ONE_ACCOUNT + + +@pytest.fixture +def client(deployments: tuple[SubscriptionDeployment, ...]) -> Iterator[tuple[TestClient, _FakeStore]]: + store: Final = _FakeStore() + app: Final = FastAPI() + app.include_router(router) + app.dependency_overrides[get_subscription_account_repository] = lambda: store + app.dependency_overrides[get_subscription_deployments] = lambda: deployments + app.dependency_overrides[user_api_key_auth] = _resolve_auth + with TestClient(app) as test_client: + yield test_client, store + + +def _usage(test_client: TestClient, start_date: str, end_date: str, **headers: str) -> dict: + response = test_client.get( + "/subscription_accounts/usage", params={"start_date": start_date, "end_date": end_date}, headers=headers + ) + assert response.status_code == 200, response.text + return response.json() + + +def test_untracked_account_lists_its_deployments_with_no_fee(client): + test_client, _ = client + + body = _usage(test_client, "2026-10-01", "2026-10-07") + + assert body["subscription_providers"] == ["chatgpt"] + assert body["accounts"] == [ + { + "custom_llm_provider": "chatgpt", + "account_id": _ACCOUNT, + "deployments": ["gpt-5.6-sol", "gpt-5.6-terra"], + "fee": None, + "billing_periods": [], + "fixed_cost": None, + } + ] + + +def test_fee_counts_once_per_billing_period_however_many_deployments_share_the_account(client): + test_client, _ = client + created = test_client.post("/subscription_accounts/new", json=_TEST_FEE) + assert created.status_code == 200, created.text + + one_period = _usage(test_client, "2026-10-01", "2026-10-07") + two_periods = _usage(test_client, "2026-09-15", "2026-11-15") + between_starts = _usage(test_client, "2026-10-02", "2026-10-31") + + (account,) = one_period["accounts"] + assert account["deployments"] == ["gpt-5.6-sol", "gpt-5.6-terra"] + assert account["fee"]["monthly_fee"] == 25.0 + assert account["fee"]["currency"] == "USD" + assert account["fee"]["label"] == "Team plan (test value)" + assert account["billing_periods"] == [{"start": "2026-10-01", "end": "2026-10-31"}] + assert account["fixed_cost"] == 25.0 + assert two_periods["accounts"][0]["fixed_cost"] == 50.0 + assert [period["start"] for period in two_periods["accounts"][0]["billing_periods"]] == [ + "2026-10-01", + "2026-11-01", + ] + assert between_starts["accounts"][0]["fixed_cost"] == 0.0 + assert between_starts["accounts"][0]["billing_periods"] == [] + + +def test_fee_of_an_account_no_deployment_uses_stays_listed_and_counted(client): + test_client, _ = client + other = {**_TEST_FEE, "account_id": "org-other", "monthly_fee": 200.0, "currency": "EUR"} + assert test_client.post("/subscription_accounts/new", json=other).status_code == 200 + assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200 + + body = _usage(test_client, "2026-10-01", "2026-10-07") + + unused, used = body["accounts"] + assert (unused["account_id"], unused["deployments"], unused["fee"]["currency"]) == ("org-other", [], "EUR") + assert unused["fixed_cost"] == 200.0 + assert (used["account_id"], used["fixed_cost"]) == (_ACCOUNT, 25.0) + + +@pytest.mark.parametrize( + "deployments", + [(SubscriptionDeployment(model_name="gpt-5.6-terra", custom_llm_provider="chatgpt", account_id=None),)], +) +def test_account_not_signed_in_shows_no_account_and_takes_no_fee(client): + test_client, _ = client + assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200 + + body = _usage(test_client, "2026-10-01", "2026-10-07") + + unsigned = next(account for account in body["accounts"] if account["account_id"] is None) + assert unsigned["deployments"] == ["gpt-5.6-terra"] + assert unsigned["fee"] is None + assert unsigned["fixed_cost"] is None + + +def test_update_changes_the_fee_and_delete_returns_the_account_to_untracked(client): + test_client, store = client + created = test_client.post("/subscription_accounts/new", json=_TEST_FEE).json() + account_id = created["subscription_account_id"] + + updated = test_client.post( + "/subscription_accounts/update", + json={"subscription_account_id": account_id, "monthly_fee": 30.0, "billing_period_start": "2026-10-05"}, + ) + assert updated.status_code == 200, updated.text + assert updated.json()["monthly_fee"] == 30.0 + assert updated.json()["currency"] == "USD" + assert updated.json()["billing_period_start"] == "2026-10-05" + assert updated.json()["label"] == _TEST_FEE["label"] + assert _usage(test_client, "2026-10-01", "2026-10-07")["accounts"][0]["fixed_cost"] == 30.0 + + cleared = test_client.post( + "/subscription_accounts/update", json={"subscription_account_id": account_id, "label": None} + ) + assert cleared.status_code == 200, cleared.text + assert cleared.json()["label"] is None + assert cleared.json()["monthly_fee"] == 30.0 + assert store.written_columns == [{"monthly_fee": 30.0, "billing_period_start": "2026-10-05"}, {"label": None}] + + unknown = test_client.post( + "/subscription_accounts/update", json={"subscription_account_id": "no-such-account", "monthly_fee": 1.0} + ) + assert unknown.status_code == 404 + + deleted = test_client.post("/subscription_accounts/delete", json={"subscription_account_id": account_id}) + assert deleted.status_code == 200, deleted.text + assert store.rows == {} + assert _usage(test_client, "2026-10-01", "2026-10-07")["accounts"][0]["fixed_cost"] is None + + missing = test_client.post("/subscription_accounts/delete", json={"subscription_account_id": account_id}) + assert missing.status_code == 404 + + +def test_rejects_a_second_fee_for_the_same_account(client): + test_client, _ = client + assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200 + + duplicate = test_client.post("/subscription_accounts/new", json={**_TEST_FEE, "monthly_fee": 99.0}) + + assert duplicate.status_code == 400 + assert "/subscription_accounts/update" in duplicate.json()["detail"]["error"] + + +def test_rejects_a_fee_on_a_metered_provider(client): + test_client, store = client + + response = test_client.post("/subscription_accounts/new", json={**_TEST_FEE, "custom_llm_provider": "openai"}) + + assert response.status_code == 400 + assert store.rows == {} + + +@pytest.mark.parametrize( + "payload", + [ + {**_TEST_FEE, "monthly_fee": 0}, + {**_TEST_FEE, "account_id": ""}, + {**_TEST_FEE, "currency": "usd"}, + {**_TEST_FEE, "currency": "$"}, + {**_TEST_FEE, "billing_period_start": "10/01/2026"}, + ], +) +def test_rejects_malformed_fee_fields(client, payload): + test_client, store = client + + assert test_client.post("/subscription_accounts/new", json=payload).status_code == 422 + assert store.rows == {} + + +def test_rejects_an_infinite_monthly_fee(): + with pytest.raises(ValidationError): + SubscriptionAccountFeeRequest.model_validate({**_TEST_FEE, "monthly_fee": float("inf")}) + with pytest.raises(ValidationError): + UpdateSubscriptionAccountFeeRequest.model_validate( + {"subscription_account_id": "fee-1", "monthly_fee": float("inf")} + ) + + +@pytest.mark.parametrize("deployments", [()]) +def test_fee_stays_listed_after_the_last_subscription_deployment_is_removed(client): + test_client, _ = client + assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200 + + body = _usage(test_client, "2026-10-01", "2026-10-07") + + (account,) = body["accounts"] + assert (account["account_id"], account["deployments"], account["fixed_cost"]) == (_ACCOUNT, [], 25.0) + + +def test_usage_rejects_a_malformed_or_inverted_range(client): + test_client, _ = client + + malformed = [ + test_client.get("/subscription_accounts/usage", params={"start_date": start, "end_date": "2026-10-07"}) + for start in ("2026-10-1", "20261001", "2026-W40-1", "2026-13-01") + ] + inverted = test_client.get( + "/subscription_accounts/usage", params={"start_date": "2026-10-07", "end_date": "2026-10-01"} + ) + + assert [response.status_code for response in malformed] == [400, 400, 400, 400] + assert inverted.status_code == 400 + + +def test_viewer_reads_usage_but_cannot_write(client): + test_client, store = client + viewer = {"x-user-role": LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value} + + assert _usage(test_client, "2026-10-01", "2026-10-07", **viewer)["accounts"][0]["account_id"] == _ACCOUNT + assert test_client.post("/subscription_accounts/new", json=_TEST_FEE, headers=viewer).status_code == 403 + assert store.rows == {} + + +_SUBSCRIPTION_ROUTES: Final = ( + "/subscription_accounts/usage", + "/subscription_accounts/new", + "/subscription_accounts/update", + "/subscription_accounts/delete", +) + + +def test_routes_follow_the_admin_route_policy(): + jwt_admin_defaults = LiteLLM_JWTAuth() + + for route in _SUBSCRIPTION_ROUTES: + assert allowed_routes_check( + user_role=LitellmUserRoles.PROXY_ADMIN, user_route=route, litellm_proxy_roles=jwt_admin_defaults + ) + assert RouteChecks.is_management_route(route) + + +def test_viewer_route_gate_reads_usage_and_blocks_writes(): + viewer_role: Final = LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value + get_usage = MagicMock(spec=Request, method="GET", query_params={}) + post_new = MagicMock(spec=Request, method="POST", query_params={}) + + RouteChecks._check_proxy_admin_viewer_access( + route="/subscription_accounts/usage", _user_role=viewer_role, request_data={}, request=get_usage + ) + with pytest.raises(HTTPException) as denied: + RouteChecks._check_proxy_admin_viewer_access( + route="/subscription_accounts/new", _user_role=viewer_role, request_data=_TEST_FEE, request=post_new + ) + assert denied.value.status_code == 403 + + +def test_internal_user_cannot_read_subscription_accounts(client): + test_client, _ = client + + response = test_client.get( + "/subscription_accounts/usage", + params={"start_date": "2026-10-01", "end_date": "2026-10-07"}, + headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value}, + ) + + assert response.status_code == 403 + + +def test_deployments_take_the_account_of_their_own_login(): + model_list = [ + {"model_name": "gpt-5.6-terra", "litellm_params": {"model": "chatgpt/gpt-5.6-terra"}}, + {"model_name": "gpt-5.6-sol", "litellm_params": {"model": "chatgpt/gpt-5.6-sol"}}, + { + "model_name": "gpt-5.6-luna", + "litellm_params": {"model": "chatgpt/gpt-5.6-luna", "chatgpt_auth_file": "/logins/other.json"}, + }, + {"model_name": "sonnet", "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-x"}}, + {"model_name": "broken", "litellm_params": {"model": "no-such-provider-model"}}, + ] + lookups: list[tuple[str, object]] = [] + + def resolver(provider: str, litellm_params) -> str | None: + lookups.append((provider, litellm_params.get("model"))) + return "org-other" if litellm_params.get("chatgpt_auth_file") == "/logins/other.json" else _ACCOUNT + + deployments = subscription_deployments_from_model_list(model_list, resolver) + + assert deployments == ( + *_TWO_DEPLOYMENTS_ONE_ACCOUNT, + SubscriptionDeployment(model_name="gpt-5.6-luna", custom_llm_provider="chatgpt", account_id="org-other"), + ) + assert lookups == [ + ("chatgpt", "chatgpt/gpt-5.6-terra"), + ("chatgpt", "chatgpt/gpt-5.6-sol"), + ("chatgpt", "chatgpt/gpt-5.6-luna"), + ] + + +def test_deployments_list_each_router_deployment_once_by_the_name_callers_use(): + llm_router: Final = Router( + model_list=[ + {"model_name": "gpt-5.6-sol", "litellm_params": {"model": "chatgpt/gpt-5.6-sol"}}, + { + "model_name": "model_name_team-1_7f3a", + "litellm_params": {"model": "chatgpt/gpt-5.6-terra"}, + "model_info": {"team_id": "team-1", "team_public_model_name": "team-terra"}, + }, + ], + model_group_alias={"sol-alias": "gpt-5.6-sol"}, + ) + + deployments = subscription_deployments_of_router(llm_router, lambda provider, litellm_params: _ACCOUNT) + + assert sorted(deployment.model_name for deployment in deployments) == ["gpt-5.6-sol", "team-terra"] + + +@pytest.mark.parametrize( + "deployments", + [ + ( + *_TWO_DEPLOYMENTS_ONE_ACCOUNT, + SubscriptionDeployment(model_name="gpt-5.6-luna", custom_llm_provider="chatgpt", account_id="org-other"), + ) + ], +) +def test_two_logins_each_count_their_own_fee_once(client): + test_client, _ = client + other = {**_TEST_FEE, "account_id": "org-other", "monthly_fee": 40.0} + assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200 + assert test_client.post("/subscription_accounts/new", json=other).status_code == 200 + + body = _usage(test_client, "2026-10-01", "2026-10-07") + + assert [(a["account_id"], a["deployments"], a["fixed_cost"]) for a in body["accounts"]] == [ + ("org-other", ["gpt-5.6-luna"], 40.0), + (_ACCOUNT, ["gpt-5.6-sol", "gpt-5.6-terra"], 25.0), + ] diff --git a/tests/unit/proxy/management_endpoints/test_ui_sso.py b/tests/unit/proxy/management_endpoints/test_ui_sso.py index cfb64a85359..136cc86e67a 100644 --- a/tests/unit/proxy/management_endpoints/test_ui_sso.py +++ b/tests/unit/proxy/management_endpoints/test_ui_sso.py @@ -4049,8 +4049,8 @@ class TestCLIKeyRegenerationFlow: assert ( '" ) in page @@ -4530,6 +4530,39 @@ class TestCLIKeyRegenerationFlow: == () ) + @pytest.mark.asyncio + async def test_fetch_cli_sso_team_details_follows_the_user_record_order(self): + """The CLI login binds to the first detail, so it must be the first team on the user record. + + Postgres returns the rows in whatever order the table holds them, which every write to a + team row changes, so the bound team used to flip between sign-ins. + """ + from litellm.proxy.management_endpoints.ui_sso import ( + fetch_cli_sso_team_details, + ) + + def team_row(team_id: str) -> MagicMock: + row: Final = MagicMock() + row.model_dump.return_value = { + "team_id": team_id, + "team_alias": None, + "models": [], + "litellm_model_table": None, + } + return row + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_teamtable.find_many = AsyncMock( + return_value=[team_row("team-alpha"), team_row("team-zeta")] + ) + + details: Final = await fetch_cli_sso_team_details( + prisma_client=prisma_client, + teams=["team-zeta", "team-gone", "team-alpha", "team-zeta"], + ) + + assert [detail.team_id for detail in details] == ["team-zeta", "team-alpha"] + @pytest.mark.asyncio async def test_cli_poll_key_mints_jwt_with_selected_team_grants(self): """The selected team's grants must reach the mint, not just its alias.""" @@ -6692,72 +6725,57 @@ class TestPKCEFunctionality: class TestAddMissingTeamMember: """Tests for the add_missing_team_member function""" + @staticmethod + def _recording_add() -> tuple[ + list[tuple[str, str | None]], + Callable[[str, NewUserResponse | LiteLLM_UserTable], Awaitable[None]], + ]: + added: Final[list[tuple[str, str | None]]] = [] + + async def add(team_id: str, user_info: NewUserResponse | LiteLLM_UserTable) -> None: + added.append((team_id, user_info.user_id)) + + return added, add + @pytest.mark.asyncio async def test_add_missing_team_member_with_new_user_response_teams_none(self): - """ - Bug reproduction: When a NewUserResponse has teams=None (new SSO user), - add_missing_team_member() should still add the user to the SSO teams. - - Currently FAILS: The function returns early when teams is None. - """ + """A new SSO user's NewUserResponse has teams=None, and the user must still be added to the SSO teams.""" from litellm.proxy._types import NewUserResponse from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member - # Simulate a new SSO user - NewUserResponse has teams=None by default - new_user = NewUserResponse( - user_id="new-sso-user-123", - key="sk-xxxxx", - teams=None, # This is the default for NewUserResponse + new_user = NewUserResponse(user_id="new-sso-user-123", key="sk-xxxxx", teams=None) + added, add = self._recording_add() + + await add_missing_team_member( + user_info=new_user, + sso_teams=["team-from-entra-1", "team-from-entra-2"], + add_team_member=add, ) - sso_teams = ["team-from-entra-1", "team-from-entra-2"] - - with patch( - "litellm.proxy.management_endpoints.ui_sso.create_team_member_add_task" - ) as mock_add_task: - mock_add_task.return_value = AsyncMock() - - await add_missing_team_member(user_info=new_user, sso_teams=sso_teams) - - # Bug: This assertion currently FAILS - no teams are added - # because function returns early when teams is None - assert ( - mock_add_task.call_count == 2 - ), f"Expected 2 calls to add user to teams, but got {mock_add_task.call_count}" - called_team_ids = [call.args[0] for call in mock_add_task.call_args_list] - assert set(called_team_ids) == { - "team-from-entra-1", - "team-from-entra-2", - } + assert added == [ + ("team-from-entra-1", "new-sso-user-123"), + ("team-from-entra-2", "new-sso-user-123"), + ] @pytest.mark.asyncio async def test_add_missing_team_member_with_litellm_user_table_empty_teams(self): - """ - Control test: When a LiteLLM_UserTable has teams=[] (existing user, no teams), - add_missing_team_member() should add the user to SSO teams. - - This test PASSES because LiteLLM_UserTable defaults teams to [] not None. - """ + """An existing user with teams=[] is added to every SSO team.""" from litellm.proxy._types import LiteLLM_UserTable from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member - # Existing user has teams=[] by default (not None) - existing_user = LiteLLM_UserTable( - user_id="existing-user-456", - teams=[], # Empty list, not None + existing_user = LiteLLM_UserTable(user_id="existing-user-456", teams=[]) + added, add = self._recording_add() + + await add_missing_team_member( + user_info=existing_user, + sso_teams=["team-from-entra-1", "team-from-entra-2"], + add_team_member=add, ) - sso_teams = ["team-from-entra-1", "team-from-entra-2"] - - with patch( - "litellm.proxy.management_endpoints.ui_sso.create_team_member_add_task" - ) as mock_add_task: - mock_add_task.return_value = AsyncMock() - - await add_missing_team_member(user_info=existing_user, sso_teams=sso_teams) - - # This PASSES - teams are added because teams=[] not None - assert mock_add_task.call_count == 2 + assert added == [ + ("team-from-entra-1", "existing-user-456"), + ("team-from-entra-2", "existing-user-456"), + ] @pytest.mark.asyncio async def test_add_user_to_teams_from_sso_response_new_user(self): @@ -6800,96 +6818,160 @@ class TestAddMissingTeamMember: @pytest.mark.asyncio async def test_sso_first_login_full_flow_adds_user_to_teams(self): - """ - End-to-end test: Simulates complete first-time SSO login with Entra groups. - Verifies teams are created AND user is added as a member. - """ + """A first-time SSO login with Entra groups adds the new user as a member of every group's team.""" from litellm.proxy._types import NewUserResponse from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member - team_member_calls = [] + new_user = NewUserResponse(user_id="first-time-sso-user", key="sk-xxxxx", teams=None) + added, add = self._recording_add() - async def track_team_member_add(team_id, user_info): - team_member_calls.append({"team_id": team_id, "user_id": user_info.user_id}) - - # New SSO user with Entra groups - new_user = NewUserResponse( - user_id="first-time-sso-user", - key="sk-xxxxx", - teams=None, # The problematic default + await add_missing_team_member( + user_info=new_user, + sso_teams=["entra-team-alpha", "entra-team-beta"], + add_team_member=add, ) - sso_teams = ["entra-team-alpha", "entra-team-beta"] - - with patch( - "litellm.proxy.management_endpoints.ui_sso.create_team_member_add_task", - side_effect=track_team_member_add, - ): - await add_missing_team_member(user_info=new_user, sso_teams=sso_teams) - - # Bug: With current code, team_member_calls will be empty - # After fix: Should have 2 entries - assert ( - len(team_member_calls) == 2 - ), f"Expected 2 teams to be added, but got {len(team_member_calls)}" - assert {c["team_id"] for c in team_member_calls} == { - "entra-team-alpha", - "entra-team-beta", - } - assert all(c["user_id"] == "first-time-sso-user" for c in team_member_calls) + assert added == [ + ("entra-team-alpha", "first-time-sso-user"), + ("entra-team-beta", "first-time-sso-user"), + ] @pytest.mark.asyncio @pytest.mark.parametrize( - "user_info_factory,teams_value,expected_teams_added", + "user_info_factory,expected_teams_added", [ - # Bug case: NewUserResponse with teams=None pytest.param( lambda uid: NewUserResponse(user_id=uid, key="sk-xxx", teams=None), - None, - ["team-1", "team-2"], # Should still add teams + ["team-1", "team-2"], id="new_user_teams_none", ), - # Working case: LiteLLM_UserTable with teams=[] pytest.param( lambda uid: LiteLLM_UserTable(user_id=uid, teams=[]), - [], ["team-1", "team-2"], id="existing_user_empty_teams", ), - # Existing user with some teams already pytest.param( lambda uid: LiteLLM_UserTable(user_id=uid, teams=["team-1"]), - ["team-1"], - ["team-2"], # Only missing team should be added + ["team-2"], id="existing_user_partial_teams", ), + pytest.param( + lambda uid: LiteLLM_UserTable(user_id=uid, teams=["team-2", "team-1"]), + [], + id="existing_user_all_teams", + ), ], ) async def test_add_missing_team_member_handles_all_user_types( - self, user_info_factory, teams_value, expected_teams_added + self, user_info_factory, expected_teams_added ): - """ - Parametrized test ensuring add_missing_team_member works for all user types. - """ + """Only the SSO teams the user is not already on are added.""" from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member - user_info = user_info_factory("test-user-id") - sso_teams = ["team-1", "team-2"] + added, add = self._recording_add() - added_teams = [] + await add_missing_team_member( + user_info=user_info_factory("test-user-id"), + sso_teams=["team-1", "team-2"], + add_team_member=add, + ) - async def mock_create_task(team_id, user): - added_teams.append(team_id) + assert added == [(team_id, "test-user-id") for team_id in expected_teams_added] - with patch( - "litellm.proxy.management_endpoints.ui_sso.create_team_member_add_task", - side_effect=mock_create_task, - ): - await add_missing_team_member(user_info=user_info, sso_teams=sso_teams) + @pytest.mark.asyncio + async def test_add_missing_team_member_lands_teams_in_claim_order(self): + """user.teams is appended to as each add finishes, and the first team on it is the one a CLI + login binds to, so the adds must finish in the claim's order even when an earlier add is slower. - assert set(added_teams) == set( - expected_teams_added - ), f"Expected teams {expected_teams_added}, but got {added_teams}" + The adds used to run concurrently from a set, so they finished in an arbitrary order. + """ + from litellm.proxy._types import LiteLLM_UserTable + from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member + + claim: Final = ["team-zeta", "team-alpha", "team-held", "team-zeta", "team-mu"] + yields_before_finishing: Final = {"team-zeta": 3, "team-alpha": 2, "team-mu": 1} + finished: Final[list[str]] = [] + + async def slower_when_earlier(team_id: str, user_info: NewUserResponse | LiteLLM_UserTable) -> None: + for _ in range(yields_before_finishing[team_id]): + await asyncio.sleep(0) + finished.append(team_id) + + await add_missing_team_member( + user_info=LiteLLM_UserTable(user_id="group-sync-user", teams=["team-held"]), + sso_teams=claim, + add_team_member=slower_when_earlier, + ) + + assert finished == ["team-zeta", "team-alpha", "team-mu"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "existing,expected_teams_added", + [ + pytest.param(frozenset({"team-zeta", "team-alpha"}), ["team-zeta", "team-alpha"], id="only_teams"), + pytest.param(None, ["grp-everyone", "team-zeta", "grp-vpn", "team-alpha"], id="lookup_failed"), + ], + ) + async def test_add_missing_team_member_looks_up_missing_groups_once_and_adds_only_teams( + self, existing: frozenset[str] | None, expected_teams_added: list[str] + ): + from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member + + added, add = self._recording_add() + lookups: Final[list[tuple[str, ...]]] = [] + + async def existing_team_ids(team_ids: Sequence[str]) -> frozenset[str] | None: + lookups.append(tuple(team_ids)) + return existing + + await add_missing_team_member( + user_info=LiteLLM_UserTable(user_id="group-sync-user", teams=["team-held"]), + sso_teams=["grp-everyone", "team-zeta", "team-held", "grp-vpn", "team-alpha"], + add_team_member=add, + existing_team_ids=existing_team_ids, + ) + + assert lookups == [("grp-everyone", "team-zeta", "grp-vpn", "team-alpha")] + assert added == [(team_id, "group-sync-user") for team_id in expected_teams_added] + + @pytest.mark.asyncio + async def test_add_missing_team_member_skips_the_lookup_when_every_group_is_held(self): + from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member + + added, add = self._recording_add() + lookup: Final = AsyncMock(return_value=frozenset()) + + await add_missing_team_member( + user_info=LiteLLM_UserTable(user_id="group-sync-user", teams=["team-alpha", "team-zeta"]), + sso_teams=["team-zeta", "team-alpha"], + add_team_member=add, + existing_team_ids=lookup, + ) + + lookup.assert_not_awaited() + assert added == [] + + @pytest.mark.asyncio + async def test_find_existing_team_ids_reads_the_writer_and_returns_none_when_the_db_cannot_answer(self): + from litellm.proxy.management_endpoints.ui_sso import find_existing_team_ids + + stored_on_writer: Final = ("team-zeta", "team-alpha") + + async def find_many(*, where: Mapping[str, Mapping[str, Sequence[str]]]) -> list[SimpleNamespace]: + return [ + SimpleNamespace(team_id=team_id) for team_id in stored_on_writer if team_id in where["team_id"]["in"] + ] + + prisma: Final = MagicMock() + prisma.writer_db.litellm_teamtable.find_many = AsyncMock(side_effect=find_many) + prisma.db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + assert await find_existing_team_ids(("grp-vpn", "team-alpha")) == frozenset({"team-alpha"}) + prisma.writer_db.litellm_teamtable.find_many.side_effect = RuntimeError("db unavailable") + assert await find_existing_team_ids(("team-alpha",)) is None + with patch("litellm.proxy.proxy_server.prisma_client", None): + assert await find_existing_team_ids(("team-alpha",)) is None @pytest.mark.asyncio diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 4011bc0294a..7f8aac983f5 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -323,6 +323,35 @@ def test_openrouter_decisions_response_uses_provider_cost_without_cost_map_row( assert response["kwargs"]["combined_usage_object"].total_tokens == 302 +@pytest.mark.parametrize("reported_cost", [float("nan"), float("inf")]) +def test_openrouter_non_finite_cost_logs_the_token_price(monkeypatch: pytest.MonkeyPatch, reported_cost: float) -> None: + monkeypatch.setitem( + litellm.model_cost, + "openrouter/typesafe/jev-1.13", + {"input_cost_per_token": 1.0, "output_cost_per_token": 3.0, "litellm_provider": "openrouter", "mode": "chat"}, + ) + logging_obj: Final = _logging_obj() + + result: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={ + "model": "typesafe/jev-1.13-20260917", + "usage": {"input_tokens": 2, "output_tokens": 1, "cost": reported_cost}, + }, + logging_obj=logging_obj, + url_route="https://openrouter.ai/api/alpha/decisions", + result='{"answers": {}}', + start_time=datetime(2000, 1, 1), + end_time=datetime(2000, 1, 1), + cache_hit=False, + request_body={"model": "typesafe/jev-1.13"}, + custom_llm_provider="openrouter", + ) + + assert result["kwargs"]["response_cost"] == pytest.approx(5.0) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(5.0) + + def test_success_handler_dispatches_openrouter_to_the_shared_handler(): logging_obj = _logging_obj() normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( diff --git a/tests/unit/proxy/spend_tracking/test_savings.py b/tests/unit/proxy/spend_tracking/test_savings.py index f9566fd1ba0..8503a124b01 100644 --- a/tests/unit/proxy/spend_tracking/test_savings.py +++ b/tests/unit/proxy/spend_tracking/test_savings.py @@ -137,6 +137,34 @@ def test_compression_savings_priced_at_input_rate(): assert result.prompt_caching == 0.0 +def test_an_overflowing_deployment_price_records_finite_savings() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "overflowing-sonnet", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "input_cost_per_token": 1e308, + "cache_read_input_token_cost": 1e-07, + }, + }, + ] + ) + deployment_id: Final = router.get_model_list(model_name="overflowing-sonnet")[0]["model_info"]["id"] + + result: Final = compute_savings_spend( + model="claude-sonnet-5-5", + custom_llm_provider="anthropic", + compression_saved_tokens=4389, + gateway_injected_cache=True, + usage_object=_caching_usage(read=1000, written=0), + model_id=deployment_id, + llm_router=lambda: router, + ) + + assert result == (0.0, 0.0, 0.0, 0.0) + + def test_prompt_caching_savings_priced_at_input_minus_cache_read(): input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5") # A model that supports prompt caching must charge less to read from cache; diff --git a/tests/unit/proxy/spend_tracking/test_subscription_billing_periods.py b/tests/unit/proxy/spend_tracking/test_subscription_billing_periods.py new file mode 100644 index 00000000000..c2c98a05b3f --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_subscription_billing_periods.py @@ -0,0 +1,73 @@ +from datetime import date + +from litellm.proxy.spend_tracking.subscription_billing_periods import ( + BillingPeriodRange, + billing_periods_starting_within, +) + + +def test_one_period_per_month_start_inside_the_filter(): + periods = billing_periods_starting_within(date(2026, 8, 5), date(2026, 9, 1), date(2026, 10, 31)) + + assert periods == ( + BillingPeriodRange(start=date(2026, 9, 5), end=date(2026, 10, 4)), + BillingPeriodRange(start=date(2026, 10, 5), end=date(2026, 11, 4)), + ) + + +def test_filter_boundaries_are_inclusive(): + anchor = date(2026, 10, 1) + + assert billing_periods_starting_within(anchor, date(2026, 10, 1), date(2026, 10, 7)) == ( + BillingPeriodRange(start=date(2026, 10, 1), end=date(2026, 10, 31)), + ) + assert billing_periods_starting_within(anchor, date(2026, 9, 25), date(2026, 10, 1)) == ( + BillingPeriodRange(start=date(2026, 10, 1), end=date(2026, 10, 31)), + ) + + +def test_filter_between_two_period_starts_attributes_nothing(): + assert billing_periods_starting_within(date(2026, 10, 15), date(2026, 10, 1), date(2026, 10, 7)) == () + + +def test_no_period_before_the_anchor(): + assert billing_periods_starting_within(date(2026, 10, 1), date(2026, 8, 1), date(2026, 9, 30)) == () + assert billing_periods_starting_within(date(2026, 10, 1), date(2026, 8, 1), date(2026, 10, 1)) == ( + BillingPeriodRange(start=date(2026, 10, 1), end=date(2026, 10, 31)), + ) + + +def test_anchor_day_clamps_to_short_months_without_drifting(): + periods = billing_periods_starting_within(date(2026, 1, 31), date(2026, 2, 1), date(2026, 5, 31)) + + assert [period.start for period in periods] == [ + date(2026, 2, 28), + date(2026, 3, 31), + date(2026, 4, 30), + date(2026, 5, 31), + ] + assert periods[0].end == date(2026, 3, 30) + assert periods[-1].end == date(2026, 6, 29) + + +def test_period_ends_the_day_before_the_next_start_across_a_year_boundary(): + periods = billing_periods_starting_within(date(2025, 12, 20), date(2025, 12, 1), date(2026, 1, 31)) + + assert periods == ( + BillingPeriodRange(start=date(2025, 12, 20), end=date(2026, 1, 19)), + BillingPeriodRange(start=date(2026, 1, 20), end=date(2026, 2, 19)), + ) + + +def test_inverted_filter_is_empty(): + assert billing_periods_starting_within(date(2026, 1, 1), date(2026, 3, 1), date(2026, 2, 1)) == () + + +def test_the_calendar_s_last_month_ends_on_its_last_day_instead_of_failing(): + assert billing_periods_starting_within(date(9999, 11, 15), date(9999, 11, 1), date(9999, 12, 31)) == ( + BillingPeriodRange(start=date(9999, 11, 15), end=date(9999, 12, 14)), + BillingPeriodRange(start=date(9999, 12, 15), end=date(9999, 12, 31)), + ) + assert billing_periods_starting_within(date(9999, 12, 1), date(9999, 12, 1), date(9999, 12, 31)) == ( + BillingPeriodRange(start=date(9999, 12, 1), end=date(9999, 12, 31)), + ) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 2d08609c93b..a7596b51e3e 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -83,6 +83,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import ( team_member_budget_entity_id, ) from litellm.proxy.utils import ProxyLogging +from litellm.responses.mcp.parked_stream_dispatch import IN_FLIGHT_ROUND from litellm.router import Router from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.utils import ModelResponseListIterator @@ -9051,6 +9052,33 @@ class TestStreamingClientDisconnectBilling: proxy_logging_obj.arelease_max_parallel_requests_on_disconnect.assert_awaited_once() + @pytest.mark.asyncio + async def test_disconnect_whose_in_flight_round_billing_raises_still_closes_the_stream_and_frees_the_slot(self): + import types + + response = await self._start_partial_stream() + closed: Final = AsyncMock() + + async def cannot_price_the_in_flight_round() -> None: + raise RuntimeError("the in-flight round could not be priced") + + setattr(response.logging_obj, IN_FLIGHT_ROUND, cannot_price_the_in_flight_round) + stream: Final = types.SimpleNamespace(chunks=[], messages=None, aclose=closed) + proxy_logging_obj: Final = types.SimpleNamespace(arelease_max_parallel_requests_on_disconnect=AsyncMock()) + + await ProxyBaseLLMRequestProcessing._finalize_streaming_generator_cleanup( + request=None, + request_data={"litellm_logging_obj": response.logging_obj}, + response=stream, + stream_completed=False, + client_disconnected=True, + user_api_key_dict=MagicMock(), + proxy_logging_obj=proxy_logging_obj, + ) + + closed.assert_awaited_once() + proxy_logging_obj.arelease_max_parallel_requests_on_disconnect.assert_awaited_once() + async def _bill_and_collect_success_event(self, prepare=None, request_data=None): recorder = _RecordingSuccessLogger() original_callbacks = litellm.callbacks @@ -12468,6 +12496,55 @@ class TestResponseCostFromLoggingObjForCacheHits: ) assert {name: headers.get(name) for name in cost_headers} == dict.fromkeys(cost_headers, expected_header) + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("response_cost", "expected_header"), + ((float("nan"), None), (float("inf"), None), (0.0042, "0.0042"), (2, "2")), + ids=("nan", "inf", "finite", "int"), + ) + async def test_a_non_finite_response_cost_sends_no_cost_header( + self, response_cost: float, expected_header: str | None + ): + response: Final = litellm.ModelResponse() + response._hidden_params = {"response_cost": response_cost} + + headers: Final = await ProxyBaseLLMRequestProcessing.build_litellm_proxy_success_headers_from_llm_response( + response=response, + request_data={"model": "gpt-5.4-mini"}, + request=Request({"type": "http", "method": "POST", "headers": []}), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + logging_obj=self._logging_obj(), + version=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + + assert headers.get("x-litellm-response-cost") == expected_header + + @pytest.mark.asyncio + async def test_an_overflowing_cost_breakdown_sends_no_infinite_cost_header(self): + response: Final = litellm.ModelResponse() + response._hidden_params = {"response_cost": 0.0} + logging_obj: Final = self._logging_obj() + logging_obj.cost_breakdown = { + "original_cost": float("inf"), + "input_cost": float("inf"), + "output_cost": 0.0012, + "total_cost": float("inf"), + } + + headers: Final = await ProxyBaseLLMRequestProcessing.build_litellm_proxy_success_headers_from_llm_response( + response=response, + request_data={"model": "gpt-5.4-mini"}, + request=Request({"type": "http", "method": "POST", "headers": []}), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + logging_obj=logging_obj, + version=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + + cost_headers: Final = {name: value for name, value in headers.items() if name.startswith("x-litellm-response-cost")} + assert cost_headers == {"x-litellm-response-cost-output": "0.0012"} + def test_a_fresh_zero_cost_v1_messages_call_still_sends_no_cost_header(self): logging_obj: Final = self._logging_obj() logging_obj.model_call_details["response_cost"] = 0.0 diff --git a/tests/unit/proxy/test_fallback_management_endpoints.py b/tests/unit/proxy/test_fallback_management_endpoints.py index d28d61af72b..e3d0d072e0a 100644 --- a/tests/unit/proxy/test_fallback_management_endpoints.py +++ b/tests/unit/proxy/test_fallback_management_endpoints.py @@ -12,7 +12,7 @@ import copy import json from collections.abc import AsyncIterator, Mapping from types import MappingProxyType, SimpleNamespace -from typing import Final, get_args +from typing import Final, Literal, get_args from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -20,6 +20,7 @@ from fastapi import HTTPException from pydantic import JsonValue, TypeAdapter from redis.exceptions import NoPermissionError +import litellm from litellm import Router from litellm.proxy.config_resolvers.settings_rules import Section from litellm.proxy.management_endpoints.fallback_management_endpoints import ( @@ -388,6 +389,88 @@ class TestFallbackWritesSeeTheLatestStoredRules: assert stored.stored_fallbacks() == [GATEWAY_RULE, TEAM_RULE] + async def test_content_policy_writes_keep_the_litellm_settings_rules_for_other_models( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "content_policy_fallbacks", [TEAM_RULE]) + router: Final = _team_router() + stored: Final = _StoredRouterSettings(None) + with ( + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client), + patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + await create_fallback( + FallbackCreateRequest( + model="gpt-5.4-mini", fallback_models=["team-fallback"], fallback_type="content_policy" + ), + MagicMock(), + ) + after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock()) + await delete_fallback("gpt-5.4-mini", "content_policy", MagicMock()) + after_delete: Final = await get_fallback("team-primary", "content_policy", MagicMock()) + + assert after_create.fallback_models == ["team-fallback"] + assert after_delete.fallback_models == ["team-fallback"] + + async def test_a_content_policy_write_applies_beside_an_invalid_stored_rule(self) -> None: + router: Final = _team_router() + stored: Final = _StoredRouterSettings({"content_policy_fallbacks": [MALFORMED_GATEWAY_RULE]}) + with ( + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client), + patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + await create_fallback( + FallbackCreateRequest( + model="team-primary", fallback_models=["team-fallback"], fallback_type="content_policy" + ), + MagicMock(), + ) + after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock()) + + assert after_create.fallback_models == ["team-fallback"] + + @pytest.mark.parametrize( + ("fallback_type", "router_setting"), + [("general", "fallbacks"), ("content_policy", "content_policy_fallbacks")], + ) + async def test_get_names_the_model_of_a_dict_target( + self, fallback_type: Literal["general", "content_policy"], router_setting: str + ) -> None: + router: Final = _team_router() + router.update_settings( + **{router_setting: [{"team-primary": [{"model": "gpt-5.4-mini", "api_key": "sk-test"}, "team-fallback"]}]} + ) + with patch("litellm.proxy.proxy_server.llm_router", router): + response: Final = await get_fallback("team-primary", fallback_type, MagicMock()) + + assert response.fallback_models == ["gpt-5.4-mini", "team-fallback"] + + @pytest.mark.parametrize( + ("fallback_type", "router_setting"), + [("general", "fallbacks"), ("content_policy", "content_policy_fallbacks")], + ) + async def test_get_names_the_rules_own_model_for_a_dict_target_that_names_none( + self, fallback_type: Literal["general", "content_policy"], router_setting: str + ) -> None: + router: Final = _team_router() + router.update_settings(**{router_setting: [{"team-primary": [{"temperature": 0}, "team-fallback"]}]}) + with patch("litellm.proxy.proxy_server.llm_router", router): + response: Final = await get_fallback("team-primary", fallback_type, MagicMock()) + + assert response.fallback_models == ["team-primary", "team-fallback"] + + async def test_get_answers_an_error_for_a_string_valued_rule_instead_of_its_characters(self) -> None: + router: Final = _team_router() + router.update_settings(fallbacks=[{"team-primary": "team-fallback"}]) + with patch("litellm.proxy.proxy_server.llm_router", router), pytest.raises(HTTPException) as raised: + await get_fallback("team-primary", "general", MagicMock()) + + assert raised.value.status_code == 500 + async def test_a_create_keeps_a_rule_another_instance_stored_since_this_one_last_read(self) -> None: stored: Final = _StoredRouterSettings({}) await get_config_param(stored.prisma_client, "router_settings") @@ -568,6 +651,21 @@ class TestFallbackWritesHonourTheConfigFile: assert stored.row is None assert router.fallbacks == [CONFIG_RULE, CONFIG_WINDOW_RULE] + async def test_a_content_policy_restate_keeps_the_config_list_without_the_litellm_settings_rules( + self, router: Router, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "content_policy_fallbacks", [TEAM_RULE]) + stored: Final = _StoredRouterSettings(None, config_file={"content_policy_fallbacks": [CONFIG_RULE]}) + request: Final = FallbackCreateRequest( + model="gpt-5.4-mini", fallback_models=["team-fallback"], fallback_type="content_policy" + ) + + response: Final = await self._create(request, router, stored) + + assert response.fallback_models == ["team-fallback"] + assert stored.row is None + assert router.content_policy_fallbacks == [CONFIG_RULE] + async def test_a_create_that_restates_a_config_rule_next_to_an_env_reference_writes_nothing( self, router: Router, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/proxy/test_health_check_functions.py b/tests/unit/proxy/test_health_check_functions.py index 2c0198cd538..79eec44958f 100644 --- a/tests/unit/proxy/test_health_check_functions.py +++ b/tests/unit/proxy/test_health_check_functions.py @@ -741,6 +741,7 @@ async def test_perform_health_check_and_save_passes_model_id_to_perform_health_c result = await _perform_health_check_and_save( model_list=model_list, target_model=None, + recorded_model_name=None, cli_model=None, details=True, prisma_client=None, @@ -777,6 +778,7 @@ async def test_perform_health_check_and_save_forwards_skip_disabled_background_f await _perform_health_check_and_save( model_list=model_list, target_model=None, + recorded_model_name=None, cli_model=None, details=True, prisma_client=None, diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index df0429316ed..26d9fe2fb3d 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1,5 +1,6 @@ import asyncio import copy +import io import json import logging from typing import Final @@ -1564,3 +1565,84 @@ async def test_a_failing_health_probe_leaves_the_deployment_serving_requests(mon assert "error" in probe assert reply.choices[0].message.content == "ok" + + +@pytest.mark.parametrize( + ("component_params", "probed_component"), + [ + ({"extra_headers": {"X-Amzn-SageMaker-Inference-Component": "header-component"}}, "header-component"), + ( + {"model_id": "own-component", "extra_headers": {"X-Amzn-SageMaker-Inference-Component": "header-component"}}, + "own-component", + ), + ( + {"model_id": "own-component", "extra_headers": {1: "other", "X-Amzn-SageMaker-Inference-Component": "x"}}, + "own-component", + ), + ( + {"extra_headers": {1: "other", "X-Amzn-SageMaker-Inference-Component": "header-component"}}, + "header-component", + ), + ( + { + "configurable_clientside_auth_params": ["model_id"], + "extra_headers": {"X-Amzn-SageMaker-Inference-Component": "header-component"}, + }, + None, + ), + ], + ids=[ + "header_pinned", + "own_model_id_beats_the_header", + "own_model_id_beside_a_non_string_header_key", + "header_pinned_beside_a_non_string_header_key", + "client_configurable_model_id_takes_no_header_pin", + ], +) +@pytest.mark.asyncio +async def test_run_model_health_check_probes_a_legacy_sagemaker_embedding_at_the_deployments_component( + component_params: dict[str, object], probed_component: str | None +) -> None: + embedding_reply: Final = {"object": "list", "data": [{"object": "embedding", "index": 0, "embedding": [0.1]}]} + invoke_endpoint: Final = MagicMock(return_value={"Body": io.BytesIO(json.dumps(embedding_reply).encode())}) + client: Final = MagicMock(invoke_endpoint=invoke_endpoint) + deployment: Final = { + "litellm_params": { + "model": "sagemaker/openai/embed-endpoint", + "aws_region_name": "us-east-1", + "aws_access_key_id": "test-key", + "aws_secret_access_key": "test-secret", + **component_params, + }, + "model_info": {"mode": "embedding"}, + } + + with patch("boto3.Session", return_value=MagicMock(client=MagicMock(return_value=client))): + result: Final = await hc_module._run_model_health_check(deployment) + + assert "error" not in result, result + assert invoke_endpoint.call_count == 1 + assert invoke_endpoint.call_args.kwargs.get("InferenceComponentName") == probed_component + + +@pytest.mark.asyncio +async def test_run_model_health_check_reports_why_a_string_configurable_clientside_auth_params_fails() -> None: + invoke_endpoint: Final = MagicMock() + deployment: Final = { + "litellm_params": { + "model": "sagemaker/openai/embed-endpoint", + "model_id": "own-component", + "aws_region_name": "us-east-1", + "aws_access_key_id": "test-key", + "aws_secret_access_key": "test-secret", + }, + "model_info": {"mode": "embedding", "health_check_params": {"configurable_clientside_auth_params": "model_id"}}, + } + + client: Final = MagicMock(invoke_endpoint=invoke_endpoint) + + with patch("boto3.Session", return_value=MagicMock(client=MagicMock(return_value=client))): + result: Final = await hc_module._run_model_health_check(deployment) + + assert "configurable_clientside_auth_params" in result["error"] + invoke_endpoint.assert_not_called() diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 2edb032b048..c5136ff6e4a 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -10143,9 +10143,12 @@ class TestResolveUserProviderCredentials: assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" @pytest.mark.asyncio - async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypatch): - """The proxy prefers key/team router_settings.fallbacks over router-level - fallbacks (_configured_fallbacks), so discovery must read them too.""" + @pytest.mark.parametrize("fallback_key", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) + async def test_key_router_settings_fallback_loads_per_user_tokens( + self, monkeypatch: pytest.MonkeyPatch, fallback_key: str + ): + """The proxy forwards key/team router_settings fallback lists into the + request after discovery, so discovery must read every one of them.""" from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request table = self._env(monkeypatch) @@ -10175,7 +10178,7 @@ class TestResolveUserProviderCredentials: authenticated_user_id="user-a", team_id=None, llm_router=router, - router_settings={"fallbacks": [{"gpt-4o": [{"model": "copilot-chat"}]}]}, + router_settings={fallback_key: [{"gpt-4o": [{"model": "copilot-chat"}]}]}, ) table.find_many.assert_awaited() assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 62a90109d26..bc3a228321a 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -10,6 +10,7 @@ import subprocess import sys import time import types +from collections.abc import Callable from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final @@ -5583,6 +5584,215 @@ async def test_valid_db_routing_groups_still_replace_router_groups(): assert router._get_routing_context("m2", None)[0] == "least-busy" +def _content_policy_router(**router_kwargs: object) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake", "mock_response": "primary answered"}, + }, + { + "model_name": "secondary", + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake", "mock_response": "secondary answered"}, + }, + { + "model_name": "backup", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake", "mock_response": "backup answered"}, + }, + ], + num_retries=0, + **router_kwargs, + ) + + +def _router_settings_row(content_policy_fallbacks: object) -> MagicMock: + row: Final = MagicMock() + row.param_value = {"allowed_fails": 7, "content_policy_fallbacks": content_policy_fallbacks} + prisma: Final = MagicMock() + prisma.db.litellm_config.find_first = AsyncMock(return_value=row) + return prisma + + +async def _answer_after_content_policy_refusal(router: litellm.Router, model: str = "primary") -> str | None: + response: Final = await router.acompletion( + model=model, messages=[{"role": "user", "content": "hi"}], mock_testing_content_policy_fallbacks=True + ) + return response.choices[0].message.content + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_target", ["backup", {"model": "backup"}]) +async def test_db_content_policy_fallbacks_reach_the_router(stored_target: str | dict[str, str]): + router: Final = _content_policy_router() + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([{"primary": [stored_target]}]) + ) + + assert await _answer_after_content_policy_refusal(router) == "backup answered" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_rules", [[], [{"primary": "backup"}]]) +async def test_a_saved_content_policy_list_with_no_valid_rule_leaves_refusals_to_the_general_fallbacks( + monkeypatch: pytest.MonkeyPatch, stored_rules: list[object] +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", None) + router: Final = _content_policy_router(fallbacks=[{"primary": ["backup"]}]) + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(stored_rules) + ) + + assert await _answer_after_content_policy_refusal(router) == "backup answered" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("litellm_settings_rules", "stored_rules", "refused_model"), + [ + ([{"secondary": ["backup"]}], [{"primary": ["backup"]}], "secondary"), + ([{"*": ["secondary"]}], [{"*": ["backup"]}], "primary"), + ([{"primary": ["backup"]}], [], "primary"), + ], +) +async def test_db_content_policy_fallbacks_layer_over_the_litellm_settings_rules( + monkeypatch: pytest.MonkeyPatch, + litellm_settings_rules: list[dict[str, list[str]]], + stored_rules: list[dict[str, list[str]]], + refused_model: str, +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", litellm_settings_rules) + router: Final = _content_policy_router() + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(stored_rules) + ) + + assert await _answer_after_content_policy_refusal(router, refused_model) == "backup answered" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "invalid_rule", + [{"secondary": "backup"}, ["backup"], dict[str, list[str]](), {"secondary": ["backup"], "primary": ["secondary"]}], +) +async def test_an_invalid_db_content_policy_rule_skips_only_itself(monkeypatch: pytest.MonkeyPatch, invalid_rule: object): + monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup"]}]) + router: Final = _content_policy_router() + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([invalid_rule, {"secondary": ["backup"]}]) + ) + + assert router.allowed_fails == 7 + assert await _answer_after_content_policy_refusal(router) == "backup answered" + assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered" + + +@pytest.mark.asyncio +async def test_a_db_content_policy_value_that_is_not_a_list_keeps_the_litellm_settings_rules( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup"]}]) + router: Final = _content_policy_router() + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row("oops") + ) + + assert router.allowed_fails == 7 + assert await _answer_after_content_policy_refusal(router) == "backup answered" + + +@pytest.mark.asyncio +async def test_a_db_sync_keeps_a_litellm_settings_rule_with_a_target_router_init_accepted( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup", 5]}]) + router: Final = _content_policy_router() + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([{"secondary": ["backup"]}]) + ) + + assert await _answer_after_content_policy_refusal(router) == "backup answered" + assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered" + + +@pytest.mark.asyncio +async def test_an_invalid_db_content_policy_rule_is_logged_once_per_change_of_the_rules( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", []) + router: Final = _content_policy_router() + proxy_config: Final = proxy_server_module.ProxyConfig() + stored_rules: Final = [{"secondary": "backup"}, {"primary": ["backup"]}] + changed_rules: Final = [{"secondary": "backup"}, {"secondary": ["backup"]}] + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(stored_rules) + ) + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(stored_rules) + ) + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(changed_rules) + ) + + notices: Final = [ + record for record in caplog.records if "Ignoring invalid content_policy_fallbacks" in record.getMessage() + ] + assert len(notices) == 2 + assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered" + + +@pytest.mark.asyncio +async def test_a_db_sync_logs_a_new_invalid_content_policy_rule_when_no_valid_rule_changes( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", []) + router: Final = _content_policy_router() + proxy_config: Final = proxy_server_module.ProxyConfig() + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([{"secondary": "backup"}]) + ) + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([{"secondary": "backup"}, {"primary": "backup"}]) + ) + + notices: Final = [ + record.getMessage() + for record in caplog.records + if "Ignoring invalid content_policy_fallbacks" in record.getMessage() + ] + assert len(notices) == 3 + assert "{'primary': 'backup'}" in notices[-1] + + +@pytest.mark.asyncio +async def test_config_file_content_policy_fallbacks_stay_the_only_rules_after_a_db_sync( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"secondary": ["backup"]}]) + file_rules: Final = [{"primary": ["backup"]}] + router: Final = _content_policy_router(content_policy_fallbacks=file_rules) + proxy_config: Final = proxy_server_module.ProxyConfig() + proxy_config.router_settings.load_yaml({"content_policy_fallbacks": file_rules}) + + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([{"secondary": ["backup"]}]) + ) + + assert router.allowed_fails == 7 + assert await _answer_after_content_policy_refusal(router) == "backup answered" + with pytest.raises(litellm.ContentPolicyViolationError): + await _answer_after_content_policy_refusal(router, "secondary") + + @pytest.mark.asyncio async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_config_fallbacks(): """ @@ -5591,6 +5801,7 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_ empty lists are "no value" and must not clobber config.yaml fallbacks, matching _deep_merge_dicts semantics. Non-empty DB lists still win. """ + from collections import ChainMap from unittest.mock import AsyncMock, MagicMock from litellm.proxy.proxy_server import ProxyConfig @@ -5625,10 +5836,10 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_ prisma_client=mock_prisma_client, ) - combined_settings = mock_router.update_settings.call_args.kwargs + combined_settings = ChainMap(*(call.kwargs for call in reversed(mock_router.update_settings.call_args_list))) assert combined_settings["fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] assert combined_settings["context_window_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] - assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] + assert combined_settings.get("content_policy_fallbacks") != [{"gpt-oss-120b": ["other-model"]}] assert combined_settings["num_retries"] == 3 @@ -9997,6 +10208,34 @@ def test_update_config_accepts_disjoint_routing_groups(_update_config_setup): restore() +def test_update_config_validates_content_policy_fallbacks_like_fallbacks( + _update_config_setup: Callable[..., tuple[TestClient, _FakePrismaClient, Callable[[], None]]], +): + client, prisma, restore = _update_config_setup(initial_rows={"router_settings": {"num_retries": 2}}) + rule: Final = [{"gpt-5.4-mini": ["gpt-5.4-nano", {"model": "gpt-5.4-nano", "temperature": 1}]}] + malformed: Final = ( + "oops", + 5, + [1, 2], + [{"gpt-5.4-mini": "gpt-5.4-nano"}], + [{"gpt-5.4-mini": ["gpt-5.4-nano", 7]}], + [dict[str, list[str]]()], + [{"gpt-5.4-mini": ["gpt-5.4-nano"], "gpt-5.4": ["gpt-5.4-nano"]}], + ) + try: + rejected: Final = tuple( + client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": value}}).status_code + for value in malformed + ) + assert rejected == (422,) * len(malformed) + assert prisma.db.litellm_config.rows["router_settings"] == {"num_retries": 2} + accepted: Final = client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": rule}}) + assert accepted.status_code == 200 + assert prisma.db.litellm_config.rows["router_settings"] == {"num_retries": 2, "content_policy_fallbacks": rule} + finally: + restore() + + def test_update_config_env_var_round_trip_not_double_encrypted(_update_config_setup, monkeypatch): """Endpoint-level regression for the /config/update double-encryption bug. diff --git a/tests/unit/proxy/utils/helpers/test_error_helpers.py b/tests/unit/proxy/utils/helpers/test_error_helpers.py index df399c8b1d2..b20857dbdaf 100644 --- a/tests/unit/proxy/utils/helpers/test_error_helpers.py +++ b/tests/unit/proxy/utils/helpers/test_error_helpers.py @@ -1,13 +1,20 @@ import asyncio import json +from typing import Final from unittest.mock import AsyncMock, MagicMock +import httpx +import openai import pytest from fastapi import HTTPException from prisma.errors import DataError +import litellm from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy.common_request_processing import proxy_exception_from_route_error +from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.utils import get_error_message_str, handle_exception_on_proxy +from litellm.types.router import RouterErrors def normalize(value): @@ -96,7 +103,7 @@ def test_handle_exception_on_proxy_happy_path_http_exception(): assert snapshot == { "is_proxy_exception": True, "message": "forbidden", - "type": ProxyErrorTypes.internal_server_error.value, + "type": "auth_error", "code": "403", } @@ -156,7 +163,7 @@ def test_handle_exception_on_proxy_uses_attached_status_code_when_present(): assert snapshot == { "code": "418", "message": "teapot", - "type": ProxyErrorTypes.internal_server_error.value, + "type": "invalid_request_error", } @@ -176,6 +183,176 @@ def test_handle_exception_on_proxy_error_path_none_input_wraps_as_500(): } +@pytest.mark.parametrize( + "status_code, expected_type", + [ + (400, "invalid_request_error"), + (401, "auth_error"), + (403, "auth_error"), + (404, "invalid_request_error"), + (408, "invalid_request_error"), + (409, "invalid_request_error"), + (422, "invalid_request_error"), + (429, "rate_limit_error"), + (500, ProxyErrorTypes.internal_server_error.value), + (503, ProxyErrorTypes.internal_server_error.value), + ], +) +def test_handle_exception_on_proxy_labels_an_http_exception_by_the_status_it_answers( + status_code: int, expected_type: str +): + """A management route that rejects a caller's own argument answers 400, and calling that + internal_server_error tells the caller's retry loop the gateway is down when nothing but + their request is wrong.""" + result: Final = handle_exception_on_proxy(HTTPException(status_code=status_code, detail="boom")) + + assert (result.code, result.type) == (str(status_code), expected_type) + + +def _exception_carrying_status(carried: int | None, message: str = "boom") -> Exception: + class _CarriedStatus(Exception): + status_code = carried + + return _CarriedStatus(message) + + +@pytest.mark.parametrize( + "status_code, expected_type", + [ + (400, "invalid_request_error"), + (404, "invalid_request_error"), + (408, "invalid_request_error"), + (429, "rate_limit_error"), + (500, ProxyErrorTypes.internal_server_error.value), + ], +) +def test_handle_exception_on_proxy_labels_a_carried_status_the_same_way(status_code: int, expected_type: str): + """Routes also reach this handler with plain exceptions carrying their own status, and + those must be named by that status rather than all landing on the 500 label.""" + result: Final = handle_exception_on_proxy(_exception_carrying_status(status_code)) + + assert (result.code, result.type) == (str(status_code), expected_type) + + +def test_handle_exception_on_proxy_keeps_the_type_the_exception_already_names(): + """A rate limit litellm raised already names itself, and overwriting that with the status + map's guess would drop what the client branches on.""" + result: Final = handle_exception_on_proxy( + litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-4o") + ) + + assert (result.code, result.type) == ("429", "throttling_error") + + +@pytest.mark.parametrize( + "exc", + [ + openai.InternalServerError( + message="upstream down", + response=httpx.Response(500, request=httpx.Request("GET", "https://api.openai.com/v1/batches/b")), + body={"type": "server_error"}, + ), + litellm.Timeout(message="took too long", model="gpt-6.1-sol", llm_provider="openai"), + litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-6.1-sol"), + litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-6.1-sol"), + litellm.BadRequestError(message="bad input", model="gpt-6.1-sol", llm_provider="openai"), + HTTPException(status_code=404, detail="no such batch"), + ], + ids=["provider_5xx_naming_its_own_type", "timeout", "rate_limit", "bad_key", "bad_request", "http_404"], +) +def test_handle_exception_on_proxy_labels_a_failure_the_way_the_chat_route_does(exc: Exception): + """Batch, file, and management routes answer through this handler while chat answers through the route + error path, and one client calling both must read the same type for the same failure.""" + shared: Final = handle_exception_on_proxy(exc) + chat: Final = proxy_exception_from_route_error(exc) + + assert (shared.code, shared.type) == (chat.code, chat.type) + + +@pytest.mark.parametrize( + "exc", + [ + HTTPException(status_code=400, detail="bad budget_duration"), + HTTPException(status_code=401, detail="not allowed"), + HTTPException(status_code=403, detail="admin only"), + HTTPException(status_code=404, detail="no such team"), + litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-6.1-sol"), + ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['paid']"), + _exception_carrying_status(409, "duplicate credential"), + ], + ids=["http_400", "http_401", "http_403", "http_404", "provider_401", "tag_denial", "carried_409"], +) +def test_handle_exception_on_proxy_labels_a_failure_the_way_route_catch_alls_do(exc: Exception): + """Management routes answer some failures through their own catch-all and the rest through this handler, + so a client reading /key/list and /key/generate must see the same type and status for the same failure.""" + shared: Final = handle_exception_on_proxy(exc) + catch_all: Final = proxy_exception_for(exc, default_status_code=500) + + assert (shared.code, shared.type, shared.message) == (catch_all.code, catch_all.type, catch_all.message) + + +@pytest.mark.parametrize( + "message, expected", + [ + (f"{RouterErrors.no_deployments_with_provider_budget_routing.value}, try later", ("429", "rate_limit_error")), + (f"{RouterErrors.no_deployments_with_tag_routing.value}=['paid']", ("401", "auth_error")), + ], + ids=["no_deployment_left", "no_deployment_for_tags"], +) +def test_handle_exception_on_proxy_names_a_router_failure_by_the_status_it_answers( + message: str, expected: tuple[str, str] +): + """A router failure that carries no status answers the 429 or 401 its message maps to, and a 429 labelled + internal_server_error reads as an outage to a client that branches on the type.""" + result: Final = handle_exception_on_proxy(ValueError(message)) + + assert (result.code, result.type) == expected + + +def test_handle_exception_on_proxy_keeps_the_path_a_rejected_request_names(): + """A provider rejecting a batch input names the caller's own bucket path, and redacting the /tmp part of it + leaves the caller nothing to fix their request with.""" + result: Final = handle_exception_on_proxy( + litellm.BadRequestError( + message="Invalid input_file_id gs://my-bucket/tmp/batch_input.jsonl", + model="gemini-3.1-pro-preview", + llm_provider="vertex_ai", + ) + ) + + assert result.code == "400" + assert "gs://my-bucket/tmp/batch_input.jsonl" in result.message + + +def test_handle_exception_on_proxy_keeps_the_message_an_exception_with_a_status_carries(): + """Main sends a status-bearing failure's message verbatim, and relabeling its type must not change that text.""" + result: Final = handle_exception_on_proxy(_exception_carrying_status(503, "Cannot connect to host 10.12.4.7:8443")) + + assert (result.code, result.message) == ("503", "Cannot connect to host 10.12.4.7:8443") + + +@pytest.mark.parametrize( + "message", + [ + "boom", + "Cannot connect to host 10.12.4.7:8443", + "Cannot connect to host guardrail.internal.svc.cluster.local:443", + "Cannot connect to host [fd00::12]:8443", + ], + ids=["plain", "ipv4", "cluster_dns", "ipv6"], +) +def test_handle_exception_on_proxy_answers_a_generic_500_for_an_exception_whose_status_is_none(message: str): + """A guardrail whose upstream refuses the connection raises with status_code=None, which crashed the handler + into a 503 blaming the auth database, and its text names private hosts no virtual key was ever sent.""" + result: Final = handle_exception_on_proxy(_exception_carrying_status(None, message)) + + assert (result.code, result.type, result.message) == ( + "500", + ProxyErrorTypes.internal_server_error.value, + "Internal server error", + ) + + @pytest.mark.parametrize( "exc", [ diff --git a/tests/unit/repositories/test_repositories.py b/tests/unit/repositories/test_repositories.py index cbcd0b1cd28..4e410e22b7f 100644 --- a/tests/unit/repositories/test_repositories.py +++ b/tests/unit/repositories/test_repositories.py @@ -30,6 +30,11 @@ from litellm.repositories.object_permission_repository import ( ) from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.project_repository import ProjectRepository +from litellm.repositories.subscription_account_repository import ( + SetLabel, + SubscriptionAccountFeeChanges, + SubscriptionAccountRepository, +) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import ( @@ -140,6 +145,7 @@ class MockPrismaClient: self.db.litellm_projecttable = MockTable(pk_field="project_id") self.db.litellm_objectpermissiontable = MockTable(pk_field="object_permission_id") self.db.litellm_credentialstable = MockTable() + self.db.litellm_subscriptionaccounttable = MockTable(pk_field="subscription_account_id") class TestBaseRepository: @@ -2461,3 +2467,40 @@ class TestAutoRouterSessionRepository: assert AutoRouterSessionRepository(client).table is client.db.litellm_autoroutersession with pytest.raises(RuntimeError, match="No DB Connected"): _ = AutoRouterSessionRepository(None).table + + +class TestSubscriptionAccountRepository: + @pytest.mark.asyncio + async def test_update_fee_writes_only_the_named_columns(self): + client = MockPrismaClient() + stored_at = datetime(2026, 10, 1) + await client.db.litellm_subscriptionaccounttable.create( + data={ + "subscription_account_id": "sub-1", + "custom_llm_provider": "chatgpt", + "account_id": "acct_123", + "label": "Team plan (test value)", + "monthly_fee": 25.0, + "currency": "USD", + "billing_period_start": "2026-10-01", + "created_at": stored_at, + "created_by": "admin-a", + "updated_at": stored_at, + "updated_by": "admin-a", + } + ) + repo = SubscriptionAccountRepository(client) + + fee_only = await repo.update_fee("sub-1", SubscriptionAccountFeeChanges(monthly_fee=30.0), "admin-b") + label_cleared = await repo.update_fee("sub-1", SubscriptionAccountFeeChanges(label=SetLabel(None)), "admin-c") + missing = await repo.update_fee("sub-2", SubscriptionAccountFeeChanges(monthly_fee=1.0), "admin-b") + + assert fee_only is not None and label_cleared is not None + assert (fee_only.monthly_fee, fee_only.currency, fee_only.label, fee_only.updated_by) == ( + 30.0, + "USD", + "Team plan (test value)", + "admin-b", + ) + assert (label_cleared.monthly_fee, label_cleared.label, label_cleared.updated_by) == (30.0, None, "admin-c") + assert missing is None diff --git a/tests/unit/responses/mcp/test_chat_completions_handler.py b/tests/unit/responses/mcp/test_chat_completions_handler.py index ec5968c2a1a..c0d32742e8a 100644 --- a/tests/unit/responses/mcp/test_chat_completions_handler.py +++ b/tests/unit/responses/mcp/test_chat_completions_handler.py @@ -1,17 +1,25 @@ import asyncio +import gc import json +import queue import sys +import threading import types -from collections.abc import Mapping, Sequence +import warnings +from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager from datetime import datetime from types import MappingProxyType from typing import Final, cast -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import respx from httpx import Response from mcp.types import Tool as MCPTool +from starlette.responses import StreamingResponse +from starlette.types import Message import litellm from litellm.integrations.custom_logger import CustomLogger @@ -25,7 +33,8 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.utils import ModelResponse, StandardLoggingPayload +from litellm.types.utils import ModelResponse, ModelResponseStream, StandardLoggingPayload, Usage +from litellm.utils import CustomStreamWrapper @pytest.mark.asyncio @@ -1556,6 +1565,7 @@ _CHAT_ROUND_TWO: Final = MappingProxyType( ), } ) +_NEVER_LOGGED_SECONDS: Final = 60 _CONTEXT_WINDOW_ERROR: Final = MappingProxyType( { "error": MappingProxyType( @@ -1575,16 +1585,31 @@ class _ChatRoundRecorder(CustomLogger): super().__init__() self.logged: tuple[StandardLoggingPayload, ...] = () self.failed: tuple[StandardLoggingPayload, ...] = () + self._recorded: Final = asyncio.Condition() async def async_log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime ) -> None: self.logged = (*self.logged, cast(StandardLoggingPayload, kwargs["standard_logging_object"])) + await self._announce() async def async_log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime ) -> None: self.failed = (*self.failed, cast(StandardLoggingPayload, kwargs["standard_logging_object"])) + await self._announce() + + async def _announce(self) -> None: + async with self._recorded: + self._recorded.notify_all() + + async def settled(self, logged: int = 0, failed: int = 0) -> None: + async with self._recorded: + await asyncio.wait_for( + self._recorded.wait_for(lambda: len(self.logged) >= logged and len(self.failed) >= failed), + timeout=_NEVER_LOGGED_SECONDS, + ) + await _settled() def _chat_round_cost(body: Mapping[str, object]) -> float: @@ -1600,18 +1625,20 @@ async def _settled() -> None: await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10) -async def _auto_execute_chat_echo( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response], call_id: str -) -> tuple[_ChatRoundRecorder, ModelResponse | Exception]: +def _serve_chat_rounds( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response] +) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") litellm.in_memory_llm_clients_cache.flush_cache() respx_mock.post("https://mcp-chat-rounds.example.com/v1/chat/completions").mock(side_effect=list(rounds)) - recorder: Final = _ChatRoundRecorder() - logging_obj: Final = Logging( + + +def _chat_rounds_logging_obj(recorder: _ChatRoundRecorder, call_id: str, stream: bool) -> Logging: + return Logging( model="openai/gpt-5.6", messages=[{"role": "user", "content": "Call echo with PING"}], - stream=False, + stream=stream, call_type="acompletion", start_time=datetime(2026, 1, 1), litellm_call_id=call_id, @@ -1619,6 +1646,10 @@ async def _auto_execute_chat_echo( dynamic_async_success_callbacks=[recorder], dynamic_async_failure_callbacks=[recorder], ) + + +@contextmanager +def _echo_tool_served() -> Iterator[None]: echo_tool: Final = MCPTool.model_validate( {"name": "echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}}, by_name=False, @@ -1635,6 +1666,16 @@ async def _auto_execute_chat_echo( AsyncMock(return_value=[{"tool_call_id": "call_echo", "result": "PING", "name": "echo"}]), ), ): + yield + + +async def _auto_execute_chat_echo( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response], call_id: str +) -> tuple[_ChatRoundRecorder, ModelResponse | Exception]: + _serve_chat_rounds(respx_mock, monkeypatch, rounds) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, call_id, stream=False) + with _echo_tool_served(): try: response: Final = await litellm.acompletion( model="openai/gpt-5.6", @@ -1712,3 +1753,994 @@ async def test_non_stream_auto_execute_bills_the_paid_first_round_when_the_follo assert (recorder.failed[0]["prompt_tokens"], recorder.failed[0]["completion_tokens"]) == (71, 19) assert recorder.failed[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE)) assert recorder.failed[0]["response_cost"] > 0 + + +_FIXED_MARGIN: Final = 0.001 + + +def _chat_stream_events(body: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + completed: Final = json.loads(json.dumps(body, default=dict)) + choice: Final = completed["choices"][0] + message: Final = choice["message"] + calls: Final = message.get("tool_calls") or [] + delta: Final = ( + {**message, "tool_calls": [{**call, "index": index} for index, call in enumerate(calls)]} if calls else message + ) + head: Final = {"id": completed["id"], "object": "chat.completion.chunk", "created": 1, "model": completed["model"]} + return ( + {**head, "choices": [{"index": 0, "delta": delta, "finish_reason": None}]}, + {**head, "choices": [{"index": 0, "delta": {}, "finish_reason": choice["finish_reason"]}]}, + {**head, "choices": [], "usage": completed["usage"]}, + ) + + +def _sse(events: Sequence[Mapping[str, object]]) -> bytes: + return b"".join(f"data: {json.dumps(event)}\n\n".encode() for event in events) + + +def _streamed(body: Mapping[str, object]) -> Response: + return Response( + 200, + content=_sse(_chat_stream_events(body)) + b"data: [DONE]\n\n", + headers={"content-type": "text/event-stream"}, + ) + + +class _CutOnceTheAnswerStarts(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield _sse(_chat_stream_events(_CHAT_ROUND_TWO)[:1]) + raise httpx.ReadError("connection reset mid-stream") + + +def _cut_once_the_answer_starts() -> Response: + return Response(200, stream=_CutOnceTheAnswerStarts(), headers={"content-type": "text/event-stream"}) + + +async def _open_chat_echo_stream( + logging_obj: Logging, metadata: dict[str, object] | None = None +) -> CustomStreamWrapper: + stream: Final = await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + api_key="sk-mcp-chat-rounds", + api_base="https://mcp-chat-rounds.example.com/v1", + stream=True, + stream_options={"include_usage": True}, + litellm_logging_obj=logging_obj, + metadata=metadata, + ) + assert isinstance(stream, CustomStreamWrapper) + return stream + + +async def _read_all(stream: CustomStreamWrapper) -> tuple[ModelResponseStream, ...]: + return tuple([chunk async for chunk in stream]) + + +async def _read_until(stream: CustomStreamWrapper, content: str) -> None: + async for chunk in stream: + if chunk.choices and chunk.choices[0].delta.content == content: + return + pytest.fail(f"the stream never sent {content!r}") + + +def _streamed_usage(chunks: Sequence[ModelResponseStream]) -> Usage: + usages: Final = tuple(usage for chunk in chunks if isinstance(usage := getattr(chunk, "usage", None), Usage)) + assert len(usages) == 1, usages + return usages[0] + + +def _assert_one_stream_row_bills_both_rounds_once( + logged: Sequence[StandardLoggingPayload], chunks: Sequence[ModelResponseStream] +) -> None: + charged_once: Final = _chat_round_cost(_CHAT_ROUND_ONE) + _chat_round_cost(_CHAT_ROUND_TWO) - _FIXED_MARGIN + assert len(logged) == 1 + row: Final = logged[0] + assert row["id"] == chunks[0].id + assert (row["prompt_tokens"], row["completion_tokens"]) == (71 + 149, 19 + 5) + assert row["metadata"]["usage_object"]["prompt_tokens_details"]["cached_tokens"] == 32 + 64 + assert row["response_cost"] == pytest.approx(charged_once) + assert row["cost_breakdown"]["total_cost"] == pytest.approx(charged_once) + assert row["metadata"]["usage_object"]["cost"] == pytest.approx(row["response_cost"]) + usage: Final = _streamed_usage(chunks) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (71 + 149, 19 + 5, 90 + 154) + assert usage.cost == pytest.approx(row["response_cost"]) + + +def _prompt_tokens_the_follow_up_sent(respx_mock: respx.MockRouter) -> int: + follow_up_request: Final = json.loads(respx_mock.calls[1].request.content) + return litellm.token_counter(model="gpt-5.6", messages=follow_up_request["messages"]) + + +def _assert_one_row_bills_round_one(logged: Sequence[StandardLoggingPayload]) -> None: + assert len(logged) == 1 + assert (logged[0]["prompt_tokens"], logged[0]["completion_tokens"]) == (71, 19) + assert logged[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE)) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_read_to_the_end_logs_one_row_whose_usage_cost_counts_the_fixed_margin_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-read-to-the-end", stream=True) + + with _echo_tool_served(): + chunks: Final = await _read_all(await _open_chat_echo_stream(logging_obj)) + await recorder.settled(logged=1) + + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, chunks) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_deferred_for_a_post_call_guardrail_logs_both_rounds_once_when_the_proxy_releases( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rounds-deferred", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + chunks: Final = await _read_all(stream) + await _settled() + logged_before_release: Final = recorder.logged + ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) + await recorder.settled(logged=1) + + assert logged_before_release == () + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, chunks) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_bills_both_rounds_on_the_failure_row( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.exceptions import GuardrailRaisedException + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rounds-blocked", stream=True) + blocked: Final = GuardrailRaisedException(guardrail_name="output-policy", message="blocked") + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + await _read_all(stream) + assert ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, blocked) + await logging_obj.async_failure_handler(blocked, "blocked by output-policy") + await recorder.settled(failed=1) + + charged_once: Final = _chat_round_cost(_CHAT_ROUND_ONE) + _chat_round_cost(_CHAT_ROUND_TWO) - _FIXED_MARGIN + assert recorder.logged == () + assert len(recorder.failed) == 1 + assert (recorder.failed[0]["prompt_tokens"], recorder.failed[0]["completion_tokens"]) == (71 + 149, 19 + 5) + assert recorder.failed[0]["response_cost"] == pytest.approx(charged_once) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_rejected_by_the_proxy_during_the_follow_up_bills_round_one_with_its_own_breakdown( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from fastapi import HTTPException + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rejected-mid-follow-up", stream=True) + rejected: Final = HTTPException(status_code=400, detail="rejected by a stream hook") + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + await _read_until(stream, "PING") + assert ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, rejected) + await logging_obj.async_failure_handler(rejected, "rejected by a stream hook") + await recorder.settled(failed=1) + + assert recorder.logged == () + assert len(recorder.failed) == 1 + failed: Final = recorder.failed[0] + breakdown: Final = failed["cost_breakdown"] + assert (failed["prompt_tokens"], failed["completion_tokens"]) == (71, 19) + assert failed["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE)) + assert breakdown is not None + assert breakdown["total_cost"] == pytest.approx(failed["response_cost"]) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_answered_without_a_tool_call_logs_its_one_round_once_the_answer_finishes( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_TWO),)) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-one-round", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + async for chunk in stream: + if chunk.choices and chunk.choices[0].finish_reason is not None: + break + await recorder.settled(logged=1) + logged_once_the_answer_finished: Final = recorder.logged + after_the_answer: Final = await _read_all(stream) + await _settled() + + assert len(logged_once_the_answer_finished) == 1 + assert recorder.logged == logged_once_the_answer_finished + assert recorder.logged[0]["id"] == _CHAT_ROUND_TWO["id"] + assert (recorder.logged[0]["prompt_tokens"], recorder.logged[0]["completion_tokens"]) == (149, 5) + assert after_the_answer[-1:] and isinstance(after_the_answer[-1].usage, Usage) + usage: Final = _streamed_usage(after_the_answer) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == ( + recorder.logged[0]["prompt_tokens"], + recorder.logged[0]["completion_tokens"], + recorder.logged[0]["total_tokens"], + ) + assert recorder.logged[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_TWO)) + assert recorder.failed == () + + +@pytest.mark.asyncio +@pytest.mark.parametrize("closes", (True, False), ids=("aclose", "dropped")) +async def test_stream_auto_execute_bills_the_finished_first_round_when_the_caller_stops_reading_the_follow_up( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, closes: bool +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-caller-stops", stream=True) + + async def stop_reading_once_the_follow_up_answers() -> None: + stream: Final = await _open_chat_echo_stream(logging_obj) + await _read_until(stream, "PING") + if closes: + await stream.aclose() + + with _echo_tool_served(): + await stop_reading_once_the_follow_up_answers() + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one(recorder.logged) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_whose_first_round_breaks_mid_answer_bills_only_its_failure_row( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_chat_rounds(respx_mock, monkeypatch, (_cut_once_the_answer_starts(),)) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-first-round-breaks", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + with pytest.raises(litellm.ServiceUnavailableError): + await _read_all(stream) + await recorder.settled(failed=1) + + assert recorder.logged == () + assert len(recorder.failed) == 1 + assert recorder.failed[0]["completion_tokens"] > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_bills_a_broken_follow_up_on_its_own_failure_row_with_no_second_fixed_margin( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging, _failure_fields_to_lift + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _cut_once_the_answer_starts())) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-follow-up-breaks", stream=True) + request_data: Final = MappingProxyType({"litellm_logging_obj": logging_obj}) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + with pytest.raises(litellm.ServiceUnavailableError) as broken: + await _read_all(stream) + if not ProxyLogging._discard_deferred_stream_logging_for_failure(dict(request_data), broken.value): + ProxyLogging.fire_deferred_stream_logging(dict(request_data)) + await recorder.settled(logged=1, failed=1) + + _assert_one_row_bills_round_one(recorder.logged) + assert len(recorder.failed) == 1 + failed: Final = recorder.failed[0] + unmargined: Final = ( + litellm.completion_cost( + completion_response=ModelResponse( + model="gpt-5.6", + usage=Usage( + prompt_tokens=failed["prompt_tokens"], + completion_tokens=failed["completion_tokens"], + total_tokens=failed["total_tokens"], + ), + ), + model="gpt-5.6", + custom_llm_provider="openai", + ) + - _FIXED_MARGIN + ) + assert failed["prompt_tokens"] == _prompt_tokens_the_follow_up_sent(respx_mock) + assert failed["completion_tokens"] > 0 + assert failed["response_cost"] == pytest.approx(unmargined) + request_failure: Final = _failure_fields_to_lift(request_data) + assert cast(StandardLoggingPayload, request_failure["standard_logging_object"])["id"] != recorder.logged[0]["id"] + assert request_failure["response_cost"] == pytest.approx(unmargined) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_whose_follow_up_breaks_reports_the_request_guardrail_result_on_the_failure_row_only( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _cut_once_the_answer_starts())) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-follow-up-breaks-guarded", stream=True) + metadata: Final[dict[str, object]] = {} + request_data: Final = {"metadata": metadata, "litellm_logging_obj": logging_obj} + content_filter: Final = CustomGuardrail(guardrail_name="request-content-filter", event_hook="post_call") + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj, metadata) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + with pytest.raises(litellm.ServiceUnavailableError) as broken: + await _read_all(stream) + content_filter.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=str(broken.value), + request_data=request_data, + guardrail_status="guardrail_failed_to_respond", + ) + if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, broken.value): + ProxyLogging.fire_deferred_stream_logging(request_data) + await recorder.settled(logged=1, failed=1) + + _assert_one_row_bills_round_one(recorder.logged) + assert recorder.logged[0]["guardrail_information"] is None + assert [entry["guardrail_name"] for entry in recorder.failed[0]["guardrail_information"] or ()] == [ + "request-content-filter" + ] + + +@pytest.mark.asyncio +async def test_stream_auto_execute_served_through_the_proxy_stream_hook_logs_both_rounds_once_when_it_ends( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.caching.caching import DualCache + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-proxy-stream-hook", stream=True) + + with _echo_tool_served(): + served: Final = ProxyLogging(user_api_key_cache=DualCache()).async_post_call_streaming_iterator_hook( + response=await _open_chat_echo_stream(logging_obj), + user_api_key_dict=UserAPIKeyAuth(), + request_data={"litellm_logging_obj": logging_obj}, + ) + chunks: Final = tuple([chunk async for chunk in served]) + await recorder.settled(logged=1) + + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, chunks) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_whose_follow_up_streams_nothing_still_bills_the_first_round_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + nothing_streamed: Final = Response(200, content=b"data: [DONE]\n\n", headers={"content-type": "text/event-stream"}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), nothing_streamed)) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-empty-follow-up", stream=True) + + with _echo_tool_served(): + await _read_all(await _open_chat_echo_stream(logging_obj)) + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one(recorder.logged) + + +_ANSWER_WITH_UNREADABLE_AUDIO: Final = MappingProxyType( + { + **_CHAT_ROUND_TWO, + "choices": ( + MappingProxyType( + { + "index": 0, + "finish_reason": "stop", + "message": MappingProxyType( + { + "role": "assistant", + "content": "PING", + "audio": MappingProxyType({"id": "audio_echo", "data": "not base64", "transcript": "PING"}), + } + ), + } + ), + ), + } +) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_whose_follow_up_cannot_be_assembled_ends_with_the_usage_of_the_round_it_billed( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_ANSWER_WITH_UNREADABLE_AUDIO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-follow-up-unassembled", stream=True) + + with _echo_tool_served(): + chunks: Final = await _read_all(await _open_chat_echo_stream(logging_obj)) + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one(recorder.logged) + assert any(chunk.choices and chunk.choices[0].delta.content == "PING" for chunk in chunks) + usage: Final = _streamed_usage(chunks) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (71, 19, 90) + assert usage.cost == pytest.approx(recorder.logged[0]["response_cost"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +@pytest.mark.parametrize("answer", [_CHAT_ROUND_TWO, _ANSWER_WITH_UNREADABLE_AUDIO], ids=("assembled", "unassembled")) +async def test_stream_auto_execute_sends_its_usage_chunk_under_the_id_of_the_row_it_billed( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + deferred_for_a_post_call_guardrail: bool, + answer: Mapping[str, object], +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(answer))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-usage-chunk-id", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + chunks: Final = await _read_all(stream) + ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) + await recorder.settled(logged=1) + + usage_chunks: Final = tuple(chunk for chunk in chunks if isinstance(getattr(chunk, "usage", None), Usage)) + answer_chunks: Final = tuple(chunk for chunk in chunks if chunk.choices and chunk.choices[0].delta.content == "PING") + assert len(recorder.logged) == 1 + assert [chunk.id for chunk in answer_chunks] == [_CHAT_ROUND_TWO["id"]] + assert len(usage_chunks) == 1 + assert usage_chunks[0].id == chunks[0].id == recorder.logged[0]["id"] == _CHAT_ROUND_ONE["id"] + + +def _assert_one_row_bills_round_one_and_the_partial_answer( + recorder: _ChatRoundRecorder, respx_mock: respx.MockRouter +) -> None: + assert len(recorder.logged) == 1 + row: Final = recorder.logged[0] + assert row["id"] == _CHAT_ROUND_ONE["id"] + partial_prompt, partial_completion = (row["prompt_tokens"] - 71, row["completion_tokens"] - 19) + assert partial_prompt == _prompt_tokens_the_follow_up_sent(respx_mock) + assert partial_completion > 0 + partial_answer: Final = litellm.completion_cost( + completion_response=ModelResponse( + model="gpt-5.6", + usage=Usage( + prompt_tokens=partial_prompt, + completion_tokens=partial_completion, + total_tokens=partial_prompt + partial_completion, + ), + ), + model="gpt-5.6", + custom_llm_provider="openai", + ) + assert row["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE) + partial_answer - _FIXED_MARGIN) + assert recorder.failed == () + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_hung_up_during_the_follow_up_bills_round_one_and_the_partial_answer_on_one_row( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-hung-up-mid-answer", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + await _read_until(stream, "PING") + await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( + request=None, request_data={"litellm_logging_obj": logging_obj}, response=stream, client_disconnected=True + ) + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one_and_the_partial_answer(recorder, respx_mock) + + +def _chat_rounds_router() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "mcp-chat-rounds", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-mcp-chat-rounds", + "api_base": "https://mcp-chat-rounds.example.com/v1", + }, + } + ] + ) + + +class _StallsOnceTheAnswerStarts(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield _sse(_chat_stream_events(_CHAT_ROUND_TWO)[:1]) + await asyncio.Event().wait() + + +def _streams_content(body: object, content: str) -> bool: + lines: Final = body.decode().splitlines() if isinstance(body, bytes) else () + events: Final = tuple(json.loads(line.removeprefix("data: ")) for line in lines if line.startswith("data: {")) + return any(event["choices"] and event["choices"][0]["delta"].get("content") == content for event in events) + + +async def _serve_until_the_client_hangs_up_after(response: StreamingResponse, content: str) -> None: + read: Final = asyncio.Event() + + async def receive() -> Message: + await read.wait() + return {"type": "http.disconnect"} + + async def send(message: Message) -> None: + if message["type"] == "http.response.body" and _streams_content(message.get("body"), content): + read.set() + + uvicorn_scope: Final = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}, "headers": []} + await response(uvicorn_scope, receive, send) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_routed_through_the_proxy_bills_the_partial_answer_when_the_client_hangs_up( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing, create_response + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds( + respx_mock, + monkeypatch, + ( + _streamed(_CHAT_ROUND_ONE), + Response(200, stream=_StallsOnceTheAnswerStarts(), headers={"content-type": "text/event-stream"}), + ), + ) + router: Final = _chat_rounds_router() + monkeypatch.setattr(proxy_server, "llm_router", router) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-routed-hang-up", stream=True) + + with _echo_tool_served(): + stream: Final = await router.acompletion( + model="mcp-chat-rounds", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + stream=True, + stream_options={"include_usage": True}, + litellm_logging_obj=logging_obj, + ) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + served: Final = await create_response( + proxy_server.async_data_generator( + response=stream, + user_api_key_dict=UserAPIKeyAuth(), + request_data={"litellm_logging_obj": logging_obj, "model": "mcp-chat-rounds", "metadata": {}}, + ), + "text/event-stream", + {}, + ) + assert isinstance(served, StreamingResponse) + await _serve_until_the_client_hangs_up_after(served, "PING") + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one_and_the_partial_answer(recorder, respx_mock) + + +async def _stop_reading_once_the_proxy_sent(served: AsyncIterator[object], content: str) -> None: + async for frame in served: + if _streams_content(frame, content): + return + pytest.fail(f"the proxy never sent {content!r}") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_routed_through_the_proxy_bills_the_partial_answer_when_hung_up_between_reads( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + router: Final = _chat_rounds_router() + monkeypatch.setattr(proxy_server, "llm_router", router) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-routed-hang-up-between-reads", stream=True) + + with _echo_tool_served(): + stream: Final = await router.acompletion( + model="mcp-chat-rounds", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + stream=True, + stream_options={"include_usage": True}, + litellm_logging_obj=logging_obj, + ) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + served: Final = proxy_server.async_data_generator( + response=stream, + user_api_key_dict=UserAPIKeyAuth(), + request_data={"litellm_logging_obj": logging_obj, "model": "mcp-chat-rounds", "metadata": {}}, + ) + await _stop_reading_once_the_proxy_sent(served, "PING") + await served.aclose() + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one_and_the_partial_answer(recorder, respx_mock) + + +_LONGEST_WAIT_FOR_THE_LOOP_TO_MOVE_SECONDS: Final = 10 + + +class _CountsTheToolResultWhileTheLoopRuns: + def __init__(self, loop: asyncio.AbstractEventLoop, count: Callable[..., int]) -> None: + self._loop: Final = loop + self._count: Final = count + self._counting: Final = asyncio.Event() + self._loop_moved: Final = threading.Event() + self.loop_moved_while_counting: tuple[bool, ...] = () + + def __call__(self, model: str | None = None, messages: list[object] | None = None, **kwargs: object) -> int: + if any(isinstance(message, Mapping) and message.get("role") == "tool" for message in messages or ()): + self._loop.call_soon_threadsafe(self._counting.set) + moved: Final = self._loop_moved.wait(timeout=_LONGEST_WAIT_FOR_THE_LOOP_TO_MOVE_SECONDS) + self.loop_moved_while_counting = (*self.loop_moved_while_counting, moved) + return self._count(model=model, messages=messages, **kwargs) + + async def keep_the_loop_moving(self) -> None: + await self._counting.wait() + self._loop_moved.set() + + +@contextmanager +def _partial_count_watched(monkeypatch: pytest.MonkeyPatch) -> Iterator[_CountsTheToolResultWhileTheLoopRuns]: + from litellm.litellm_core_utils import streaming_chunk_builder_utils + + counter: Final = _CountsTheToolResultWhileTheLoopRuns( + asyncio.get_running_loop(), streaming_chunk_builder_utils.token_counter + ) + monkeypatch.setattr(streaming_chunk_builder_utils, "token_counter", counter) + other_task: Final = asyncio.create_task(counter.keep_the_loop_moving()) + try: + yield counter + finally: + other_task.cancel() + + +class _StallsAfterTheAnswerStarts(httpx.AsyncByteStream): + def __init__(self) -> None: + self.stalled: Final = asyncio.Event() + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield _sse(_chat_stream_events(_CHAT_ROUND_TWO)[:1]) + self.stalled.set() + await asyncio.Event().wait() + + +@pytest.mark.asyncio +async def test_stream_auto_execute_cancelled_mid_answer_counts_the_partial_off_the_loop_and_bills_it_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + answer: Final = _StallsAfterTheAnswerStarts() + _serve_chat_rounds( + respx_mock, + monkeypatch, + (_streamed(_CHAT_ROUND_ONE), Response(200, stream=answer, headers={"content-type": "text/event-stream"})), + ) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-cancelled-off-the-loop", stream=True) + + with _echo_tool_served(), _partial_count_watched(monkeypatch) as counter: + stream: Final = await _open_chat_echo_stream(logging_obj) + reading: Final = asyncio.create_task(_read_all(stream)) + await asyncio.wait_for(answer.stalled.wait(), timeout=10) + reading.cancel() + with pytest.raises(asyncio.CancelledError): + await reading + await stream.aclose() + await recorder.settled(logged=1) + + assert counter.loop_moved_while_counting == (True,) + _assert_one_row_bills_round_one_and_the_partial_answer(recorder, respx_mock) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_routed_through_the_proxy_and_closed_between_reads_counts_the_partial_off_the_loop( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + router: Final = _chat_rounds_router() + monkeypatch.setattr(proxy_server, "llm_router", router) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-routed-closed-off-the-loop", stream=True) + + with _echo_tool_served(), _partial_count_watched(monkeypatch) as counter: + stream: Final = await router.acompletion( + model="mcp-chat-rounds", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + stream=True, + stream_options={"include_usage": True}, + litellm_logging_obj=logging_obj, + ) + served: Final = proxy_server.async_data_generator( + response=stream, + user_api_key_dict=UserAPIKeyAuth(), + request_data={"litellm_logging_obj": logging_obj, "model": "mcp-chat-rounds", "metadata": {}}, + ) + await _stop_reading_once_the_proxy_sent(served, "PING") + await served.aclose() + await recorder.settled(logged=1) + + assert counter.loop_moved_while_counting == (True,) + _assert_one_row_bills_round_one_and_the_partial_answer(recorder, respx_mock) + + +class _StallsBeforeTheAnswerStarts(httpx.AsyncByteStream): + def __init__(self) -> None: + self.opened: Final = asyncio.Event() + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b": the answer has not started\n\n" + self.opened.set() + await asyncio.Event().wait() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_hung_up_before_the_answer_starts_bills_round_one_alone_on_one_row( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + answer: Final = _StallsBeforeTheAnswerStarts() + _serve_chat_rounds( + respx_mock, + monkeypatch, + (_streamed(_CHAT_ROUND_ONE), Response(200, stream=answer, headers={"content-type": "text/event-stream"})), + ) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-hung-up-before-answer", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + reading: Final = asyncio.create_task(_read_all(stream)) + await asyncio.wait_for(answer.opened.wait(), timeout=10) + reading.cancel() + with pytest.raises(asyncio.CancelledError): + await reading + await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( + request=None, request_data={"litellm_logging_obj": logging_obj}, response=stream, client_disconnected=True + ) + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one(recorder.logged) + assert recorder.failed == () + + +async def _read_through_the_usage_chunk( + chunks: AsyncIterator[ModelResponseStream], read: tuple[ModelResponseStream, ...] = () +) -> tuple[ModelResponseStream, ...]: + chunk: Final = await anext(chunks) + if isinstance(getattr(chunk, "usage", None), Usage): + return (*read, chunk) + return await _read_through_the_usage_chunk(chunks, (*read, chunk)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_hung_up_after_its_whole_answer_bills_both_rounds_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-hung-up-after-answer", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + chunks: Final = await _read_through_the_usage_chunk(aiter(stream)) + await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( + request=None, request_data={"litellm_logging_obj": logging_obj}, response=stream, client_disconnected=True + ) + await recorder.settled(logged=1) + + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, chunks) + assert recorder.failed == () + + +@pytest.mark.asyncio +async def test_stream_auto_execute_bridged_to_messages_bills_round_one_when_its_follow_up_breaks( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _cut_once_the_answer_starts())) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-bridged-to-messages", stream=True) + + async def bridged(stream: CustomStreamWrapper) -> AsyncIterator[ModelResponseStream]: + async for chunk in stream: + yield chunk + + with _echo_tool_served(): + relayed: Final = bridged(await _open_chat_echo_stream(logging_obj)) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=relayed, route_type="anthropic_messages", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + with pytest.raises(litellm.ServiceUnavailableError): + _ = [chunk async for chunk in relayed] + await recorder.settled(logged=1, failed=1) + + _assert_one_row_bills_round_one(recorder.logged) + + +def _read_synchronously_until_the_follow_up_answers( + logging_obj: Logging, loop: asyncio.AbstractEventLoop, before_dropping: Callable[[], None] = lambda: None +) -> None: + stream: Final = loop.run_until_complete(_open_chat_echo_stream(logging_obj)) + for chunk in stream: + if chunk.choices and chunk.choices[0].delta.content == "PING": + before_dropping() + return + pytest.fail("the stream never sent 'PING'") + + +_CHAT_STREAM_BILLING_NAMES: Final = ("MCPStreamingIterator", "ChatStreamRoundBilling", "_log_success") + + +def _names_chat_stream_billing(text: str) -> bool: + return any(name in text for name in _CHAT_STREAM_BILLING_NAMES) + + +class _UnraisableRecorder: + def __init__(self) -> None: + self.raised: tuple[str, ...] = () + + def __call__(self, unraisable: "sys.UnraisableHookArgs") -> None: + self.raised = (*self.raised, f"{unraisable.object!r}: {unraisable.exc_value!r}") + + +def test_stream_auto_execute_read_synchronously_and_dropped_mid_answer_bills_the_finished_first_round( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + unraisable: Final = _UnraisableRecorder() + loop: Final = asyncio.new_event_loop() + gc.collect() + monkeypatch.setattr(sys, "unraisablehook", unraisable) + asyncio.set_event_loop(loop) + + try: + with _echo_tool_served(), warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _read_synchronously_until_the_follow_up_answers( + _chat_rounds_logging_obj(recorder, "mcp-chat-stream-sync-dropped", stream=True), loop + ) + gc.collect() + loop.run_until_complete(recorder.settled(logged=1)) + finally: + asyncio.set_event_loop(None) + loop.close() + + _assert_one_row_bills_round_one(recorder.logged) + assert [raised for raised in unraisable.raised if _names_chat_stream_billing(raised)] == [] + assert [str(warning.message) for warning in caught if _names_chat_stream_billing(str(warning.message))] == [] + + +def test_stream_auto_execute_read_synchronously_and_dropped_after_its_loop_closed_drops_without_errors( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + unraisable: Final = _UnraisableRecorder() + loop: Final = asyncio.new_event_loop() + gc.collect() + monkeypatch.setattr(sys, "unraisablehook", unraisable) + asyncio.set_event_loop(loop) + + try: + with _echo_tool_served(), warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _read_synchronously_until_the_follow_up_answers( + _chat_rounds_logging_obj(recorder, "mcp-chat-stream-sync-dropped-after-close", stream=True), + loop, + before_dropping=loop.close, + ) + gc.collect() + finally: + asyncio.set_event_loop(None) + loop.close() + + assert [raised for raised in unraisable.raised if _names_chat_stream_billing(raised)] == [] + assert [str(warning.message) for warning in caught if _names_chat_stream_billing(str(warning.message))] == [] diff --git a/tests/unit/responses/mcp/test_parked_stream_dispatch.py b/tests/unit/responses/mcp/test_parked_stream_dispatch.py new file mode 100644 index 00000000000..dd380c582c8 --- /dev/null +++ b/tests/unit/responses/mcp/test_parked_stream_dispatch.py @@ -0,0 +1,17 @@ +from typing import Final + +import pytest + +from litellm.responses.mcp.parked_stream_dispatch import parked_success +from litellm.types.utils import ModelResponse + +_ASSEMBLED: Final = ModelResponse(id="chatcmpl-parked", model="gpt-5.6") + + +@pytest.mark.parametrize( + "parked", + [(), (_ASSEMBLED,), (_ASSEMBLED, False, "not a logger"), (_ASSEMBLED, False, None, None)], + ids=("empty", "assembled-alone", "third-part-is-not-a-logger", "four-parts"), +) +def test_a_parked_dispatch_of_an_unknown_shape_is_never_dispatched(parked: tuple[object, ...]): + assert parked_success(parked) is None diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index ab52e6d29c3..d0e26798090 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -8,15 +8,21 @@ Tests that: """ import asyncio +import itertools import json -from collections.abc import Mapping -from typing import Final -from unittest.mock import MagicMock +from collections.abc import AsyncIterator, Mapping +from datetime import datetime, timezone +from functools import partial +from typing import TYPE_CHECKING, Final +from unittest.mock import AsyncMock, MagicMock +import httpx import pytest +import respx from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig +from litellm.llms.custom_llm import CustomLLM from litellm.llms.databricks.responses.transformation import ( DatabricksResponsesAPIConfig, ) @@ -42,6 +48,13 @@ from litellm.llms.volcengine.responses.transformation import ( VolcEngineResponsesAPIConfig, ) from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig +from litellm.types.utils import GenericStreamingChunk + +if TYPE_CHECKING: + from litellm.caching.dual_cache import DualCache + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.responses.streaming_iterator import ResponsesWsTurnAdmission + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck class TestResponsesAPIWebSocketSupport: @@ -1591,7 +1604,7 @@ def _admission_refusing_the_next_turn(): from litellm.exceptions import RateLimitError - return AsyncMock(side_effect=[RateLimitError(message="crossed budget", llm_provider="", model=""), None]) + return AsyncMock(side_effect=[RateLimitError(message="crossed budget", llm_provider="", model=""), {}]) class _ClientSocket: @@ -1630,7 +1643,7 @@ class _StalledClientSocket(_ClientSocket): def _admission_crashing_the_next_turn(): from unittest.mock import AsyncMock - return AsyncMock(side_effect=[ValueError("custom check failed"), None]) + return AsyncMock(side_effect=[ValueError("custom check failed"), {}]) def _guard_blocking(word: str): @@ -1655,6 +1668,229 @@ def _guard_recording(bill_refusal: MagicMock): return guard +_OPENED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:otpm:12-34" +_TURNED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:otpm:12-35" +_OPENED_AT: Final = datetime(2026, 1, 1, 12, 34, 56, tzinfo=timezone.utc) +_TURNED_AT: Final = datetime(2026, 1, 1, 12, 35, 5, tzinfo=timezone.utc) +_WS_DEPLOYMENT: Final = { + "litellm_params": {"model": "openai/gpt-5-mini", "otpm": 4096}, + "model_info": {"id": "ws-dep"}, +} +_OTHER_USAGE: Final = 10 +_OUTPUT_TOKENS_PER_TURN: Final = 2 +_UPSTREAM_ERROR: Final = json.dumps({"type": "error", "error": {"type": "server_error", "message": "upstream failed"}}) + + +def _turn_completed() -> str: + return json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_turn", + "status": "completed", + "output": [], + "usage": {"input_tokens": 5, "output_tokens": _OUTPUT_TOKENS_PER_TURN, "total_tokens": 7}, + }, + } + ) + + +async def _socket_opened_on(dual_cache: "DualCache") -> Mapping[str, object]: + from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn + + await dual_cache.async_increment_cache(key=_OPENED_ON, value=_OTHER_USAGE, ttl=60) + await dual_cache.async_increment_cache(key=_TURNED_ON, value=_OTHER_USAGE, ttl=60) + return await async_admit_io_token_turn(dual_cache, _WS_DEPLOYMENT, _OPENED_AT) + + +def _turns_finish_at(monkeypatch: pytest.MonkeyPatch, finished_at: datetime) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: finished_at + ) + + +def _later_turns_admitted_through(dual_cache: "DualCache") -> "ResponsesWsTurnAdmission": + from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn + + return partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT, _TURNED_AT) + + +_ITPM_OPENED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:itpm:12-34" +_ITPM_DEPLOYMENT: Final = { + "litellm_params": {"model": "openai/gpt-5-mini", "itpm": 40_000}, + "model_info": {"id": "ws-dep"}, +} +_TURN_INPUT_TOKENS: Final = 1_000 + + +def _turn_completed_reading_cache(cached_tokens: int) -> str: + return json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_cached_turn", + "status": "completed", + "output": [], + "usage": { + "input_tokens": _TURN_INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": cached_tokens}, + "output_tokens": _OUTPUT_TOKENS_PER_TURN, + "total_tokens": _TURN_INPUT_TOKENS + _OUTPUT_TOKENS_PER_TURN, + }, + }, + } + ) + + +def _connection_logged_through(rate_limits: "ModelRateLimitingCheck", opened_with: Mapping[str, object]) -> "Logging": + import time + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import CallTypes + + connection: Final = Logging( + model="gpt-5-mini", + messages=[], + stream=False, + call_type=CallTypes.aresponses_websocket.value, + start_time=time.time(), + litellm_call_id="connection-call", + function_id="connection-call", + dynamic_async_success_callbacks=[rate_limits], + dynamic_async_failure_callbacks=[rate_limits], + ) + connection.update_environment_variables( + litellm_params={"metadata": dict(opened_with), CallTypes.aresponses_websocket.value: True}, + optional_params={}, + ) + return connection + + +def _provider_turn_stream(_request: httpx.Request) -> httpx.Response: + completed: Final = { + "id": "resp_turn", + "object": "response", + "created_at": 1767271505, + "status": "completed", + "model": "gpt-5-mini", + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 5, "output_tokens": _OUTPUT_TOKENS_PER_TURN, "total_tokens": 7}, + } + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text="".join( + f"event: {kind}\ndata: {json.dumps({'type': kind, 'response': completed})}\n\n" + for kind in ("response.created", "response.completed") + ), + ) + + +class _ProviderResponsesSocket: + """A provider's native Responses socket that answers each ``response.create`` with a completed turn.""" + + def __init__(self) -> None: + self.sent: Final[list[dict[str, object]]] = [] + self._replies: Final[asyncio.Queue[str]] = asyncio.Queue() + + def connect(self, _url: str, **_kwargs: object) -> "_ProviderResponsesSocket": + return self + + async def __aenter__(self) -> "_ProviderResponsesSocket": + return self + + async def __aexit__(self, *_exc_info: object) -> None: + return None + + async def send(self, message: str) -> None: + self.sent.append(json.loads(message)) + await self._replies.put(_turn_completed()) + + async def recv(self, decode: bool = False) -> str: + return await self._replies.get() + + async def close(self) -> None: + return None + + +class _TwoOutputTokenTurns(CustomLLM): + def __init__(self, failing_turns: frozenset[int] = frozenset()) -> None: + super().__init__() + self._failing_turns: Final = failing_turns + self._turns: Final = itertools.count(1) + + async def astreaming(self, *args: object, **kwargs: object) -> AsyncIterator[GenericStreamingChunk]: + if next(self._turns) in self._failing_turns: + raise RuntimeError("upstream failed") + yield GenericStreamingChunk( + text="hi", + is_finished=True, + finish_reason="stop", + index=0, + tool_use=None, + usage={"prompt_tokens": 5, "completion_tokens": _OUTPUT_TOKENS_PER_TURN, "total_tokens": 7}, + ) + + +async def _run_managed_relay( + monkeypatch: pytest.MonkeyPatch, + provider: CustomLLM, + rate_limits: "ModelRateLimitingCheck", + frames: tuple[str, ...], + turn_admission: "ResponsesWsTurnAdmission | None", + opened_with: Mapping[str, object], +) -> _ClientSocket: + import litellm + + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "ws-turn-llm", "custom_handler": provider}]) + return await _relay_turns_to(monkeypatch, "ws-turn-llm/turns", rate_limits, frames, turn_admission, opened_with) + + +async def _relay_turns_to( + monkeypatch: pytest.MonkeyPatch, + model: str, + rate_limits: "ModelRateLimitingCheck", + frames: tuple[str, ...], + turn_admission: "ResponsesWsTurnAdmission | None", + opened_with: Mapping[str, object], + **connection: object, +) -> _ClientSocket: + import litellm + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler, drain_logging_tasks + + monkeypatch.setattr(litellm, "callbacks", [rate_limits]) + first, *later = frames + websocket: Final = _ClientSocket(*later) + handler: Final = ManagedResponsesWebSocketHandler( + websocket=websocket, + model=model, + logging_obj=Logging( + model=model, + messages=[], + stream=True, + call_type="aresponses", + start_time=0, + litellm_call_id="test-id", + function_id="test-func", + ), + litellm_metadata={"user_api_key_hash": "k1", **opened_with}, + first_message=first, + turn_admission=turn_admission, + **connection, + ) + + await handler.run() + await drain_logging_tasks() + await asyncio.sleep(0) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + return websocket + + def _errors(websocket: _ClientSocket) -> list[tuple[object, object]]: return [(frame.get("status"), frame["error"]) for frame in websocket.sent if frame.get("type") == "error"] @@ -1722,6 +1958,195 @@ class TestWebSocketTurnAdmission: assert isinstance(failure, RateLimitError) assert "crossed budget" in str(failure) + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("first_turn_end", "first_turn_tokens"), + [ + pytest.param(_turn_completed(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"), + pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"), + ], + ) + async def test_native_relay_counts_every_turn_on_the_minute_it_finishes( + self, first_turn_end: str, first_turn_tokens: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + import websockets.exceptions # noqa: F401 (lazy submodule must be importable) + + from litellm.caching.dual_cache import DualCache + from litellm.responses.streaming_iterator import drain_logging_tasks + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + _turns_finish_at(monkeypatch, _TURNED_AT) + dual_cache: Final = DualCache() + opened_with: Final = await _socket_opened_on(dual_cache) + first, second, third = _turn_frames() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock( + side_effect=[first_turn_end, _turn_completed(), _turn_completed(), Exception("stop")] + ) + handler: Final = _make_streaming( + websocket=_ClientSocket(second, third), + backend_ws=backend_ws, + request_data={}, + logging_obj=_connection_logged_through(ModelRateLimitingCheck(dual_cache), opened_with), + first_message=first, + turn_admission=_later_turns_admitted_through(dual_cache), + ) + + await handler.client_to_backend() + await handler.backend_to_client() + await drain_logging_tasks() + + assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + assert ( + await dual_cache.async_get_cache(key=_TURNED_ON) + == _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("cached_tokens", "counted_input_tokens"), + [ + pytest.param(800, _TURN_INPUT_TOKENS - 800, id="cached_input"), + pytest.param(0, _TURN_INPUT_TOKENS, id="uncached_input"), + ], + ) + async def test_native_relay_leaves_a_turn_s_cached_input_out_of_the_deployment_s_itpm( + self, cached_tokens: int, counted_input_tokens: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + import websockets.exceptions # noqa: F401 (lazy submodule must be importable) + + from litellm.caching.dual_cache import DualCache + from litellm.responses.streaming_iterator import drain_logging_tasks + from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + _turns_finish_at(monkeypatch, _OPENED_AT) + dual_cache: Final = DualCache() + opened_with: Final = await async_admit_io_token_turn(dual_cache, _ITPM_DEPLOYMENT, _OPENED_AT) + first, _, _ = _turn_frames() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=[_turn_completed_reading_cache(cached_tokens), Exception("stop")]) + handler: Final = _make_streaming( + websocket=_ClientSocket(), + backend_ws=backend_ws, + request_data={}, + logging_obj=_connection_logged_through(ModelRateLimitingCheck(dual_cache), opened_with), + first_message=first, + ) + + await handler.client_to_backend() + await handler.backend_to_client() + await drain_logging_tasks() + + assert await dual_cache.async_get_cache(key=_ITPM_OPENED_ON) == counted_input_tokens + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("failing_turns", "first_turn_tokens"), + [ + pytest.param(frozenset(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"), + pytest.param(frozenset({1}), 0, id="first_turn_failed"), + ], + ) + async def test_managed_relay_counts_every_turn_on_the_minute_it_finishes( + self, monkeypatch: pytest.MonkeyPatch, failing_turns: frozenset[int], first_turn_tokens: int + ) -> None: + from litellm.caching.dual_cache import DualCache + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + _turns_finish_at(monkeypatch, _TURNED_AT) + dual_cache: Final = DualCache() + opened_with: Final = await _socket_opened_on(dual_cache) + + await _run_managed_relay( + monkeypatch, + _TwoOutputTokenTurns(failing_turns), + ModelRateLimitingCheck(dual_cache), + _turn_frames(), + _later_turns_admitted_through(dual_cache), + opened_with, + ) + + assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + assert ( + await dual_cache.async_get_cache(key=_TURNED_ON) + == _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN + ) + + @pytest.mark.asyncio + async def test_managed_relay_sends_the_provider_only_the_metadata_each_turn_carries( + self, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + ) -> None: + import litellm + from litellm.caching.dual_cache import DualCache + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + _turns_finish_at(monkeypatch, _TURNED_AT) + provider: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=_provider_turn_stream) + client_metadata: Final = {"session": "abc", "team": "search"} + dual_cache: Final = DualCache() + + await _relay_turns_to( + monkeypatch, + "openai/gpt-5-mini", + ModelRateLimitingCheck(dual_cache), + tuple( + json.dumps({"type": "response.create", "input": word, "metadata": client_metadata}) + for word in ("one", "two") + ), + _later_turns_admitted_through(dual_cache), + await _socket_opened_on(dual_cache), + api_key="sk-fake", + ) + + assert [json.loads(call.request.read()).get("metadata") for call in provider.calls] == [ + client_metadata, + client_metadata, + ] + assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN + + @pytest.mark.asyncio + async def test_native_relay_sends_the_provider_only_the_metadata_the_socket_opened_with( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from unittest.mock import patch + + from litellm import Router + + monkeypatch.setenv("LITELLM_RUST", "false") + provider: Final = _ProviderResponsesSocket() + client_metadata: Final = {"session": "abc", "team": "search"} + first, second, _ = _turn_frames() + router: Final = Router( + model_list=[ + { + "model_name": "ws-io", + "litellm_params": { + "model": "openai/gpt-5-mini", + "api_key": "sk-fake", + "itpm": 40_000, + "otpm": 4096, + }, + "model_info": {"id": "ws-dep"}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + ) + + with patch("websockets.connect", provider.connect): + await router._aresponses_websocket( + model="ws-io", + websocket=_ClientSocket(second), + first_message=first, + metadata=dict(client_metadata), + ) + + assert [frame.get("metadata") for frame in provider.sent] == [client_metadata, client_metadata] + @pytest.mark.asyncio async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch): from unittest.mock import AsyncMock @@ -1875,7 +2300,7 @@ class TestWebSocketTurnAdmission: request_data={}, first_message=first, frame_guard=_guard_recording(bill_refusal), - turn_admission=AsyncMock(side_effect=[refusal, None]), + turn_admission=AsyncMock(side_effect=[refusal, {}]), ) close_time_failure: Final = await handler.bidirectional_forward() @@ -1904,7 +2329,7 @@ class TestWebSocketTurnAdmission: request_data={}, first_message=first, frame_guard=_guard_recording(bill_refusal), - turn_admission=AsyncMock(side_effect=[refusal, None]), + turn_admission=AsyncMock(side_effect=[refusal, {}]), ) relay: Final = asyncio.ensure_future(handler.bidirectional_forward()) await asyncio.wait_for(client.send_started.wait(), timeout=1) @@ -1955,7 +2380,7 @@ class TestWebSocketTurnAdmission: ), first_message=first, frame_guard=_guard_recording(bill_refusal), - turn_admission=AsyncMock(side_effect=[refusal, None]), + turn_admission=AsyncMock(side_effect=[refusal, {}]), ) close_time_failure: Final = await handler.run() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index b2ae6b43d26..9f2d2a9bbc5 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -2463,6 +2463,463 @@ _BRIDGED_RESPONSE_ANSWER: Final = json.dumps( ).encode() +@pytest.fixture +def non_openai_responses_bridge( + restore_model_registry, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> Callable[[str], tuple[str, Mapping[str, str], respx.Route]]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("LITELLM_REASONING_AUTO_SUMMARY", raising=False) + for chatgpt_env in ("CHATGPT_AUTH_FILE", "CHATGPT_API_BASE", "OPENAI_CHATGPT_API_BASE"): + monkeypatch.delenv(chatgpt_env, raising=False) + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "auth.json").write_text( + json.dumps({"access_token": "token", "account_id": "acct", "expires_at": 4102444800}) + ) + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "api-key.json").write_text( + json.dumps({"token": "token", "expires_at": 4102444800, "endpoints": {"api": "https://api.githubcopilot.com"}}) + ) + litellm.register_model( + { + **{ + model_name: { + "litellm_provider": "openai", + "mode": "chat", + "supports_reasoning": True, + "reasoning_modes": ["standard"], + "reasoning_contexts": ["auto", "current_turn", "all_turns"], + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + for model_name in ("gpt-6", "gpt-6.1-standard-only") + }, + **{ + f"{provider}/{model_name}": { + "litellm_provider": provider, + "mode": "responses", + "supports_reasoning": True, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + for provider, model_name in ( + ("chatgpt", "gpt-6.1-bridge-test"), + ("perplexity", "bridge-test-model"), + ("github_copilot", "gpt-6.1-bridge-test"), + ) + }, + }, + persist_across_reloads=False, + ) + backends: Final = { + "azure": ( + "azure/responses/gpt-6.1-standard-only", + "https://test.openai.azure.com/", + {"api_base": "https://test.openai.azure.com", "api_key": "fake-key"}, + ), + "azure_ai": ( + "azure_ai/responses/gpt-6.1-standard-only-prod", + _FOUNDRY_API_BASE, + {"api_base": _FOUNDRY_API_BASE, "api_key": "fake-key"}, + ), + "chatgpt": ("chatgpt/gpt-6.1-bridge-test", "https://chatgpt.com/backend-api/codex/responses", {}), + "github_copilot": ("github_copilot/gpt-6.1-bridge-test", "https://api.githubcopilot.com/responses", {}), + "databricks": ( + "databricks/responses/databricks-gpt-bridge-test", + "https://dbx.example/serving-endpoints/responses", + {"api_base": "https://dbx.example/serving-endpoints", "api_key": "fake-key"}, + ), + "perplexity": ( + "perplexity/bridge-test-model", + "https://api.perplexity.ai/v1/responses", + {"api_key": "fake-key"}, + ), + "xai": ("xai/responses/grok-bridge-test", "https://api.x.ai/v1/responses", {"api_key": "fake-key"}), + "fireworks_ai": ( + "fireworks_ai/responses/accounts/fireworks/models/bridge-test", + "https://api.fireworks.ai/inference/v1/responses", + {"api_key": "fake-key"}, + ), + "openrouter": ( + "openrouter/responses/openai/gpt-6.1-standard-only", + "https://openrouter.ai/api/v1/responses", + {"api_key": "fake-key"}, + ), + "litellm_proxy": ( + "litellm_proxy/responses/gpt-6.1-standard-only", + "https://proxy.example/v1/responses", + {"api_base": "https://proxy.example/v1", "api_key": "fake-key"}, + ), + } + + def route_backend(backend: str) -> tuple[str, Mapping[str, str], respx.Route]: + model, url, kwargs = backends[backend] + route: Final = respx_mock.post(url__startswith=url).respond( + content=_BRIDGED_RESPONSE_ANSWER, headers={"content-type": "application/json"} + ) + return model, kwargs, route + + return route_backend + + +@pytest.mark.parametrize( + ("backend", "reasoning_effort"), + [ + ("azure", {"effort": "low", "mode": "pro", "context": "all_turns"}), + ("chatgpt", {"effort": "low", "mode": "pro", "context": "all_turns"}), + ("github_copilot", {"effort": "low", "mode": "standard", "context": "all_turns"}), + ("databricks", {"effort": "low", "mode": "pro", "context": "all_turns"}), + ("openrouter", {"effort": "low", "mode": "pro", "context": "all_turns"}), + ("perplexity", {"effort": "low", "mode": "standard", "context": "all_turns", "generate_summary": "auto"}), + ("xai", {"effort": "low", "mode": "standard", "generate_summary": "auto"}), + ("fireworks_ai", {"effort": "low", "mode": "standard", "generate_summary": "auto"}), + ], +) +@pytest.mark.parametrize("summary", [{}, {"summary": "auto"}]) +def test_non_openai_responses_bridge_sends_the_whole_reasoning_effort_dict_with_or_without_summary( + backend: str, + reasoning_effort: dict[str, str], + summary: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + model, kwargs, route = non_openai_responses_bridge(backend) + requested: Final = {**reasoning_effort, **summary} + + litellm.completion( + model=model, messages=[{"role": "user", "content": "hello"}], reasoning_effort=requested, **kwargs + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == requested + + +@pytest.mark.parametrize( + ("backend", "reasoning_effort", "expected_reasoning"), + [ + ("azure", {"effort": "low", "mode": "pro", "context": "all_turns"}, {"effort": "low", "context": "all_turns"}), + ( + "chatgpt", + {"effort": "low", "mode": "pro", "context": "all_turns"}, + {"effort": "low", "context": "all_turns"}, + ), + ( + "chatgpt", + {"effort": "low", "mode": "standard", "context": "current_turn"}, + {"effort": "low", "mode": "standard", "context": "current_turn"}, + ), + ( + "github_copilot", + {"effort": "low", "mode": "pro", "context": "all_turns"}, + {"effort": "low", "context": "all_turns"}, + ), + ( + "openrouter", + {"effort": "low", "mode": "pro", "context": "all_turns"}, + {"effort": "low", "context": "all_turns"}, + ), + ( + "perplexity", + {"effort": "low", "mode": "pro", "context": "all_turns"}, + {"effort": "low", "context": "all_turns"}, + ), + ("perplexity", {"effort": "low", "mode": "standard", "context": "every_turn"}, {"effort": "low", "mode": "standard"}), + ("xai", {"effort": "low", "mode": "pro", "context": "all_turns"}, {"effort": "low"}), + ("fireworks_ai", {"effort": "low", "mode": "pro", "context": "all_turns"}, {"effort": "low"}), + ], +) +@pytest.mark.parametrize("summary", [{}, {"summary": "auto"}]) +def test_non_openai_responses_bridge_drops_reasoning_fields_the_backend_refuses_under_drop_params( + backend: str, + reasoning_effort: dict[str, str], + expected_reasoning: dict[str, str], + summary: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + model, kwargs, route = non_openai_responses_bridge(backend) + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={**reasoning_effort, **summary}, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {**expected_reasoning, **summary} + + +@pytest.mark.parametrize( + ("backend", "ignored"), + [ + ("perplexity", {"mode": "pro"}), + ("perplexity", {"context": "every_turn"}), + ("xai", {"context": "all_turns"}), + ("fireworks_ai", {"context": "all_turns"}), + ("github_copilot", {"mode": "pro"}), + ], +) +@pytest.mark.parametrize("summary", [{}, {"summary": "auto"}]) +def test_responses_bridge_rejects_reasoning_fields_the_backend_ignores_without_drop_params( + backend: str, + ignored: dict[str, str], + summary: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + model, kwargs, route = non_openai_responses_bridge(backend) + + with pytest.raises(litellm.UnsupportedParamsError, match=rf"reasoning_effort\.{next(iter(ignored))}") as raised: + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", **ignored, **summary}, + **kwargs, + ) + + assert raised.value.status_code == 400 + assert route.called is False + + +@pytest.mark.parametrize("backend", ["perplexity", "xai", "fireworks_ai"]) +def test_responses_bridge_sends_the_rest_when_additional_drop_params_removes_the_refused_fields( + backend: str, + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + model, kwargs, route = non_openai_responses_bridge(backend) + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro", "context": "all_turns", "summary": "auto"}, + additional_drop_params=["reasoning_effort.mode", "reasoning_effort.context"], + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "summary": "auto"} + + +@pytest.mark.parametrize("backend", ["perplexity", "xai", "fireworks_ai"]) +def test_responses_bridge_forwards_a_null_reasoning_field_instead_of_rejecting_it( + backend: str, + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + model, kwargs, route = non_openai_responses_bridge(backend) + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": None}, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "mode": None} + + +def test_chatgpt_bridge_keeps_a_reasoning_mode_its_catalog_row_declares_under_drop_params( + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge("chatgpt") + litellm.register_model( + { + "chatgpt/gpt-6.1-pro-bridge-test": { + "litellm_provider": "chatgpt", + "mode": "responses", + "supports_reasoning": True, + "reasoning_modes": ["pro"], + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + + litellm.completion( + model="chatgpt/gpt-6.1-pro-bridge-test", + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro"}, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "mode": "pro"} + + +@pytest.mark.parametrize( + ("model", "model_info"), + [ + ("azure/responses/custom-deployment-name", {"base_model": "azure/gpt-6.1-standard-only"}), + ("azure/responses/custom-deployment-name", {"base_model": "azure/eu/gpt-6.1-standard-only"}), + ("azure/responses/custom-deployment-name", {"base_model": "azure/gpt-6.1-standard-only-2026-07-09"}), + ("azure/responses/gpt-6.1-standard-only-prod", {}), + ("azure/responses/eu-gpt-6.1-standard-only-prod", {}), + ("azure/responses/gpt-6-2026-07-09", {}), + ], +) +def test_azure_bridge_reads_reasoning_modes_from_the_deployment_model_family_under_drop_params( + model: str, + model_info: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge("azure") + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro", "context": "all_turns"}, + model_info=model_info, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "context": "all_turns"} + + +def test_azure_ai_bridge_judges_reasoning_fields_by_the_request_api_base_under_drop_params( + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("AZURE_AI_API_BASE", "https://legacy-models.eastus2.inference.ai.azure.com") + model, kwargs, route = non_openai_responses_bridge("azure_ai") + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro", "context": "all_turns"}, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "context": "all_turns"} + + +@pytest.mark.parametrize("model", ["azure/responses/custom-gpt-6x", "azure/responses/gpt-6-1-sol"]) +@pytest.mark.parametrize("summary", [{}, {"summary": "auto"}]) +def test_azure_bridge_sends_only_effort_and_summary_to_an_unknown_gpt_series_deployment_under_drop_params( + model: str, + summary: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge("azure") + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro", "context": "all_turns", **summary}, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", **summary} + + +@pytest.mark.parametrize( + ("backend", "model"), + [ + ("azure", "azure/responses/terra-prod"), + ("databricks", "databricks/responses/databricks-gpt-bridge-test"), + ("databricks", "databricks/responses/databricks-gpt-6.1-standard-only"), + ("litellm_proxy", "litellm_proxy/responses/gpt-6.1-standard-only"), + ], +) +@pytest.mark.parametrize("summary", [{}, {"summary": "auto"}]) +def test_responses_bridge_forwards_mode_and_context_it_cannot_judge_under_drop_params( + backend: str, + model: str, + summary: dict[str, str], + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge(backend) + requested: Final = {"effort": "low", "mode": "pro", "context": "all_turns", **summary} + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort=requested, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == requested + + +def test_azure_bridge_applies_a_deployment_reasoning_modes_only_to_that_deployment_under_drop_params( + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge("azure") + litellm.register_model( + { + "gpt-6.1-pro-capable": { + "litellm_provider": "openai", + "mode": "chat", + "supports_reasoning": True, + "reasoning_modes": ["standard", "pro"], + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + litellm_params: Final = {"model": "azure/responses/gpt-6.1-pro-capable", **kwargs} + router: Final = litellm.Router( + model_list=[ + { + "model_name": "standard-only", + "litellm_params": litellm_params, + "model_info": {"reasoning_modes": ["standard"]}, + }, + {"model_name": "pro-capable", "litellm_params": litellm_params}, + ] + ) + requested: Final = {"effort": "low", "mode": "pro"} + + for model_name in ("standard-only", "pro-capable"): + router.completion( + model=model_name, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort=requested, + drop_params=True, + ) + litellm.completion( + model="azure/responses/gpt-6.1-pro-capable", + messages=[{"role": "user", "content": "hello"}], + reasoning_effort=requested, + drop_params=True, + **kwargs, + ) + + assert [json.loads(call.request.content)["reasoning"] for call in route.calls] == [ + {"effort": "low"}, + requested, + requested, + ] + + +def test_azure_bridge_reads_reasoning_modes_from_the_deployment_cost_map_row_under_drop_params( + non_openai_responses_bridge: Callable[[str], tuple[str, Mapping[str, str], respx.Route]], +): + _, kwargs, route = non_openai_responses_bridge("azure") + litellm.register_model( + { + "azure/custom-standard-deployment": { + "litellm_provider": "azure", + "mode": "responses", + "supports_reasoning": True, + "reasoning_modes": ["standard"], + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + + litellm.completion( + model="azure/responses/custom-standard-deployment", + messages=[{"role": "user", "content": "hello"}], + reasoning_effort={"effort": "low", "mode": "pro", "context": "all_turns"}, + drop_params=True, + **kwargs, + ) + + assert json.loads(route.calls[0].request.content)["reasoning"] == {"effort": "low", "context": "all_turns"} + + def test_responses_bridge_fake_stream_is_bounded_by_the_request_timeout_not_the_stream_timeout( restore_model_registry: None, ) -> None: @@ -5443,6 +5900,28 @@ def test_stream_chunk_builder_keeps_provider_reported_usage_cost(): assert response._hidden_params["response_cost"] == pytest.approx(0.5) +@pytest.mark.parametrize("reported_cost", [float("nan"), float("inf")]) +def test_stream_chunk_builder_bills_the_token_price_when_the_provider_cost_is_not_finite(reported_cost: float): + usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") + usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=reported_cost) + chunks: Final = [ + _stream_builder_text_chunk("gpt-4o", "Hello "), + _stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=_stream_builder_logging_obj() + ) + + assert response is not None + prompt_cost, completion_cost = litellm.cost_per_token(model="gpt-4o", usage_object=response.usage) + token_price: Final = prompt_cost + completion_cost + assert token_price > 0 + assert getattr(response.usage, "cost", None) == pytest.approx(token_price) + assert response._hidden_params["response_cost"] == pytest.approx(token_price) + + def test_completion_cost_on_a_stamped_stream_still_bills_the_reported_charge(): usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5) @@ -9783,9 +10262,10 @@ def test_a_non_numeric_thinking_budget_on_adaptive_claude_is_rejected_with_a_400 @pytest.mark.parametrize("drop_globally", [False, True]) @pytest.mark.parametrize("model_info", [None, _OPUS_DECLARING_LOW_AND_HIGH]) @pytest.mark.parametrize("body_sent", [_anthropic_chat_body_sent, _bedrock_converse_request_fields_sent]) -def test_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( +def test_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude_with_one_warning( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, body_sent: Callable[..., Mapping[str, object]], model_info: Mapping[str, object] | None, drop_globally: bool, @@ -9793,13 +10273,40 @@ def test_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( monkeypatch.setattr(litellm, "drop_params", drop_globally) request_drop: Final = {} if drop_globally else {"drop_params": True} - dropped: Final = body_sent( - respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + dropped: Final = body_sent( + respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] adaptive_alone: Final = body_sent(respx_mock, model_info, thinking={"type": "adaptive"}, **request_drop) assert "output_config" not in adaptive_alone assert dropped == adaptive_alone + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("thinking.budget_tokens='lots'", "claude-opus-5-5", "(drop_params=True)") + ) + + +@pytest.mark.parametrize("model_info", [None, _KIMI_DEPLOYMENT]) +def test_drop_params_drops_a_non_numeric_thinking_budget_on_hosted_vllm_with_one_warning( + respx_mock: respx.MockRouter, caplog: pytest.LogCaptureFixture, model_info: Mapping[str, object] | None +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + sent: Final = _chat_body_sent( + respx_mock, + "hosted_vllm/my-kimi", + None, + model_info, + thinking={"type": "enabled", "budget_tokens": "lots"}, + drop_params=True, + ) + + assert not {"reasoning_effort", "thinking"} & sent.keys() + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("thinking.budget_tokens='lots'", "my-kimi")) @pytest.mark.parametrize("model", ["azure_ai/my-kimi", "hosted_vllm/my-kimi"]) diff --git a/tests/unit/test_router/test_io_token_rate_limits.py b/tests/unit/test_router/test_io_token_rate_limits.py index e5ff4531f84..819c6b9d4f5 100644 --- a/tests/unit/test_router/test_io_token_rate_limits.py +++ b/tests/unit/test_router/test_io_token_rate_limits.py @@ -4,14 +4,19 @@ Tests for separate ITPM/OTPM deployment rate limits (enforce_model_rate_limits). import asyncio from collections.abc import Awaitable, Callable, Mapping -from datetime import datetime +from datetime import datetime, timezone from typing import Final +from unittest.mock import AsyncMock, MagicMock import pytest +from fakeredis import FakeRedis, FakeServer +from fakeredis.aioredis import FakeRedis as FakeAsyncRedis import litellm from litellm import Router from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -22,6 +27,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( OTPM_RESERVED_KEY, _explicit_max_tokens, _reservation_value, + async_admit_io_token_turn, async_io_token_pre_call_check, async_io_token_reconcile_success, build_io_token_rate_limit_headers, @@ -35,6 +41,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( from litellm.router_utils.pre_call_checks.model_rate_limit_check import ( ModelRateLimitingCheck, ) +from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.router import RouterCacheEnum from litellm.types.utils import ModelResponse, Usage from litellm.utils import get_utc_datetime @@ -115,7 +122,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 10, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) minute = get_utc_datetime().strftime("%H-%M") @@ -173,7 +180,8 @@ class TestModelRateLimitingCheckIOTokens: ], "max_tokens": 10, "metadata": {}, - } + }, + metadata_channel="metadata", ) with pytest.raises(litellm.RateLimitError) as exc_info: @@ -203,7 +211,8 @@ class TestModelRateLimitingCheckIOTokens: "messages": [{"role": "user", "content": "hi"}], "max_tokens": max_tokens, "metadata": {}, - } + }, + metadata_channel="metadata", ) async def _attempt(): @@ -255,7 +264,8 @@ class TestModelRateLimitingCheckIOTokens: { "max_tokens": 5, "metadata": {}, - } + }, + metadata_channel="metadata", ) async def _attempt(): @@ -320,7 +330,7 @@ class TestModelRateLimitingCheckIOTokens: } request_kwargs = {"max_tokens": 5, "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) minute = get_utc_datetime().strftime("%H-%M") @@ -363,15 +373,15 @@ class TestModelRateLimitingCheckIOTokens: "model_name": "opus", } request_kwargs = {"max_tokens": 5, "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) # Second zero-estimate call still fits within the itpm=2 limit. - set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}}) + set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}}, metadata_channel="metadata") await check.async_pre_call_check(deployment) # A third exceeds the limit and is rejected. - set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}}) + set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}}, metadata_channel="metadata") with pytest.raises(litellm.RateLimitError): await check.async_pre_call_check(deployment) @@ -392,7 +402,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 0, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(zero_output_kwargs) + set_io_token_rate_limit_request_kwargs(zero_output_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) zero_output_otpm_key = zero_output_kwargs["metadata"][OTPM_CACHE_KEY] @@ -404,7 +414,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 5, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(normal_output_kwargs) + set_io_token_rate_limit_request_kwargs(normal_output_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) normal_output_otpm_key = normal_output_kwargs["metadata"][OTPM_CACHE_KEY] @@ -432,7 +442,7 @@ class TestModelRateLimitingCheckIOTokens: "model_name": "no-max-tokens", } request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") if use_sync_check: check.pre_call_check(deployment) @@ -462,7 +472,7 @@ class TestModelRateLimitingCheckIOTokens: async def _attempt() -> dict[str, object] | None: request_kwargs: Final = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") try: await check.async_pre_call_check(deployment) except litellm.RateLimitError: @@ -493,7 +503,7 @@ class TestModelRateLimitingCheckIOTokens: "model_name": "roomy", } request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) @@ -507,7 +517,7 @@ class TestModelRateLimitingCheckIOTokens: async def _reserved(model_info: dict[str, str]) -> int: request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") deployment: Final = { "litellm_params": {"model": "azure/litellm-io-custom-deployment", "otpm": otpm_limit}, "model_info": model_info, @@ -536,7 +546,7 @@ class TestModelRateLimitingCheckIOTokens: } async def _pre_call(request_kwargs: dict[str, object]) -> None: - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") if use_sync_check: check.pre_call_check(deployment) else: @@ -563,12 +573,14 @@ class TestModelRateLimitingCheckIOTokens: "model_name": "opus", } request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 100, "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) assert request_kwargs["metadata"][OTPM_RESERVED_KEY] == 100 - set_io_token_rate_limit_request_kwargs({"messages": [{"role": "user", "content": "hi"}], "metadata": {}}) + set_io_token_rate_limit_request_kwargs( + {"messages": [{"role": "user", "content": "hi"}], "metadata": {}}, metadata_channel="metadata" + ) with pytest.raises(litellm.RateLimitError): await check.async_pre_call_check(deployment) @@ -585,7 +597,7 @@ class TestModelRateLimitingCheckIOTokens: with_cap: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 300, "metadata": {}} for request_kwargs in (without_cap, with_cap): - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") if use_sync_check: check.pre_call_check(deployment) else: @@ -602,7 +614,7 @@ class TestModelRateLimitingCheckIOTokens: async def reserved(deployment_cap: Mapping[str, int], request_shape: Mapping[str, object]) -> int: request_kwargs: Final = {**request_shape, "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check( { "litellm_params": {"model": MANTLE_OPUS, "otpm": 4000, **deployment_cap}, @@ -631,7 +643,7 @@ class TestModelRateLimitingCheckIOTokens: async def reserved(deployment_cap: Mapping[str, int]) -> int: request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") deployment: Final = { "litellm_params": {"model": UNMAPPED_MODEL, "otpm": otpm_limit, **deployment_cap}, "model_info": {"id": f"io-deployment-cap-above-default-{deployment_cap}"}, @@ -667,7 +679,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 10, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") check.pre_call_check(deployment) minute = get_utc_datetime().strftime("%H-%M") @@ -803,7 +815,7 @@ class TestModelRateLimitingCheckIOTokens: # The retry proceeds to stash deployment B's own reservation on the # same dict; it starts clean, unaffected by A's cleared sentinels. - set_io_token_rate_limit_request_kwargs(shared_kwargs) + set_io_token_rate_limit_request_kwargs(shared_kwargs, metadata_channel="metadata") itpm_key_b = f"global_router:io-retry-b:bedrock_mantle/test-b:itpm:{minute}" shared_kwargs["metadata"][ITPM_RESERVED_KEY] = 4 shared_kwargs["metadata"][ITPM_CACHE_KEY] = itpm_key_b @@ -819,7 +831,7 @@ class TestModelRateLimitingCheckIOTokens: "litellm_metadata": {OTPM_RESERVED_KEY: 7}, "litellm_params": {"metadata": {OTPM_CACHE_KEY: "attacker:key:otpm:00-00"}}, } - set_io_token_rate_limit_request_kwargs(forged) + set_io_token_rate_limit_request_kwargs(forged, metadata_channel="metadata") stored = get_io_token_rate_limit_request_kwargs() assert ITPM_RESERVED_KEY not in stored["metadata"] @@ -845,7 +857,7 @@ class TestModelRateLimitingCheckIOTokens: }, } # The router sanitizes the request kwargs before the call runs. - set_io_token_rate_limit_request_kwargs(kwargs) + set_io_token_rate_limit_request_kwargs(kwargs, metadata_channel="metadata") response = ModelResponse( choices=[{"message": {"role": "assistant", "content": "ok"}, "index": 0, "finish_reason": "stop"}], usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), @@ -880,7 +892,8 @@ class TestModelRateLimitingCheckIOTokens: "messages": [{"role": "user", "content": "hello world"}], "max_tokens": 5, "metadata": {}, - } + }, + metadata_channel="metadata", ) with pytest.raises(RuntimeError): @@ -945,7 +958,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 5, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") result = await async_io_token_pre_call_check(dual_cache, deployment) @@ -1147,7 +1160,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 5, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): with pytest.raises(litellm.RateLimitError): @@ -1495,15 +1508,15 @@ class TestContextSlotRetention: "messages": [{"role": "user", "content": "x" * 1000}], "metadata": {ITPM_RESERVED_KEY: 999, ITPM_CACHE_KEY: "forged"}, } - set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False) + set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False, metadata_channel="metadata") assert get_io_token_rate_limit_request_kwargs() is None assert ITPM_RESERVED_KEY not in kwargs["metadata"] assert ITPM_CACHE_KEY not in kwargs["metadata"] - set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=True) + set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=True, metadata_channel="metadata") assert get_io_token_rate_limit_request_kwargs() is kwargs - set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False) + set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False, metadata_channel="metadata") assert get_io_token_rate_limit_request_kwargs() is None @pytest.mark.asyncio @@ -1516,7 +1529,7 @@ class TestContextSlotRetention: } ] ) - set_io_token_rate_limit_request_kwargs(None) + set_io_token_rate_limit_request_kwargs(None, metadata_channel="metadata") kwargs = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} deployment = router.get_deployment_by_model_group_name("plain") assert deployment is not None @@ -1534,7 +1547,7 @@ class TestContextSlotRetention: ], optional_pre_call_checks=["enforce_model_rate_limits"], ) - set_io_token_rate_limit_request_kwargs(None) + set_io_token_rate_limit_request_kwargs(None, metadata_channel="metadata") kwargs = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} deployment = router.get_deployment_by_model_group_name("limited") assert deployment is not None @@ -1551,7 +1564,7 @@ async def test_custom_azure_deployment_name_reserves_itpm_on_its_base_models_tok "max_tokens": 5, "metadata": metadata, } - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await async_io_token_pre_call_check(DualCache(), deployment) return metadata[ITPM_RESERVED_KEY] @@ -1571,3 +1584,278 @@ async def test_custom_azure_deployment_name_reserves_itpm_on_its_base_models_tok ) assert named == served + + +def _reply_using(prompt_tokens: int, completion_tokens: int) -> ModelResponse: + return ModelResponse( + choices=[{"message": {"role": "assistant", "content": "hi"}, "index": 0, "finish_reason": "stop"}], + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +def _proxy_success_kwargs(reservation: Mapping[str, object]) -> dict[str, object]: + return { + "litellm_params": {"metadata": {**reservation, "user_api_key_hash": "hashed-key"}}, + "standard_logging_object": { + "hidden_params": {"litellm_model_name": "openai/gpt-5-mini"}, + "metadata": {"user_api_key_hash": "hashed-key"}, + }, + } + + +async def _deliver_success( + check: ModelRateLimitingCheck, delivery: str, kwargs: dict[str, object], reply: object +) -> None: + if delivery == "async": + await check.async_log_success_event(kwargs, reply, None, None) + return + check.log_success_event(kwargs, reply, None, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "deliveries", + [("async",), ("sync",), ("async", "async"), ("async", "sync"), ("sync", "async"), ("sync", "sync")], +) +async def test_success_settles_a_reservation_held_only_in_litellm_params_metadata_exactly_once( + deliveries: tuple[str, ...], +) -> None: + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + itpm_key: Final = "global_router:proxy-shape:openai/gpt-5-mini:itpm:12-34" + otpm_key: Final = "global_router:proxy-shape:openai/gpt-5-mini:otpm:12-34" + await dual_cache.async_increment_cache(key=itpm_key, value=40, ttl=60) + await dual_cache.async_increment_cache(key=otpm_key, value=4096, ttl=60) + kwargs: Final = _proxy_success_kwargs( + {ITPM_RESERVED_KEY: 40, OTPM_RESERVED_KEY: 4096, ITPM_CACHE_KEY: itpm_key, OTPM_CACHE_KEY: otpm_key} + ) + + for delivery in deliveries: + await _deliver_success(check, delivery, kwargs, _reply_using(prompt_tokens=7, completion_tokens=3)) + + assert await dual_cache.async_get_cache(key=itpm_key) == 7 + assert await dual_cache.async_get_cache(key=otpm_key) == 3 + + +_TURN_AT: Final = datetime(2026, 1, 1, 12, 34, 56, tzinfo=timezone.utc) +_ACTUAL_TOKENS: Final = {"itpm": 5, "otpm": 2} + + +def _turn_deployment(kind: str) -> dict[str, object]: + return {"litellm_params": {"model": "openai/gpt-5-mini", kind: 100}, "model_info": {"id": f"ws-turn-{kind}"}} + + +def _turn_key(kind: str, minute: str = "12-34") -> str: + return f"global_router:ws-turn-{kind}:openai/gpt-5-mini:{kind}:{minute}" + + +@pytest.fixture +def settled_on_the_turn_s_minute(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("used", [100, 101, 250]) +async def test_turn_admission_refuses_once_the_minutes_tokens_reach_the_limit(kind: str, used: int) -> None: + dual_cache: Final = DualCache() + await dual_cache.async_increment_cache(key=_turn_key(kind), value=used, ttl=60) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage={used}"): + await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("used", [0, 1, 99]) +async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(kind: str, used: int) -> None: + dual_cache: Final = DualCache() + await dual_cache.async_increment_cache(key=_turn_key(kind), value=used, ttl=60) + actual: Final = _ACTUAL_TOKENS + + reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + counter_key: Final = reservation[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] + assert isinstance(counter_key, str) + assert await dual_cache.async_get_cache(key=counter_key) == used + await async_io_token_reconcile_success( + dual_cache, _proxy_success_kwargs(reservation), _reply_using(actual["itpm"], actual["otpm"]) + ) + + assert await dual_cache.async_get_cache(key=counter_key) == used + actual[kind] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "deployment", + [ + {"litellm_params": {"model": "openai/gpt-5-mini", "tpm": 1}, "model_info": {"id": "ws-turn-unlimited"}}, + {"litellm_params": {"model": "openai/gpt-5-mini", "otpm": 1}}, + ], +) +async def test_a_turn_without_io_token_counters_is_admitted_with_nothing_to_settle( + deployment: dict[str, object], +) -> None: + assert await async_admit_io_token_turn(DualCache(), deployment, _TURN_AT) == {} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("used_by_others", [0, 1, 99]) +async def test_a_socket_open_reserves_nothing_and_its_first_turn_counts_its_actual_tokens_on_its_own_minute( + kind: str, used_by_others: int +) -> None: + dual_cache: Final = DualCache() + await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others, ttl=60) + socket_open: Final[dict[str, object]] = {"metadata": {}} + set_io_token_rate_limit_request_kwargs(socket_open, opens_responses_socket=True, metadata_channel="metadata") + + assert await async_io_token_pre_call_check(dual_cache, _turn_deployment(kind)) == _turn_deployment(kind) + assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others + + opened_with: Final = socket_open["metadata"] + assert isinstance(opened_with, dict) + await async_io_token_reconcile_success( + dual_cache, _proxy_success_kwargs(opened_with), _reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]) + ) + + assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others + _ACTUAL_TOKENS[kind] + + +class _RedisCacheOnFakeServer(RedisCache): + def __init__(self, server: FakeServer) -> None: + super().__init__(host="io-token-turn-redis", connection_pool=FakeRedis(server=server).connection_pool) + self._fake_async_client: Final = FakeAsyncRedis(server=server) + + def init_async_client(self) -> FakeAsyncRedis: + return self._fake_async_client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_a_turn_is_refused_on_this_instance_s_count_when_reading_the_shared_counter_fails( + kind: str, +) -> None: + server: Final = FakeServer() + dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnFakeServer(server)) + server.connected = False + await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"): + await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("delivery", ["async", "sync"]) +async def test_a_turn_finishing_in_the_next_minute_counts_its_actual_tokens_on_that_minute( + kind: str, delivery: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", + lambda: datetime(2026, 1, 1, 12, 35, 10, tzinfo=timezone.utc), + ) + dual_cache: Final = DualCache() + reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + await _deliver_success( + ModelRateLimitingCheck(dual_cache=dual_cache), + delivery, + _proxy_success_kwargs(reservation), + _reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]), + ) + + assert [await dual_cache.async_get_cache(key=_turn_key(kind, minute)) for minute in ("12-34", "12-35")] == [ + None, + _ACTUAL_TOKENS[kind], + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_this_instance_last_read( + kind: str, +) -> None: + redis_cache: Final = _RedisCacheOnFakeServer(FakeServer()) + dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache) + await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100) + await redis_cache.async_increment(key=_turn_key(kind), value=99) + + admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + assert admitted[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] == _turn_key(kind) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "responses_reply", + [ + pytest.param( + ResponsesAPIResponse( + id="resp_cached", + created_at=0, + output=[], + usage=ResponseAPIUsage( + input_tokens=50, + output_tokens=5, + total_tokens=55, + input_tokens_details=InputTokensDetails(cached_tokens=30), + ), + ), + id="usage_object", + ), + pytest.param( + { + "usage": { + "input_tokens": 50, + "output_tokens": 5, + "total_tokens": 55, + "input_tokens_details": {"cached_tokens": 30}, + } + }, + id="usage_dict", + ), + ], +) +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") +async def test_a_turn_counts_only_its_uncached_responses_input_toward_itpm(responses_reply: object) -> None: + dual_cache: Final = DualCache() + await dual_cache.async_increment_cache(key=_turn_key("itpm"), value=10, ttl=60) + + reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment("itpm"), _TURN_AT) + await async_io_token_reconcile_success(dual_cache, _proxy_success_kwargs(reservation), responses_reply) + + assert await dual_cache.async_get_cache(key=_turn_key("itpm")) == 10 + 50 - 30 + + +def _shared_counter_answering(answer: int | Exception) -> DualCache: + redis_cache: Final = MagicMock(spec=RedisCache) + redis_cache.async_get_cache = AsyncMock(side_effect=[answer]) + return DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_a_later_turn_is_refused_when_another_instance_filled_the_shared_minute(kind: str) -> None: + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"): + await async_admit_io_token_turn(_shared_counter_answering(100), _turn_deployment(kind), _TURN_AT) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_a_later_turn_is_admitted_on_this_instance_s_count_when_the_shared_counter_is_unreachable( + kind: str, +) -> None: + dual_cache: Final = _shared_counter_answering(RedisCircuitBreakerOpenError("circuit open")) + await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=99) + + admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + assert admitted[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] == _turn_key(kind) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a0a7dd78193..a6436e47507 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -4,6 +4,7 @@ import copy import functools import gc import itertools +import io import json import linecache import logging @@ -2199,7 +2200,11 @@ async def test_ageneric_api_call_deployment_model_overrides_alias(): ) def inject_alias_into_kwargs( - deployment: dict, kwargs: dict, function_name: str | None = None, splits_stream_timeout: bool = False + deployment: dict, + kwargs: dict, + function_name: str | None = None, + splits_stream_timeout: bool = False, + opens_responses_socket: bool = False, ) -> None: # Simulate the alias leaking into kwargs (as happens when # _ageneric_api_call_with_fallbacks sets kwargs["model"] = alias before @@ -2730,6 +2735,39 @@ def test_update_settings_model_group_alias_drops_cached_group_info(): assert after.input_cost_per_token is not None and after.input_cost_per_token > 0 +@pytest.mark.parametrize( + "fallback_kind, mock_testing_param", + [ + ("fallbacks", "mock_testing_fallbacks"), + ("context_window_fallbacks", "mock_testing_context_fallbacks"), + ("content_policy_fallbacks", "mock_testing_content_policy_fallbacks"), + ], +) +async def test_update_settings_fallback_rule_of_every_kind_is_used_on_its_error( + fallback_kind: str, mock_testing_param: str +): + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "gpt-5.4-mini", "api_key": "fake", "mock_response": "primary answered"}, + }, + { + "model_name": "backup", + "litellm_params": {"model": "gpt-5.4-nano", "api_key": "fake", "mock_response": "backup answered"}, + }, + ], + num_retries=0, + ) + + router.update_settings(**{fallback_kind: [{"primary": ["backup"]}]}) + + response: Final = await router.acompletion( + model="primary", messages=[{"role": "user", "content": "hi"}], **{mock_testing_param: True} + ) + assert response.choices[0].message.content == "backup answered" + + _PAID_INPUT_COST_PER_TOKEN: Final = 3e-06 _PAID_OUTPUT_COST_PER_TOKEN: Final = 1.5e-05 @@ -31523,6 +31561,224 @@ def test_router_keeps_supported_fallback_shapes( assert router.fallbacks == expected_fallbacks +_SAGEMAKER_ROUTER_CREDENTIALS: Final = MappingProxyType( + {"aws_region_name": "us-east-1", "aws_access_key_id": "test-key", "aws_secret_access_key": "test-secret"} +) + +_PINNED_COMPONENT: Final = MappingProxyType({"model_id": "deployment-component"}) +_HEADER_PINNED_COMPONENT: Final = MappingProxyType( + {"extra_headers": {"x-amzn-sagemaker-inference-component": "deployment-component"}} +) + +_INFERENCE_COMPONENT_PRECEDENCE: Final = pytest.mark.parametrize( + ("deployment_params", "sent_component"), + [ + (_PINNED_COMPONENT, "deployment-component"), + (_HEADER_PINNED_COMPONENT, "deployment-component"), + (MappingProxyType({}), "client-component"), + ( + MappingProxyType({**_PINNED_COMPONENT, "configurable_clientside_auth_params": ["model_id"]}), + "client-component", + ), + ], + ids=[ + "deployment_pins_the_component", + "deployment_pins_the_component_with_a_header", + "deployment_leaves_the_component_to_the_client", + "deployment_lets_the_client_override_its_component", + ], +) + + +def _sagemaker_router( + model: str, + deployment_params: Mapping[str, object], + mode: str, + router_defaults: Mapping[str, str] = MappingProxyType({}), + model_name: str = "sagemaker-model", +) -> Router: + return Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": {"model": model, **_SAGEMAKER_ROUTER_CREDENTIALS, **deployment_params}, + "model_info": {"mode": mode}, + } + ], + default_litellm_params=dict(router_defaults), + ) + + +_CHAT_COMPLETION_REPLY: Final = MappingProxyType( + { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "served-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } +) + + +def _sagemaker_chat_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post( + "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/chat-endpoint/invocations" + ).respond(json=dict(_CHAT_COMPLETION_REPLY)) + + +@_INFERENCE_COMPONENT_PRECEDENCE +@pytest.mark.asyncio +async def test_a_client_model_id_cannot_move_a_sagemaker_chat_call_off_the_deployments_component( + respx_mock: respx.MockRouter, + _router_httpx_transport: None, + deployment_params: Mapping[str, object], + sent_component: str, +) -> None: + route: Final = _sagemaker_chat_route(respx_mock) + router: Final = _sagemaker_router("sagemaker_chat/chat-endpoint", deployment_params, "chat") + + await router.acompletion( + model="sagemaker-model", messages=[{"role": "user", "content": "hi"}], model_id="client-component" + ) + + assert route.calls.last.request.headers["X-Amzn-SageMaker-Inference-Component"] == sent_component + + +@_INFERENCE_COMPONENT_PRECEDENCE +def test_a_client_model_id_cannot_move_a_sync_sagemaker_text_completion_off_the_deployments_component( + respx_mock: respx.MockRouter, deployment_params: Mapping[str, object], sent_component: str +) -> None: + route: Final = _sagemaker_chat_route(respx_mock) + router: Final = _sagemaker_router( + "sagemaker_chat/chat-endpoint", deployment_params, "completion", model_name="sagemaker_chat/chat-endpoint" + ) + + router.text_completion(model="sagemaker_chat/chat-endpoint", prompt="hi", model_id="client-component") + + assert route.calls.last.request.headers["X-Amzn-SageMaker-Inference-Component"] == sent_component + + +@pytest.mark.parametrize( + "client_header", + [ + MappingProxyType({"extra_headers": {"x-amzn-sagemaker-inference-component": "client-component"}}), + MappingProxyType({"headers": {"x-amzn-sagemaker-inference-component": "client-component"}}), + MappingProxyType( + { + "provider_specific_header": { + "custom_llm_provider": "sagemaker_chat", + "extra_headers": {"x-amzn-sagemaker-inference-component": "client-component"}, + } + } + ), + ], + ids=["extra_headers", "headers", "provider_specific_header"], +) +@pytest.mark.parametrize( + "deployment_pin", [_PINNED_COMPONENT, _HEADER_PINNED_COMPONENT], ids=["model_id_pin", "header_pin"] +) +@pytest.mark.asyncio +async def test_a_client_inference_component_header_cannot_move_a_pinned_sagemaker_chat_call( + respx_mock: respx.MockRouter, + _router_httpx_transport: None, + client_header: Mapping[str, object], + deployment_pin: Mapping[str, object], +) -> None: + route: Final = _sagemaker_chat_route(respx_mock) + router: Final = _sagemaker_router("sagemaker_chat/chat-endpoint", deployment_pin, "chat") + + await router.acompletion(model="sagemaker-model", messages=[{"role": "user", "content": "hi"}], **client_header) + + assert route.calls.last.request.headers["X-Amzn-SageMaker-Inference-Component"] == "deployment-component" + + +@pytest.mark.parametrize( + "request_component", + [MappingProxyType({}), MappingProxyType({"model_id": "deployment-component"})], + ids=["request_without_model_id", "request_with_the_pinned_model_id"], +) +@pytest.mark.asyncio +async def test_a_router_default_model_id_cannot_move_a_sagemaker_call_off_the_deployments_component( + respx_mock: respx.MockRouter, _router_httpx_transport: None, request_component: Mapping[str, str] +) -> None: + route: Final = _sagemaker_chat_route(respx_mock) + router: Final = _sagemaker_router( + "sagemaker_chat/chat-endpoint", + _PINNED_COMPONENT, + "chat", + router_defaults=MappingProxyType({"model_id": "default-component"}), + ) + + await router.acompletion(model="sagemaker-model", messages=[{"role": "user", "content": "hi"}], **request_component) + + assert route.calls.last.request.headers["X-Amzn-SageMaker-Inference-Component"] == "deployment-component" + + +@_INFERENCE_COMPONENT_PRECEDENCE +@pytest.mark.asyncio +async def test_a_client_model_id_cannot_move_a_sagemaker_embedding_call_off_the_deployments_component( + deployment_params: Mapping[str, object], sent_component: str +) -> None: + component_embeddings: Final = MappingProxyType({"deployment-component": [0.1], "client-component": [0.2]}) + + def invoke_endpoint(**request: str | bytes) -> Mapping[str, io.BytesIO]: + served: Final = component_embeddings[str(request["InferenceComponentName"])] + return { + "Body": io.BytesIO( + json.dumps({"object": "list", "data": [{"object": "embedding", "index": 0, "embedding": served}]}).encode() + ) + } + + client: Final = MagicMock(invoke_endpoint=invoke_endpoint) + router: Final = _sagemaker_router("sagemaker/openai/embed-endpoint", deployment_params, "embedding") + + with patch("boto3.Session", return_value=MagicMock(client=MagicMock(return_value=client))): + response: Final = await router.aembedding(model="sagemaker-model", input=["hello"], model_id="client-component") + + assert response.data[0]["embedding"] == component_embeddings[sent_component] + + +@pytest.mark.asyncio +async def test_a_header_pin_holds_when_the_deployment_names_sagemaker_through_custom_llm_provider( + respx_mock: respx.MockRouter, _router_httpx_transport: None +) -> None: + route: Final = _sagemaker_chat_route(respx_mock) + router: Final = _sagemaker_router( + "chat-endpoint", MappingProxyType({**_HEADER_PINNED_COMPONENT, "custom_llm_provider": "sagemaker_chat"}), "chat" + ) + + await router.acompletion( + model="sagemaker-model", messages=[{"role": "user", "content": "hi"}], model_id="client-component" + ) + + assert route.calls.last.request.headers["X-Amzn-SageMaker-Inference-Component"] == "deployment-component" + + +@pytest.mark.asyncio +async def test_a_component_header_on_a_non_sagemaker_deployment_leaves_the_clients_model_id_alone( + respx_mock: respx.MockRouter, _router_httpx_transport: None +) -> None: + route: Final = respx_mock.post("https://gateway.local/v1/chat/completions").respond(json=dict(_CHAT_COMPLETION_REPLY)) + router: Final = Router( + model_list=[ + { + "model_name": "gateway-model", + "litellm_params": { + "model": "openai/served-model", + "api_base": "https://gateway.local/v1", + "api_key": "test-key", + "extra_headers": {"X-Amzn-SageMaker-Inference-Component": "gateway-component"}, + }, + } + ] + ) + + await router.acompletion(model="gateway-model", messages=[{"role": "user", "content": "hi"}], model_id="client-value") + + assert json.loads(route.calls.last.request.content)["model_id"] == "client-value" + + def test_router_warns_when_a_deployment_enforces_endpoints_without_a_list(caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): router: Final = Router( diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 5f1b6032ca8..5f31ded626d 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -4,13 +4,15 @@ import gc import json import os import weakref -from collections.abc import Iterator, Mapping +from collections.abc import Awaitable, Callable, Iterator, Mapping from datetime import datetime, timezone from types import MethodType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +import respx import litellm from litellm import Router @@ -538,14 +540,14 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( assert await router.cache.async_get_cache(key=tpm_key) == 17 -async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]: +async def _kwargs_routed_to(router: Router, model: str, function_name: str, **request: object) -> dict[str, object]: routed: Final[dict[str, object]] = {} async def call(**kwargs: object) -> None: routed.update(kwargs) call.__name__ = function_name - await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call) + await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call, **request) return routed @@ -715,6 +717,363 @@ async def test_responses_websocket_turns_do_not_reserve_deployment_otpm(): await admit_turn() +def _io_limited_router(kind: str, **litellm_params: object) -> Router: + return Router( + model_list=[ + { + "model_name": "ws-io", + "litellm_params": { + "model": "openai/ws-io-unpriced", + "api_key": "sk-fake", + kind: 10_000, + **litellm_params, + }, + "model_info": {"id": "ws-io-dep"}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + ) + + +def _minutes_counter_key(kind: str, minute: datetime, model: str = "openai/ws-io-unpriced") -> str: + return f"global_router:ws-io-dep:{model}:{kind}:{minute.strftime('%H-%M')}" + + +async def _count_minutes_tokens( + router: Router, kind: str, tokens: int, minute: datetime, model: str = "openai/ws-io-unpriced" +) -> None: + await router.cache.async_increment_cache(key=_minutes_counter_key(kind, minute, model), value=tokens, ttl=120) + + +async def _settle_turn( + router: Router, reservation: Mapping[str, object], input_tokens: int, output_tokens: int +) -> None: + from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + rate_limits: Final = next( + check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck) + ) + await rate_limits.async_log_success_event( + {"litellm_params": get_litellm_params(litellm_metadata=dict(reservation))}, + ResponsesWebSocketTokenUsageProcessor.create_logging_object( + [ + { + "type": "response.completed", + "response": {"usage": {"input_tokens": input_tokens, "output_tokens": output_tokens}}, + } + ] + ), + None, + None, + ) + + +_LONG_OUTPUT_MODEL: Final = "openai/gpt-5.4-mini" + + +def _long_output_router() -> tuple[Router, Mapping[str, int]]: + max_output: Final = litellm.get_model_info(_LONG_OUTPUT_MODEL)["max_output_tokens"] + assert isinstance(max_output, int) + limits: Final = {"itpm": 10_000, "otpm": max_output // 2} + return _io_limited_router("itpm", model=_LONG_OUTPUT_MODEL, otpm=limits["otpm"]), limits + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_responses_websocket_opens_without_counting_tokens_until_the_minute_reaches_deployment_io_limit( + kind: str, router_minute_pinned: datetime +): + router, limits = _long_output_router() + + await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()) + + assert [ + await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL)) + for counted in ("itpm", "otpm") + ] == [None, None] + await _count_minutes_tokens(router, kind, limits[kind], router_minute_pinned, _LONG_OUTPUT_MODEL) + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit={limits[kind]}, current usage="): + await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()) + + +@pytest.mark.asyncio +async def test_responses_websocket_first_turn_counts_its_actual_tokens_toward_deployment_itpm_and_otpm( + router_minute_pinned: datetime, +): + router, _ = _long_output_router() + socket_metadata: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()))[ + "litellm_metadata" + ] + assert isinstance(socket_metadata, dict) + + await _settle_turn(router, socket_metadata, input_tokens=20, output_tokens=15) + + assert [ + await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL)) + for counted in ("itpm", "otpm") + ] == [20, 15] + + +_ADMITTED_LATE_IN_THE_MINUTE: Final = datetime(2026, 1, 1, 12, 0, 50, tzinfo=timezone.utc) +_FINISHED_IN_THE_NEXT_MINUTE: Final = datetime(2026, 1, 1, 12, 1, 15, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("turn", ["first", "later"]) +async def test_responses_websocket_turn_finishing_in_the_next_minute_counts_its_tokens_toward_that_minute( + kind: str, turn: str, monkeypatch: pytest.MonkeyPatch +): + _pin_router_clock(monkeypatch, _ADMITTED_LATE_IN_THE_MINUTE) + router: Final = _io_limited_router(kind) + socket: Final = await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()) + admit_turn: Final = socket["turn_admission"] + assert callable(admit_turn) + reservation: Final = socket["litellm_metadata"] if turn == "first" else await admit_turn() + assert isinstance(reservation, Mapping) + _pin_router_clock(monkeypatch, _FINISHED_IN_THE_NEXT_MINUTE) + + await _settle_turn(router, reservation, input_tokens=10_000, output_tokens=10_000) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=10000, current usage=10000"): + await admit_turn() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [{}, {"websocket": True}], ids=["plain", "websocket_key"]) +async def test_messages_call_reserves_max_tokens_against_deployment_otpm_whatever_websocket_key_its_body_carries( + body: Mapping[str, object], router_minute_pinned: datetime +): + router: Final = _io_limited_router("otpm") + + for _ in range(3): + await _kwargs_routed_to(router, "ws-io", "anthropic_messages", max_tokens=3_000, **body) + + assert await router.cache.async_get_cache(key=_minutes_counter_key("otpm", router_minute_pinned)) == 9_000 + with pytest.raises(litellm.RateLimitError, match="OTPM limit=10000, current usage=12000"): + await _kwargs_routed_to(router, "ws-io", "anthropic_messages", max_tokens=3_000, **body) + + +async def _realtime_opens_until_refused(monkeypatch: pytest.MonkeyPatch, **socket: object) -> tuple[int, str]: + monkeypatch.setattr(litellm, "callbacks", []) + router: Final = _io_limited_router("otpm", otpm=50) + for opened in range(51): + try: + await _kwargs_routed_to(router, "ws-io", "_arealtime", **socket) + except litellm.RateLimitError as refused: + return opened, str(refused) + pytest.fail("realtime opens never reached the deployment's otpm") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_realtime_open_reserves_and_is_refused_at_deployment_otpm_like_an_open_without_a_socket( + monkeypatch: pytest.MonkeyPatch, +): + with_socket: Final = await _realtime_opens_until_refused(monkeypatch, websocket=object()) + + assert "OTPM limit=50, current usage=" in with_socket[1] + assert with_socket == await _realtime_opens_until_refused(monkeypatch) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_responses_websocket_turn_is_refused_once_the_minutes_tokens_reach_deployment_io_limit( + kind: str, router_minute_pinned: datetime +): + router: Final = _io_limited_router(kind) + admit_turn: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket"))["turn_admission"] + assert callable(admit_turn) + await _count_minutes_tokens(router, kind, 10_000, router_minute_pinned) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=10000, current usage="): + await admit_turn() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_responses_websocket_turn_refused_for_tokens_leaves_the_daily_request_limit_unused( + kind: str, router_minute_pinned: datetime +): + router: Final = _io_limited_router(kind, rpd=2) + admit_turn: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket"))["turn_admission"] + assert callable(admit_turn) + await _count_minutes_tokens(router, kind, 10_000, router_minute_pinned) + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=10000"): + await admit_turn() + await _count_minutes_tokens(router, kind, -10_000, router_minute_pinned) + + await admit_turn() + + with pytest.raises(litellm.RateLimitError, match="RPD"): + await admit_turn() + + +_PROVIDER_RESPONSE: Final = { + "id": "resp_io", + "object": "response", + "created_at": 1767268830, + "status": "completed", + "model": "ws-io-unpriced", + "output": [ + { + "type": "message", + "id": "msg_io", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "client_metadata", [None, {"session": "abc", "team": "search"}], ids=["without_metadata", "with_metadata"] +) +async def test_responses_call_sends_the_provider_only_the_client_metadata_and_settles_deployment_io_tokens( + client_metadata: Mapping[str, str] | None, + router_minute_pinned: datetime, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +): + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + from litellm.responses.streaming_iterator import drain_logging_tasks + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + provider: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_PROVIDER_RESPONSE) + ) + router: Final = _io_limited_router("itpm", otpm=10_000) + + await router.aresponses( + model="ws-io", + input="hi", + max_output_tokens=50, + **({} if client_metadata is None else {"metadata": dict(client_metadata)}), + ) + await drain_logging_tasks() + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + assert [json.loads(call.request.read()).get("metadata") for call in provider.calls] == [client_metadata] + assert [ + await router.cache.async_get_cache(key=_minutes_counter_key(kind, router_minute_pinned)) + for kind in ("itpm", "otpm") + ] == [10, 5] + + +@pytest.mark.parametrize( + "client_metadata", [None, {"session": "abc", "team": "search"}], ids=["without_metadata", "with_metadata"] +) +def test_sync_responses_call_sends_the_provider_only_the_client_metadata( + client_metadata: Mapping[str, str] | None, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + litellm.in_memory_llm_clients_cache.flush_cache() + provider: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_PROVIDER_RESPONSE) + ) + + _io_limited_router("itpm", otpm=10_000).responses( + model="ws-io", + input="hi", + max_output_tokens=50, + **({} if client_metadata is None else {"metadata": dict(client_metadata)}), + ) + + assert [json.loads(call.request.read()).get("metadata") for call in provider.calls] == [client_metadata] + + +_ANTHROPIC_PROVIDER_RESPONSE: Final = { + "id": "msg_io", + "type": "message", + "role": "assistant", + "model": "ws-io-unpriced", + "content": [{"type": "text", "text": "Done."}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, +} + + +async def _responses_call(router: Router) -> object: + return await router.aresponses(model="ws-io", input="hi", max_output_tokens=50) + + +async def _messages_call(router: Router) -> object: + return await router.aanthropic_messages(model="ws-io", messages=[{"role": "user", "content": "hi"}], max_tokens=50) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "provider_url", "provider_response", "call"), + [ + pytest.param( + "openai/ws-io-unpriced", + "https://api.openai.com/v1/responses", + _PROVIDER_RESPONSE, + _responses_call, + id="responses", + ), + pytest.param( + "anthropic/ws-io-unpriced", + "https://api.anthropic.com/v1/messages", + _ANTHROPIC_PROVIDER_RESPONSE, + _messages_call, + id="messages", + ), + ], +) +async def test_retried_call_settles_deployment_io_tokens_to_the_successful_attempt_s_tokens( + model: str, + provider_url: str, + provider_response: Mapping[str, object], + call: Callable[[Router], Awaitable[object]], + router_minute_pinned: datetime, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +): + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + from litellm.responses.streaming_iterator import drain_logging_tasks + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + provider: Final = respx_mock.post(provider_url).mock( + side_effect=[ + httpx.Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}}), + httpx.Response(200, json=provider_response), + ] + ) + router: Final = Router( + model_list=[ + { + "model_name": "ws-io", + "litellm_params": {"model": model, "api_key": "sk-fake", "itpm": 10_000, "otpm": 10_000}, + "model_info": {"id": "ws-io-dep"}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + num_retries=1, + retry_after=0, + ) + + await call(router) + await drain_logging_tasks() + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + assert provider.call_count == 2 + assert [ + await router.cache.async_get_cache(key=_minutes_counter_key(kind, router_minute_pinned, model)) + for kind in ("itpm", "otpm") + ] == [10, 5] + + @pytest.mark.asyncio async def test_deployment_callback_on_success_tracks_tpm_for_io_deployment(): """ @@ -1194,10 +1553,17 @@ def _rpm_tpm_router(model_id: str) -> Router: ) +def _pin_router_clock(monkeypatch: pytest.MonkeyPatch, pinned: datetime) -> None: + monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: pinned + ) + + @pytest.fixture def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime: pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) - monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) + _pin_router_clock(monkeypatch, pinned) return pinned diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx index 117796d658e..56135c779e9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx @@ -173,6 +173,24 @@ describe("RealtimePlayground", () => { expect(update.session.type).toBe("realtime"); }); + it("starts recording on the configuration the session was created with", async () => { + const user = userEvent.setup(); + render(); + + await connect(user); + await act(async () => { + latestSocket().emit({ type: "session.created" }); + }); + await user.click(screen.getByTitle("Start recording")); + + const updates = latestSocket() + .sent.map((raw) => JSON.parse(raw)) + .filter((payload) => payload.type === "session.update"); + expect(updates).toHaveLength(2); + expect(updates[1]).toEqual(updates[0]); + expect(updates[0].session.turn_detection).toEqual({ type: "server_vad" }); + }); + it("sends what was typed and then asks for a response", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx index 01151203e65..408cbd58593 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx @@ -22,6 +22,20 @@ interface RealtimePlaygroundProps { selectedGuardrails?: string[]; } +const sessionUpdate = (voice: string) => + JSON.stringify({ + type: "session.update", + session: { + type: "realtime", + modalities: ["text", "audio"], + voice, + input_audio_format: "pcm16", + output_audio_format: "pcm16", + input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, + turn_detection: { type: "server_vad" }, + }, + }); + const RealtimePlayground: React.FC = ({ accessToken, selectedModel, @@ -125,21 +139,7 @@ const RealtimePlayground: React.FC = ({ const type = data.type; if (type === "session.created") { - // GA: session.type is required ("realtime" | "transcription") - ws.send( - JSON.stringify({ - type: "session.update", - session: { - type: "realtime", - modalities: ["text", "audio"], - voice: selectedVoice, - input_audio_format: "pcm16", - output_audio_format: "pcm16", - input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, - turn_detection: null, - }, - }), - ); + ws.send(sessionUpdate(selectedVoice)); } else if (type === "session.updated") { // session configured } else if ( @@ -231,22 +231,7 @@ const RealtimePlayground: React.FC = ({ const startRecording = useCallback(async () => { if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; - // Switch to server VAD mode for voice input - // GA: session.type is required - wsRef.current.send( - JSON.stringify({ - type: "session.update", - session: { - type: "realtime", - modalities: ["text", "audio"], - voice: selectedVoice, - input_audio_format: "pcm16", - output_audio_format: "pcm16", - input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, - turn_detection: { type: "server_vad" }, - }, - }), - ); + wsRef.current.send(sessionUpdate(selectedVoice)); try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); @@ -301,7 +286,7 @@ const RealtimePlayground: React.FC = ({ } catch (err: any) { addMessage("status", `Microphone error: ${err.message}`); } - }, [addMessage]); + }, [addMessage, selectedVoice]); const stopRecording = useCallback(() => { processorRef.current?.disconnect(); @@ -317,21 +302,7 @@ const RealtimePlayground: React.FC = ({ if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; if (configureSessionRef.current) return; configureSessionRef.current = true; - // GA: session.type is required - wsRef.current.send( - JSON.stringify({ - type: "session.update", - session: { - type: "realtime", - modalities: ["text", "audio"], - voice: selectedVoice, - input_audio_format: "pcm16", - output_audio_format: "pcm16", - input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, - turn_detection: null, - }, - }), - ); + wsRef.current.send(sessionUpdate(selectedVoice)); }, [selectedVoice]); const sendTextMessage = useCallback(() => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx index 660168fa6ab..5c8b284c72f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx @@ -1160,6 +1160,34 @@ describe("EntityUsage", () => { expect(screen.getByTestId("provider-spend-total")).toHaveTextContent("$100.50"); }); + it("lists a subscription-covered provider on the team view as included in the subscription at zero spend", async () => { + const day = mockSpendData.results[0]; + mockTeamDailyActivityCall.mockResolvedValue({ + ...mockSpendData, + results: [ + { + ...day, + breakdown: { + ...day.breakdown, + providers: { + ...day.breakdown.providers, + chatgpt: { + metrics: { ...day.breakdown.providers.openai.metrics, spend: 0, api_requests: 4 }, + metadata: { subscription_covered: true }, + }, + }, + }, + }, + ], + }); + + render(); + + expect(await screen.findByText("Included in subscription")).toBeInTheDocument(); + expect(screen.getByText("chatgpt")).toBeInTheDocument(); + expect(screen.getAllByTestId("provider-share-segment")).toHaveLength(1); + }); + it("should label the chart with user_email metadata instead of the raw UUID (LIT-3889)", async () => { const userUuid = "c0e68be8-057e-4e2f-9d3a-000000000000"; const spendDataForUser = { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/SpendByProvider.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/SpendByProvider.test.tsx index 8da636e68a7..dca5ff91fa6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/SpendByProvider.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/SpendByProvider.test.tsx @@ -258,6 +258,34 @@ describe("SpendByProvider", () => { expect(screen.getByTestId("provider-spend-total")).toHaveTextContent("$200.50"); }); + it("always shows a subscription-covered provider and marks it included in the subscription", () => { + const providerSpendWithSubscription = [ + { + provider: "chatgpt", + spend: 0, + requests: 7, + successful_requests: 7, + failed_requests: 0, + tokens: 11608, + subscription_covered: true, + }, + { + provider: "google", + spend: 0, + requests: 0, + successful_requests: 0, + failed_requests: 0, + tokens: 0, + }, + ]; + render(); + expect(screen.getAllByText("chatgpt").length).toBeGreaterThan(0); + expect(screen.getByText("Included in subscription")).toBeInTheDocument(); + expect(screen.getByText("11,608")).toBeInTheDocument(); + expect(screen.queryByText("google")).not.toBeInTheDocument(); + expect(screen.getByText("Included in subscription").closest("td")).not.toHaveTextContent("$"); + }); + it("should include all providers with spend greater than zero by default", () => { const providerSpendWithMixed = [ { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/SpendByProvider.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/SpendByProvider.tsx index 13d9a812a12..befb7d13b55 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/SpendByProvider.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/SpendByProvider.tsx @@ -4,6 +4,7 @@ import { MoneyCell } from "@/components/shared/table_cells"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { Info, Server } from "lucide-react"; import type { ColumnDef } from "@tanstack/react-table"; +import { Badge } from "@/components/ui/badge"; import { Switch } from "@/components/ui/switch"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import React, { useId, useState } from "react"; @@ -11,13 +12,14 @@ import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; import { ChartLoader } from "@/components/shared/chart_loader"; import { Panel } from "../overview/Primitives"; -type ProviderSpendData = { +export type ProviderSpendData = { provider: string; spend: number; requests: number; successful_requests: number; failed_requests: number; tokens: number; + subscription_covered?: boolean; }; interface SpendByProviderProps { @@ -48,7 +50,15 @@ const columns: ColumnDef[] = [ header: "Spend", accessorKey: "spend", meta: { numeric: true, ...QUIET_HEADER }, - cell: ({ row }) => , + cell: ({ row }) => + row.original.subscription_covered ? ( +
+ {row.original.spend > 0 && } + Included in subscription +
+ ) : ( + + ), }, { header: "Successful", @@ -164,6 +174,10 @@ const SpendByProvider: React.FC = ({ loading, isDateChangi return includeUnknown; } + if (provider.subscription_covered) { + return true; + } + // If includeZeroSpend is true, include all providers (including those with 0 spend) // Otherwise, only include providers with spend > 0 if (includeZeroSpend) { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageAggregations.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageAggregations.ts index 54569608b0e..be9f44d0786 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageAggregations.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageAggregations.ts @@ -1,6 +1,7 @@ import { keyActivityLabel } from "@/components/UsagePage/keyActivityLabel"; import type { TopKeyItem } from "@/components/UsagePage/components/EntityUsage/TopKeyView"; import { BreakdownMetrics, DailyData, KeyMetricWithMetadata, TagUsage } from "@/components/UsagePage/types"; +import { subscriptionCoveredProviders } from "../SubscriptionCost/types"; export type ExtendedDailyData = DailyData & { breakdown: BreakdownMetrics; @@ -15,6 +16,7 @@ export interface ProviderSpendRow extends Record { successful_requests: number; failed_requests: number; tokens: number; + subscription_covered: boolean; } export const getTopModels = ( @@ -205,6 +207,7 @@ export const getTopAPIKeys = (results: ExtendedDailyData[], topKeysLimit: number }; export const getProviderSpend = (results: ExtendedDailyData[]): ProviderSpendRow[] => { + const covered = subscriptionCoveredProviders(results); const providerSpend: Record = {}; results.forEach((day) => { Object.entries(day.breakdown.providers || {}).forEach(([provider, metrics]) => { @@ -216,6 +219,7 @@ export const getProviderSpend = (results: ExtendedDailyData[]): ProviderSpendRow successful_requests: 0, failed_requests: 0, tokens: 0, + subscription_covered: covered.has(provider), }; } try { @@ -231,6 +235,6 @@ export const getProviderSpend = (results: ExtendedDailyData[]): ProviderSpendRow }); return Object.values(providerSpend) - .filter((provider) => provider.spend > 0) + .filter((provider) => provider.spend > 0 || provider.subscription_covered) .sort((a, b) => b.spend - a.spend); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCostCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCostCard.test.tsx new file mode 100644 index 00000000000..37f3bf4abf4 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCostCard.test.tsx @@ -0,0 +1,185 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import SubscriptionCostCard from "./SubscriptionCostCard"; +import type { SubscriptionAccountUsage, SubscriptionFee } from "./types"; + +vi.mock("@/components/shared/chart_loader", () => ({ + ChartLoader: () =>
Loading chart data...
, +})); + +vi.mock("@/components/molecules/models/ProviderLogo", () => ({ + ProviderLogo: ({ provider }: { provider: string }) =>
, +})); + +const fee = (overrides: Partial = {}): SubscriptionFee => ({ + subscription_account_id: "fee-1", + custom_llm_provider: "chatgpt", + account_id: "acct_123", + label: null, + monthly_fee: 25, + currency: "USD", + billing_period_start: "2026-10-01", + created_at: "2026-10-06T00:00:00Z", + updated_at: "2026-10-06T00:00:00Z", + created_by: "admin", + updated_by: "admin", + ...overrides, +}); + +const account = (overrides: Partial = {}): SubscriptionAccountUsage => ({ + custom_llm_provider: "chatgpt", + account_id: "acct_123", + deployments: ["gpt-5.6-terra", "gpt-5.6-sol"], + fee: null, + billing_periods: [], + fixed_cost: null, + ...overrides, +}); + +const handlers = () => ({ onSetFee: vi.fn(), onEditFee: vi.fn(), onRemoveFee: vi.fn() }); + +const renderCard = (accounts: SubscriptionAccountUsage[], props: { canEdit?: boolean; meteredSpend?: number } = {}) => { + const actions = handlers(); + const view = render( + , + ); + return { ...view, ...actions }; +}; + +describe("SubscriptionCostCard", () => { + it("renders nothing when there are no subscription accounts", () => { + const { container } = render( + , + ); + expect(container).toBeEmptyDOMElement(); + expect(screen.queryByText("Subscription fixed costs")).not.toBeInTheDocument(); + }); + + it("calls out an untracked fixed cost and offers to set the fee only to an editor", () => { + const { onSetFee, unmount } = renderCard([account()]); + expect(screen.getByText("Fixed cost not tracked")).toBeInTheDocument(); + expect(screen.getByText("gpt-5.6-terra, gpt-5.6-sol")).toBeInTheDocument(); + expect(screen.getByText("None in range")).toBeInTheDocument(); + expect(screen.getByTestId("fixed-cost-total")).toHaveTextContent("Fixed cost in range: none"); + fireEvent.click(screen.getByRole("button", { name: "Set monthly fee" })); + expect(onSetFee).toHaveBeenCalledWith(account()); + unmount(); + + renderCard([account()], { canEdit: false }); + expect(screen.getByText("Fixed cost not tracked")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Set monthly fee" })).not.toBeInTheDocument(); + }); + + it("shows a configured fee once for two deployments sharing the account and the fixed cost in range", () => { + const covered = account({ + fee: fee({ label: "Team workspace" }), + billing_periods: [{ start: "2026-10-01", end: "2026-10-31" }], + fixed_cost: 25, + }); + const { onEditFee, onRemoveFee } = renderCard([covered], { meteredSpend: 0.0838 }); + + expect(screen.getAllByText("25.00 USD")).toHaveLength(2); + expect(screen.getByText("Team workspace")).toBeInTheDocument(); + expect(screen.getByText("Oct 1 - Oct 31, 2026")).toBeInTheDocument(); + expect(screen.getByTestId("fixed-cost-total")).toHaveTextContent("Fixed cost in range: 25.00 USD"); + expect(screen.getByTestId("metered-spend")).toHaveTextContent("Usage-billed spend in range, all providers: $0.08"); + expect(screen.queryByText(/25\.08/)).not.toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: "Edit" })); + expect(onEditFee).toHaveBeenCalledWith(covered); + }); + + it("removes a fee only after the admin confirms it", async () => { + const covered = account({ fee: fee(), billing_periods: [], fixed_cost: 0 }); + const { onRemoveFee } = renderCard([covered]); + + fireEvent.click(screen.getByRole("button", { name: "Remove" })); + expect(onRemoveFee).not.toHaveBeenCalled(); + fireEvent.click(await screen.findByRole("button", { name: "Cancel" })); + expect(onRemoveFee).not.toHaveBeenCalled(); + + fireEvent.click(screen.getByRole("button", { name: "Remove" })); + fireEvent.click(await screen.findByRole("button", { name: "Remove fee" })); + expect(onRemoveFee).toHaveBeenCalledWith(covered); + }); + + it("sums fixed costs per currency and keeps a second currency separate", () => { + const october = { start: "2026-10-01", end: "2026-10-31" }; + const twoPeriods = { + fee: fee(), + billing_periods: [{ start: "2026-09-01", end: "2026-09-30" }, october], + fixed_cost: 50, + }; + const secondUsd = { subscription_account_id: "fee-2", account_id: "acct_456", monthly_fee: 10 }; + const noDeployments = { + account_id: "acct_456", + deployments: [], + fee: fee(secondUsd), + billing_periods: [october], + fixed_cost: 10, + }; + const euro = { subscription_account_id: "fee-3", account_id: "acct_789", monthly_fee: 20, currency: "EUR" }; + const inEuros = { account_id: "acct_789", fee: fee(euro), billing_periods: [october], fixed_cost: 20 }; + renderCard([account(twoPeriods), account(noDeployments), account(inEuros)]); + + const totals = screen.getAllByTestId("fixed-cost-total").map((node) => node.textContent); + expect(totals).toEqual(["Fixed cost in range: 60.00 USD", "Fixed cost in range: 20.00 EUR"]); + expect(screen.getByText("None")).toBeInTheDocument(); + expect(screen.getByText("Sep 1 - Sep 30, 2026")).toBeInTheDocument(); + }); + + it("explains an unresolved account and offers no action for it", () => { + renderCard([account({ account_id: null })]); + expect(screen.getByText("Account not signed in on the proxy")).toBeInTheDocument(); + expect(screen.getByText("Fixed cost not tracked")).toBeInTheDocument(); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("shows the loader instead of the table while a range is loading", () => { + render( + , + ); + expect(screen.getByTestId("chart-loader")).toBeInTheDocument(); + expect(screen.queryByText("25.00 USD")).not.toBeInTheDocument(); + }); + + it("says so instead of disappearing when the usage request fails", () => { + render( + , + ); + expect(screen.getByText("Could not load subscription accounts for this range.")).toBeInTheDocument(); + expect(screen.queryByTestId("chart-loader")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCostCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCostCard.tsx new file mode 100644 index 00000000000..deee30a2db5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCostCard.tsx @@ -0,0 +1,221 @@ +import type { ColumnDef } from "@tanstack/react-table"; +import { Info } from "lucide-react"; +import React from "react"; +import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; +import { ChartLoader } from "@/components/shared/chart_loader"; +import { DataTable } from "@/components/shared/DataTable"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "@/components/ui/alert-dialog"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { ATTRIBUTION_RULE, fixedCostByCurrency, formatBillingPeriod, formatFee } from "./formatting"; +import type { SubscriptionAccountUsage, SubscriptionUsageResponse } from "./types"; + +interface SubscriptionCostCardProps { + loading: boolean; + failed: boolean; + isDateChanging: boolean; + usage: SubscriptionUsageResponse | null; + meteredSpend: number; + canEdit: boolean; + onSetFee: (account: SubscriptionAccountUsage) => void; + onEditFee: (account: SubscriptionAccountUsage) => void; + onRemoveFee: (account: SubscriptionAccountUsage) => void; +} + +type FeeActions = Pick; + +const accountRowId = (account: SubscriptionAccountUsage): string => + `${account.custom_llm_provider}:${account.account_id ?? "unresolved"}`; + +const AccountCell = ({ account }: { account: SubscriptionAccountUsage }) => { + if (account.account_id === null) { + return Account not signed in on the proxy; + } + return ( +
+ {account.account_id} + {account.fee?.label && {account.fee.label}} +
+ ); +}; + +const ActionsCell = ({ + account, + onSetFee, + onEditFee, + onRemoveFee, +}: { account: SubscriptionAccountUsage } & FeeActions) => { + if (account.account_id === null) return null; + if (account.fee === null) { + return ( + + ); + } + return ( +
+ + + }>Remove + + + Remove this monthly fee? + + {account.account_id} goes back to "Fixed cost not tracked". Its requests stay included in the + subscription + + + + Cancel + onRemoveFee(account)}> + Remove fee + + + + +
+ ); +}; + +const baseColumns: ColumnDef[] = [ + { + header: "Provider", + accessorKey: "custom_llm_provider", + cell: ({ row }) => ( +
+ + {row.original.custom_llm_provider} +
+ ), + }, + { + header: "Account", + accessorKey: "account_id", + cell: ({ row }) => , + }, + { + header: "Deployments", + id: "deployments", + cell: ({ row }) => (row.original.deployments.length === 0 ? "None" : row.original.deployments.join(", ")), + }, + { + header: "Monthly fee", + id: "monthly_fee", + meta: { numeric: true }, + cell: ({ row }) => + row.original.fee === null ? ( + Fixed cost not tracked + ) : ( + formatFee(row.original.fee.monthly_fee, row.original.fee.currency) + ), + }, + { + header: "Billing periods in range", + id: "billing_periods", + cell: ({ row }) => + row.original.billing_periods.length === 0 ? ( + None in range + ) : ( +
+ {row.original.billing_periods.map((period) => ( + {formatBillingPeriod(period)} + ))} +
+ ), + }, + { + header: "Fixed cost in range", + id: "fixed_cost", + meta: { numeric: true }, + cell: ({ row }) => + row.original.fee === null || row.original.fixed_cost === null + ? "-" + : formatFee(row.original.fixed_cost, row.original.fee.currency), + }, +]; + +const actionsColumn = (actions: FeeActions): ColumnDef => ({ + header: "", + id: "actions", + meta: { numeric: true }, + cell: ({ row }) => , +}); + +const SubscriptionCostCard: React.FC = ({ + loading, + failed, + isDateChanging, + usage, + meteredSpend, + canEdit, + onSetFee, + onEditFee, + onRemoveFee, +}) => { + if (!failed && (usage === null || usage.accounts.length === 0)) return null; + const columns = canEdit ? [...baseColumns, actionsColumn({ onSetFee, onEditFee, onRemoveFee })] : baseColumns; + const accounts = usage?.accounts ?? []; + const totals = fixedCostByCurrency(accounts); + + return ( + + + + Subscription fixed costs + + } /> + {ATTRIBUTION_RULE} + + + + + {failed &&

Could not load subscription accounts for this range.

} + {!failed && loading && } + {!failed && !loading && ( +
+ +
+ {totals.length === 0 ? ( + Fixed cost in range: none + ) : ( + totals.map(({ currency, total }) => ( + + Fixed cost in range: {formatFee(total, currency)} + + )) + )} + + Usage-billed spend in range, all providers: ${formatNumberWithCommas(meteredSpend, 2)} + +
+

{ATTRIBUTION_RULE}

+
+ )} + + + ); +}; + +export default SubscriptionCostCard; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCosts.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCosts.integration.test.tsx new file mode 100644 index 00000000000..f057e98dd70 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCosts.integration.test.tsx @@ -0,0 +1,69 @@ +import { fireEvent, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { subscriptionAccountCreateCall, subscriptionUsageCall } from "@/components/networking"; +import SubscriptionCosts from "./SubscriptionCosts"; +import type { SubscriptionUsageResponse } from "./types"; + +vi.mock("@/components/networking", () => ({ + formatDate: (date: Date) => date.toISOString().slice(0, 10), + subscriptionUsageCall: vi.fn(), + subscriptionAccountCreateCall: vi.fn().mockResolvedValue({}), + subscriptionAccountUpdateCall: vi.fn().mockResolvedValue({}), + subscriptionAccountDeleteCall: vi.fn().mockResolvedValue({}), +})); +vi.mock("@/lib/toast", () => ({ toast: { success: vi.fn(), fromError: vi.fn() } })); + +const untracked: SubscriptionUsageResponse = { + start_date: "2026-10-01", + end_date: "2026-10-07", + subscription_providers: ["chatgpt"], + accounts: [ + { + custom_llm_provider: "chatgpt", + account_id: "acct_123", + deployments: ["gpt-5.6-sol", "gpt-5.6-terra"], + fee: null, + billing_periods: [], + fixed_cost: null, + }, + ], +}; + +const fill = (label: string, value: string) => fireEvent.change(screen.getByLabelText(label), { target: { value } }); + +describe("SubscriptionCosts", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.clearAllMocks(); + vi.mocked(subscriptionUsageCall).mockResolvedValue(untracked); + }); + + it("saves a fee set from the card on that card's account and reloads the card", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + + await user.click(await screen.findByRole("button", { name: "Set monthly fee" })); + expect(screen.getByText("chatgpt account acct_123")).toBeInTheDocument(); + fill("Monthly fee", "25"); + fill("Currency", "USD"); + fill("Billing period start", "2026-10-01"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => expect(subscriptionUsageCall).toHaveBeenCalledTimes(2)); + expect(subscriptionAccountCreateCall).toHaveBeenCalledWith( + "sk-admin", + expect.objectContaining({ custom_llm_provider: "chatgpt", account_id: "acct_123", monthly_fee: 25 }), + ); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCosts.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCosts.tsx new file mode 100644 index 00000000000..e326fa1ff01 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionCosts.tsx @@ -0,0 +1,38 @@ +import React from "react"; +import { useSubscriptionUsage, type SubscriptionUsageOptions } from "../../hooks/useSubscriptionUsage"; +import SubscriptionCostCard from "./SubscriptionCostCard"; +import SubscriptionFeeModal from "./SubscriptionFeeModal"; +import { useSubscriptionFeeActions } from "./useSubscriptionFeeActions"; + +interface SubscriptionCostsProps extends SubscriptionUsageOptions { + meteredSpend: number; + canEdit: boolean; +} + +const SubscriptionCosts: React.FC = ({ meteredSpend, canEdit, ...usageOptions }) => { + const usage = useSubscriptionUsage(usageOptions); + const feeActions = useSubscriptionFeeActions({ accessToken: usageOptions.accessToken, refetch: usage.refetch }); + return ( + <> + + + + ); +}; + +export default SubscriptionCosts; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionFeeModal.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionFeeModal.test.tsx new file mode 100644 index 00000000000..a920f6ec883 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionFeeModal.test.tsx @@ -0,0 +1,83 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import SubscriptionFeeModal from "./SubscriptionFeeModal"; +import type { SubscriptionAccountUsage } from "./types"; + +const account: SubscriptionAccountUsage = { + custom_llm_provider: "chatgpt", + account_id: "acct_123", + deployments: ["gpt-5.6-terra"], + fee: null, + billing_periods: [], + fixed_cost: null, +}; + +const fill = (label: string, value: string) => fireEvent.change(screen.getByLabelText(label), { target: { value } }); + +describe("SubscriptionFeeModal", () => { + it("submits normalized values for a new fee", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + render(); + + expect(screen.getByText("Set monthly fee")).toBeInTheDocument(); + expect(screen.getByText("chatgpt account acct_123")).toBeInTheDocument(); + fill("Monthly fee", "25"); + fill("Currency", "usd"); + fill("Billing period start", "2026-10-01"); + fill("Label (optional)", "Team workspace"); + await userEvent.setup().click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => expect(onSubmit).toHaveBeenCalledTimes(1)); + const normalized = { + monthly_fee: 25, + currency: "USD", + billing_period_start: "2026-10-01", + label: "Team workspace", + }; + expect(onSubmit).toHaveBeenCalledWith(normalized); + }); + + it("rejects a zero fee and a two-letter currency", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + const user = userEvent.setup(); + render(); + + fill("Monthly fee", "0"); + fill("Billing period start", "2026-10-01"); + await user.click(screen.getByRole("button", { name: "Save" })); + expect(await screen.findByText("Monthly fee must be greater than 0")).toBeInTheDocument(); + + fill("Monthly fee", "25"); + fill("Currency", "US"); + await user.click(screen.getByRole("button", { name: "Save" })); + expect(await screen.findByText("Currency must be a three-letter code")).toBeInTheDocument(); + expect(onSubmit).not.toHaveBeenCalled(); + }); + + it("prefills the existing fee when editing", () => { + const existing: SubscriptionAccountUsage = { + ...account, + fee: { + subscription_account_id: "fee-1", + custom_llm_provider: "chatgpt", + account_id: "acct_123", + label: "Team workspace", + monthly_fee: 25, + currency: "EUR", + billing_period_start: "2026-09-15", + created_at: "2026-10-06T00:00:00Z", + updated_at: "2026-10-06T00:00:00Z", + created_by: "admin", + updated_by: "admin", + }, + }; + render(); + + expect(screen.getByText("Edit monthly fee")).toBeInTheDocument(); + expect(screen.getByLabelText("Monthly fee")).toHaveValue(25); + expect(screen.getByLabelText("Currency")).toHaveValue("EUR"); + expect(screen.getByLabelText("Billing period start")).toHaveValue("2026-09-15"); + expect(screen.getByLabelText("Label (optional)")).toHaveValue("Team workspace"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionFeeModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionFeeModal.tsx new file mode 100644 index 00000000000..1c211a5280c --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/SubscriptionFeeModal.tsx @@ -0,0 +1,114 @@ +"use client"; + +import React, { useEffect } from "react"; +import { z } from "zod"; +import { formatDate } from "@/components/networking"; +import { FormField } from "@/components/shared/form/FormField"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { FieldGroup } from "@/components/ui/field"; +import { Input } from "@/components/ui/input"; +import { useZodForm } from "@/lib/forms/useZodForm"; +import type { SubscriptionAccountUsage, SubscriptionFeeFormValues } from "./types"; + +const feeShape = { + monthly_fee: z + .string() + .min(1, "Enter a monthly fee") + .refine((value) => Number(value) > 0, "Monthly fee must be greater than 0"), + currency: z + .string() + .trim() + .toUpperCase() + .regex(/^[A-Z]{3}$/, "Currency must be a three-letter code"), + billing_period_start: z.string().regex(/^\d{4}-\d{2}-\d{2}$/, "Pick a billing period start date"), + label: z.string().trim(), +}; + +const feeSchema = z.object(feeShape); + +type FeeFormInput = z.input; + +const formInputFor = (account: SubscriptionAccountUsage | null): FeeFormInput => ({ + monthly_fee: account?.fee ? String(account.fee.monthly_fee) : "", + currency: account?.fee?.currency ?? "USD", + billing_period_start: account?.fee?.billing_period_start ?? formatDate(new Date()), + label: account?.fee?.label ?? "", +}); + +interface SubscriptionFeeModalProps { + visible: boolean; + account: SubscriptionAccountUsage | null; + onCancel: () => void; + onSubmit: (values: SubscriptionFeeFormValues) => Promise; +} + +const SubscriptionFeeModal: React.FC = ({ visible, account, onCancel, onSubmit }) => { + const form = useZodForm(feeSchema, { defaultValues: formInputFor(account) }); + const { reset } = form; + + useEffect(() => { + if (visible) reset(formInputFor(account)); + }, [visible, account, reset]); + + const handleFinish = async (values: z.output) => { + const submitted: SubscriptionFeeFormValues = { + monthly_fee: Number(values.monthly_fee), + currency: values.currency, + billing_period_start: values.billing_period_start, + label: values.label === "" ? null : values.label, + }; + await onSubmit(submitted); + }; + + const handleCancel = () => { + reset(formInputFor(null)); + onCancel(); + }; + + return ( + !open && handleCancel()}> + + + {account?.fee ? "Edit monthly fee" : "Set monthly fee"} + + {account ? `${account.custom_llm_provider} account ${account.account_id ?? ""}` : ""} + + +
+ + + {({ ref, ...field }) => } + + + {({ ref, ...field }) => } + + + {({ ref, ...field }) => } + + + {({ ref, ...field }) => } + + + + + + +
+ + + ); +}; + +export default SubscriptionFeeModal; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/formatting.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/formatting.ts new file mode 100644 index 00000000000..b1d4e0a48ef --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/formatting.ts @@ -0,0 +1,34 @@ +import type { BillingPeriod } from "./types"; + +export const ATTRIBUTION_RULE = + "A subscription fee is counted once for each monthly billing period whose start date falls inside the selected dates. Fixed costs sit beside metered spend and are never added to it, to key or team spend, or to budgets."; + +export const formatFee = (amount: number, currency: string): string => `${amount.toFixed(2)} ${currency}`; + +const parseLocalDay = (day: string): Date => { + const [year, month, date] = day.split("-").map(Number); + return new Date(year, month - 1, date); +}; + +export const formatBillingPeriod = ({ start, end }: BillingPeriod): string => { + const startDate = parseLocalDay(start); + const endDate = parseLocalDay(end); + const startLabel = startDate.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: startDate.getFullYear() !== endDate.getFullYear() ? "numeric" : undefined, + }); + const endLabel = endDate.toLocaleDateString("en-US", { month: "short", day: "numeric", year: "numeric" }); + return `${startLabel} - ${endLabel}`; +}; + +export const fixedCostByCurrency = ( + accounts: readonly { fee: { currency: string } | null; fixed_cost: number | null }[], +): readonly { currency: string; total: number }[] => { + const totals = new Map(); + accounts.forEach((account) => { + if (account.fee === null || account.fixed_cost === null) return; + totals.set(account.fee.currency, (totals.get(account.fee.currency) ?? 0) + account.fixed_cost); + }); + return Array.from(totals, ([currency, total]) => ({ currency, total })); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/types.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/types.ts new file mode 100644 index 00000000000..6af5e0cdf3b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/types.ts @@ -0,0 +1,74 @@ +import type { DailyData } from "@/components/UsagePage/types"; + +export interface SubscriptionFee { + subscription_account_id: string; + custom_llm_provider: string; + account_id: string; + label: string | null; + monthly_fee: number; + currency: string; + billing_period_start: string; + created_at: string; + updated_at: string; + created_by: string; + updated_by: string; +} + +export interface BillingPeriod { + start: string; + end: string; +} + +export interface SubscriptionAccountUsage { + custom_llm_provider: string; + account_id: string | null; + deployments: string[]; + fee: SubscriptionFee | null; + billing_periods: BillingPeriod[]; + fixed_cost: number | null; +} + +export interface SubscriptionUsageResponse { + subscription_providers: string[]; + accounts: SubscriptionAccountUsage[]; +} + +export interface SubscriptionAccountCreateRequest { + custom_llm_provider: string; + account_id: string; + monthly_fee: number; + currency: string; + billing_period_start: string; + label: string | null; +} + +export interface SubscriptionAccountUpdateRequest { + subscription_account_id: string; + monthly_fee?: number; + currency?: string; + billing_period_start?: string; + label?: string | null; +} + +export interface SubscriptionAccountDeleteRequest { + subscription_account_id: string; +} + +export interface SubscriptionFeeFormValues { + monthly_fee: number; + currency: string; + billing_period_start: string; + label: string | null; +} + +export const isSubscriptionCovered = (metadata: object | undefined): boolean => + metadata !== undefined && "subscription_covered" in metadata && metadata.subscription_covered === true; + +export const subscriptionCoveredProviders = (results: readonly DailyData[]): ReadonlySet => + new Set( + results.flatMap((day) => + Object.entries(day.breakdown.providers ?? {}) + .filter(([, entry]) => isSubscriptionCovered(entry.metadata)) + .map(([provider]) => provider), + ), + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/useSubscriptionFeeActions.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/useSubscriptionFeeActions.test.tsx new file mode 100644 index 00000000000..9406e07b497 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/useSubscriptionFeeActions.test.tsx @@ -0,0 +1,79 @@ +import { act, renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { subscriptionAccountCreateCall, subscriptionAccountUpdateCall } from "@/components/networking"; +import type { SubscriptionAccountUsage, SubscriptionFeeFormValues } from "./types"; +import { useSubscriptionFeeActions } from "./useSubscriptionFeeActions"; + +vi.mock("@/components/networking", () => ({ + subscriptionAccountCreateCall: vi.fn().mockResolvedValue({}), + subscriptionAccountUpdateCall: vi.fn().mockResolvedValue({}), + subscriptionAccountDeleteCall: vi.fn(), +})); +vi.mock("@/lib/toast", () => ({ toast: { success: vi.fn(), fromError: vi.fn() } })); + +const stored: SubscriptionFeeFormValues = { + monthly_fee: 25, + currency: "USD", + billing_period_start: "2026-10-01", + label: "Team plan (test value)", +}; + +const tracked: SubscriptionAccountUsage = { + custom_llm_provider: "chatgpt", + account_id: "acct_123", + deployments: ["gpt-5.6-terra"], + fee: { + subscription_account_id: "sub-1", + custom_llm_provider: "chatgpt", + account_id: "acct_123", + ...stored, + created_at: "2026-10-01T00:00:00Z", + updated_at: "2026-10-01T00:00:00Z", + created_by: "admin", + updated_by: "admin", + }, + billing_periods: [{ start: "2026-10-01", end: "2026-10-31" }], + fixed_cost: 25, +}; + +const submit = async (account: SubscriptionAccountUsage, values: SubscriptionFeeFormValues) => { + const refetch = vi.fn().mockResolvedValue(undefined); + const { result } = renderHook(() => useSubscriptionFeeActions({ accessToken: "sk-admin", refetch })); + act(() => result.current.openFeeModal(account)); + await act(() => result.current.submitFee(values)); + return { refetch, result }; +}; + +describe("useSubscriptionFeeActions", () => { + beforeEach(() => vi.clearAllMocks()); + + it("sends only the fields the admin changed when editing a fee", async () => { + const { refetch, result } = await submit(tracked, { ...stored, monthly_fee: 30 }); + + expect(subscriptionAccountUpdateCall).toHaveBeenCalledWith("sk-admin", { + subscription_account_id: "sub-1", + monthly_fee: 30, + }); + expect(refetch).toHaveBeenCalledTimes(1); + expect(result.current.feeModalAccount).toBeNull(); + }); + + it("clears a label with an explicit null and nothing else", async () => { + await submit(tracked, { ...stored, label: null }); + + expect(subscriptionAccountUpdateCall).toHaveBeenCalledWith("sk-admin", { + subscription_account_id: "sub-1", + label: null, + }); + }); + + it("creates a fee with every field for an untracked account", async () => { + const untracked: SubscriptionAccountUsage = { ...tracked, fee: null, billing_periods: [], fixed_cost: null }; + const createRequest = { custom_llm_provider: "chatgpt", account_id: "acct_123", ...stored }; + + await submit(untracked, stored); + + expect(subscriptionAccountCreateCall).toHaveBeenCalledWith("sk-admin", createRequest); + expect(subscriptionAccountUpdateCall).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/useSubscriptionFeeActions.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/useSubscriptionFeeActions.ts new file mode 100644 index 00000000000..dd2b2f01a30 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/SubscriptionCost/useSubscriptionFeeActions.ts @@ -0,0 +1,90 @@ +import { useCallback, useState } from "react"; +import { + subscriptionAccountCreateCall, + subscriptionAccountDeleteCall, + subscriptionAccountUpdateCall, +} from "@/components/networking"; +import { toast } from "@/lib/toast"; +import type { + SubscriptionAccountUpdateRequest, + SubscriptionAccountUsage, + SubscriptionFee, + SubscriptionFeeFormValues, +} from "./types"; + +interface Options { + accessToken: string | null; + refetch: () => Promise; +} + +interface Result { + feeModalAccount: SubscriptionAccountUsage | null; + openFeeModal: (account: SubscriptionAccountUsage) => void; + closeFeeModal: () => void; + submitFee: (values: SubscriptionFeeFormValues) => Promise; + removeFee: (account: SubscriptionAccountUsage) => Promise; +} + +const changedFields = (fee: SubscriptionFee, values: SubscriptionFeeFormValues): SubscriptionAccountUpdateRequest => ({ + subscription_account_id: fee.subscription_account_id, + ...(values.monthly_fee !== fee.monthly_fee && { monthly_fee: values.monthly_fee }), + ...(values.currency !== fee.currency && { currency: values.currency }), + ...(values.billing_period_start !== fee.billing_period_start && { + billing_period_start: values.billing_period_start, + }), + ...(values.label !== fee.label && { label: values.label }), +}); + +const saveFee = ( + accessToken: string, + account: SubscriptionAccountUsage, + values: SubscriptionFeeFormValues, +): Promise | null => { + if (account.fee) { + return subscriptionAccountUpdateCall(accessToken, changedFields(account.fee, values)); + } + if (account.account_id === null) return null; + return subscriptionAccountCreateCall(accessToken, { + custom_llm_provider: account.custom_llm_provider, + account_id: account.account_id, + ...values, + }); +}; + +export function useSubscriptionFeeActions({ accessToken, refetch }: Options): Result { + const [feeModalAccount, setFeeModalAccount] = useState(null); + const closeFeeModal = useCallback(() => setFeeModalAccount(null), []); + + const submitFee = useCallback( + async (values: SubscriptionFeeFormValues) => { + if (!accessToken || feeModalAccount === null) return; + const request = saveFee(accessToken, feeModalAccount, values); + if (request === null) return; + try { + await request; + toast.success("Monthly fee saved"); + setFeeModalAccount(null); + await refetch(); + } catch (error) { + toast.fromError(error); + } + }, + [accessToken, feeModalAccount, refetch], + ); + + const removeFee = useCallback( + async (account: SubscriptionAccountUsage) => { + if (!accessToken || account.fee === null) return; + try { + await subscriptionAccountDeleteCall(accessToken, account.fee.subscription_account_id); + toast.success("Monthly fee removed"); + await refetch(); + } catch (error) { + toast.fromError(error); + } + }, + [accessToken, refetch], + ); + + return { feeModalAccount, openFeeModal: setFeeModalAccount, closeFeeModal, submitFee, removeFee }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index f46a4569369..9d39e9c080c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -8,7 +8,7 @@ import type { ComponentProps } from "react"; import userEvent from "@testing-library/user-event"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { UrlUpdateEvent } from "nuqs/adapters/testing"; -import { chooseSelectOption, renderWithProviders } from "@/../tests/test-utils"; +import { chooseSelectOption, renderWithProviders, testQueryClient } from "@/../tests/test-utils"; import { toast } from "@/lib/toast"; import { processActivityData } from "@/components/activity_metrics"; import type { Organization } from "@/components/networking"; @@ -74,6 +74,11 @@ beforeAll(() => { // Mock the networking module vi.mock("@/components/networking", () => ({ + formatDate: (date: Date) => date.toISOString().slice(0, 10), + subscriptionUsageCall: vi.fn(), + subscriptionAccountCreateCall: vi.fn(), + subscriptionAccountUpdateCall: vi.fn(), + subscriptionAccountDeleteCall: vi.fn(), dailyActivityAggregatedCall: vi.fn(), dailyActivityKeyPageCall: vi.fn(), userDailyActivityUserPageCall: vi.fn(), @@ -142,6 +147,14 @@ vi.mock("./EntityUsage/SpendByProvider", () => ({ default: () =>
Spend By Provider
, })); +vi.mock("./SubscriptionCost/SubscriptionCostCard", () => ({ + default: ({ usage, canEdit }: { usage: unknown; canEdit: boolean }) => ( +
+ Subscription Cost +
+ ), +})); + vi.mock("./EndpointUsage/EndpointUsage", () => ({ default: () =>
Endpoint Usage
, })); @@ -278,6 +291,7 @@ describe("UsagePage", () => { const mockTagListCall = vi.mocked(networking.tagListCall); const mockGatewayDailyActivityCall = vi.mocked(networking.gatewayDailyActivityCall); const mockRequestErrorActivityCall = vi.mocked(networking.requestErrorActivityCall); + const mockSubscriptionUsageCall = vi.mocked(networking.subscriptionUsageCall); const mockUseAgents = vi.mocked(useAgents); const mockUseAuthorized = vi.mocked(useAuthorized); const mockUseCurrentUser = vi.mocked(useCurrentUser); @@ -496,7 +510,24 @@ describe("UsagePage", () => { organizations: [], }; + const mockSubscriptionUsage = { + subscription_providers: ["chatgpt"], + accounts: [ + { + custom_llm_provider: "chatgpt", + account_id: "acct_123", + deployments: ["gpt-5.6-terra", "gpt-5.6-sol"], + fee: null, + billing_periods: [], + fixed_cost: null, + }, + ], + }; + beforeEach(() => { + testQueryClient.clear(); + mockSubscriptionUsageCall.mockReset(); + mockSubscriptionUsageCall.mockResolvedValue(mockSubscriptionUsage); mockClientUsagePanel.mockClear(); vi.mocked(processActivityData).mockReset(); vi.mocked(processActivityData).mockImplementation((_data, key) => ({ @@ -578,6 +609,72 @@ describe("UsagePage", () => { } as any); }); + it("hands the subscription usage to the fixed cost card for an admin and never fetches it otherwise", async () => { + const { unmount } = renderWithProviders(); + await waitFor(() => { + expect(mockSubscriptionUsageCall).toHaveBeenCalledWith("test-token", expect.any(Date), expect.any(Date)); + }); + await waitFor(() => { + expect(screen.getByTestId("subscription-cost-card")).toHaveAttribute( + "data-usage", + JSON.stringify(mockSubscriptionUsage), + ); + }); + unmount(); + + for (const session of [nonAdminSession, { ...nonAdminSession, userRole: "org_admin" }]) { + mockSubscriptionUsageCall.mockClear(); + mockUserDailyActivityAggregatedCall.mockClear(); + mockUseAuthorized.mockReturnValue(session); + const rendered = renderWithProviders(); + await waitFor(() => { + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalled(); + }); + expect(mockSubscriptionUsageCall).not.toHaveBeenCalled(); + expect(screen.getByTestId("subscription-cost-card")).toHaveAttribute("data-usage", "null"); + rendered.unmount(); + } + }); + + it("lets a proxy admin edit subscription fees but shows an admin viewer the same card read-only", async () => { + const adminRender = renderWithProviders(); + await waitFor(() => { + expect(screen.getByTestId("subscription-cost-card")).toHaveAttribute( + "data-usage", + JSON.stringify(mockSubscriptionUsage), + ); + }); + expect(screen.getByTestId("subscription-cost-card")).toHaveAttribute("data-can-edit", "true"); + adminRender.unmount(); + + const adminViewerSession = { + ...nonAdminSession, + userRole: "Admin", + userRoleLabel: "Admin Viewer", + isViewOnly: true, + }; + mockUseAuthorized.mockReturnValue(adminViewerSession); + renderWithProviders(); + await waitFor(() => { + expect(screen.getByTestId("subscription-cost-card")).toHaveAttribute( + "data-usage", + JSON.stringify(mockSubscriptionUsage), + ); + }); + expect(screen.getByTestId("subscription-cost-card")).toHaveAttribute("data-can-edit", "false"); + }); + + it("never sets an account-wide subscription fee beside one user's spend when an admin filters by user", async () => { + renderWithProviders(, { searchParams: "?user=user-001" }); + await waitFor(() => { + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledWith( + expect.objectContaining({ entityIds: ["user-001"] }), + ); + }); + expect(mockSubscriptionUsageCall).not.toHaveBeenCalled(); + expect(screen.getByTestId("subscription-cost-card")).toHaveAttribute("data-usage", "null"); + }); + it("should render and fetch usage data on mount", async () => { renderWithProviders(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index d5ec3ef7675..7c891754195 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -26,7 +26,7 @@ import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrgan import { toast } from "@/lib/toast"; import { hasCapability } from "@/utils/capabilities"; import { copyToClipboard } from "@/utils/dataUtils"; -import { hasProxyWideSpendView, internalUserRoles } from "@/utils/roles"; +import { hasProxyWideSpendView, internalUserRoles, isProxyAdminRole } from "@/utils/roles"; import { ActivityMetrics, processActivityData } from "@/components/activity_metrics"; import CloudZeroExportModal from "@/components/cloudzero_export_modal"; import UserDropdown from "@/components/common_components/UserDropdown"; @@ -92,7 +92,7 @@ import { usageViewFromParams, usageViewPatch, } from "../usageUrlState"; -import { overviewTotals, rollUpBreakdown } from "./overview/overviewData"; +import { overviewTotals, providerSpendRows } from "./overview/overviewData"; import TopAgents from "./overview/TopAgents"; import { parseUsageClient, topAgents, type AgentRow } from "./overview/agentCatalog"; import ClientUsagePanel from "./ClientUsagePanel"; @@ -100,6 +100,7 @@ import { resolveClientUsageScope } from "../hooks/clientUsageScope"; import { useUsagePanelContext } from "../hooks/useUsagePanelContext"; import ClientUsageSelect from "./ClientUsageSelect"; import SpendByProvider from "./EntityUsage/SpendByProvider"; +import SubscriptionCosts from "./SubscriptionCost/SubscriptionCosts"; import { useTagSummary } from "@/app/(dashboard)/hooks/tags/useTagSummary"; import { Panel } from "./overview/Primitives"; import UsageOverview from "./overview/UsageOverview"; @@ -112,7 +113,7 @@ interface UsagePageProps { } const UsagePage: React.FC = ({ teams, organizations, teamsLoading = false, defaultTimeRange }) => { - const { accessToken, userRole, userId: userID, premiumUser, isLoading: authorizationLoading } = useAuthorized(); + const { accessToken, userRole, userId: userID, premiumUser, isViewOnly, isLoading: authLoading } = useAuthorized(); const [gatewayActivityData, setGatewayActivityData] = useState(null); const defaultRange = useMemo(() => defaultUsageDateRange(defaultTimeRange), [defaultTimeRange]); @@ -187,7 +188,7 @@ const UsagePage: React.FC = ({ teams, organizations, teamsLoadin }; const { isLoading: organizationsLoading } = useOrganizations(); - const authorizationSettled = !authorizationLoading && userRole !== null; + const authorizationSettled = !authLoading && userRole !== null; const accessSettled = authorizationSettled && !organizationsLoading && !teamsLoading; useEffect(() => { if (!accessSettled) return; @@ -374,10 +375,7 @@ const UsagePage: React.FC = ({ teams, organizations, teamsLoadin () => overviewTotals(userSpendData.metadata, gatewayActivity), [userSpendData.metadata, gatewayActivity], ); - const providerSpend = useMemo( - () => rollUpBreakdown(userSpendData.results, "providers").map(({ key, ...row }) => ({ provider: key, ...row })), - [userSpendData.results], - ); + const providerSpend = useMemo(() => providerSpendRows(userSpendData.results), [userSpendData.results]); const { data: tagSummary, isLoading: tagSummaryLoading, @@ -546,6 +544,16 @@ const UsagePage: React.FC = ({ teams, organizations, teamsLoadin loading={loading} requestCountsPending={requestCountsPending} budget={currentUser?.max_budget ?? null} + subscriptionCosts={ + + } topKeys={ + {subscriptionCosts} + {topAgents} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts index f00d53ea1bc..1dfdfd1da78 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts @@ -7,6 +7,7 @@ import { labelForDate, modelColorMap, OTHER_COLOR, + providerSpendRows, rollUpBreakdown, seriesBy, } from "./overviewData"; @@ -88,6 +89,34 @@ describe("bucketSeries", () => { }); }); +const providerDay = (date: string, providers: Record>): DailyData => + ({ + ...day(date, 0, {}), + breakdown: { + ...day(date, 0, {}).breakdown, + providers: Object.fromEntries( + Object.entries(providers).map(([name, metadata]) => [ + name, + { metrics: metrics(0), metadata, api_key_breakdown: {} }, + ]), + ), + }, + }) as unknown as DailyData; + +describe("providerSpendRows", () => { + it("marks a provider covered by a subscription on any day of the range and leaves metered ones unmarked", () => { + const rows = providerSpendRows([ + providerDay("2026-10-01", { chatgpt: {}, openai: {} }), + providerDay("2026-10-02", { chatgpt: { subscription_covered: true }, openai: { subscription_covered: false } }), + ]); + expect(Object.fromEntries(rows.map((row) => [row.provider, row.subscription_covered]))).toEqual({ + chatgpt: true, + openai: false, + }); + expect(rows.find((row) => row.provider === "chatgpt")).toEqual(expect.objectContaining({ requests: 2 })); + }); +}); + describe("rollUpBreakdown", () => { it("sums a dimension across days without changing the input", () => { const days = [day("2026-10-01", 3, { a: 3 }), day("2026-10-02", 2, { a: 2 })]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts index c103c2d7c60..9d9e8f9fa32 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts @@ -2,6 +2,7 @@ import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; import { stackedUsageColor } from "@/components/shared/charts"; import { EMPTY_DAILY_ACTIVITY_METADATA, type DailyActivityMetadata } from "@/components/UsagePage/dailyActivityApi"; import type { GatewayActivity } from "../gatewayActivity"; +import { subscriptionCoveredProviders } from "../SubscriptionCost/types"; export type BreakdownDimension = "models" | "model_groups" | "providers"; @@ -40,6 +41,15 @@ export const rollUpBreakdown = (results: readonly DailyData[], dimension: Breakd return [...totals.values()].sort((a, b) => b.spend - a.spend); }; +export const providerSpendRows = (results: readonly DailyData[]) => { + const covered = subscriptionCoveredProviders(results); + return rollUpBreakdown(results, "providers").map(({ key, ...row }) => ({ + provider: key, + ...row, + subscription_covered: covered.has(key), + })); +}; + export const modelColorMap = (results: readonly DailyData[]): ReadonlyMap => { const groups = rollUpBreakdown(results, "model_groups"); const models = rollUpBreakdown(results, "models"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useSubscriptionUsage.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useSubscriptionUsage.ts new file mode 100644 index 00000000000..afda5ec57b8 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useSubscriptionUsage.ts @@ -0,0 +1,47 @@ +import { keepPreviousData, queryOptions, skipToken, useQuery } from "@tanstack/react-query"; +import { formatDate, subscriptionUsageCall } from "@/components/networking"; +import type { SubscriptionUsageResponse } from "../components/SubscriptionCost/types"; + +export interface SubscriptionUsageOptions { + accessToken: string | null; + startTime: Date | null; + endTime: Date | null; + enabled: boolean; +} + +interface Result { + data: SubscriptionUsageResponse | null; + loading: boolean; + failed: boolean; + refetch: () => Promise; +} + +interface UsageRequest { + accessToken: string; + startTime: Date; + endTime: Date; +} + +const subscriptionUsageQuery = (request: UsageRequest | null) => { + const queryKey = [ + "subscriptionUsage", + request ? formatDate(request.startTime) : null, + request ? formatDate(request.endTime) : null, + ]; + const queryFn = request + ? () => subscriptionUsageCall(request.accessToken, request.startTime, request.endTime) + : skipToken; + return queryOptions({ queryKey, queryFn, placeholderData: keepPreviousData }); +}; + +export function useSubscriptionUsage({ accessToken, startTime, endTime, enabled }: SubscriptionUsageOptions): Result { + const range = startTime && endTime ? { startTime, endTime } : null; + const request = enabled && accessToken && range ? { accessToken, ...range } : null; + const query = useQuery(subscriptionUsageQuery(request)); + return { + data: request && query.data ? query.data : null, + loading: request !== null && (query.isPending || query.isPlaceholderData), + failed: query.isError, + refetch: query.refetch, + }; +} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 99a9dc7a707..c13052be245 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -119,6 +119,13 @@ import { MCP_TOOLS_PREVIEW_FORBIDDEN_MESSAGE } from "./mcp_tools/constants"; import type { ComplexityRouterConfigPayload } from "./add_model/build_complexity_router_config"; import type { AutoRouterPresetsResponse } from "@/lib/autorouter_presets"; import type { VectorStoreIndex } from "@/app/(dashboard)/vector-stores/_components/IndexesTab"; +import type { + SubscriptionAccountCreateRequest, + SubscriptionAccountDeleteRequest, + SubscriptionAccountUpdateRequest, + SubscriptionFee, + SubscriptionUsageResponse, +} from "@/app/(dashboard)/usage/_components/components/SubscriptionCost/types"; import type { RoutingDecision } from "./logs/detail/RoutingDecisionCard"; import type { SpanDetail, @@ -1395,6 +1402,34 @@ export const dailyActivityAggregatedCall = ( query: dailyActivityQuery(entity, req, req.apiKeyLimit === undefined ? {} : { api_key_limit: req.apiKeyLimit }), }); +export const subscriptionUsageCall = ( + accessToken: string, + startTime: Date, + endTime: Date, +): Promise => + apiClient.get("/subscription_accounts/usage", { + accessToken, + query: { start_date: formatDate(startTime), end_date: formatDate(endTime) }, + }); + +export const subscriptionAccountCreateCall = ( + accessToken: string, + body: SubscriptionAccountCreateRequest, +): Promise => apiClient.post("/subscription_accounts/new", { accessToken, body }); + +export const subscriptionAccountUpdateCall = ( + accessToken: string, + body: SubscriptionAccountUpdateRequest, +): Promise => apiClient.post("/subscription_accounts/update", { accessToken, body }); + +export const subscriptionAccountDeleteCall = ( + accessToken: string, + subscriptionAccountId: string, +): Promise => { + const body: SubscriptionAccountDeleteRequest = { subscription_account_id: subscriptionAccountId }; + return apiClient.post("/subscription_accounts/delete", { accessToken, body }); +}; + export const dailyActivityKeyPageCall = ( entity: DailyActivityEntity, req: DailyActivityRequest, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 13a064517a4..960616e2631 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -699,8 +699,8 @@ export interface paths { * @description Update an access group's model names. * * This will: - * 1. Remove the access group from all current deployments - * 2. Add the access group to all deployments for the new model_names list + * 1. Add the access group to all deployments for the new model_names list + * 2. Remove the access group from every other deployment * * Example: * ```bash @@ -722,6 +722,7 @@ export interface paths { * Raises: * - HTTPException 400: If any model names don't exist * - HTTPException 404: If access group not found + * - HTTPException 409: If another edit holds the group or one of its models for 30 s */ put: operations["update_access_group_access_group__access_group__update_put"]; post?: never; @@ -16079,6 +16080,90 @@ export interface paths { patch?: never; trace?: never; }; + "/subscription_accounts/delete": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Delete Subscription Account + * @description Stop tracking a subscription account's fee; its requests stay marked as subscription-covered. + */ + post: operations["delete_subscription_account_subscription_accounts_delete_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/subscription_accounts/new": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * New Subscription Account + * @description Record the monthly fee of one subscription-backed provider account. + */ + post: operations["new_subscription_account_subscription_accounts_new_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/subscription_accounts/update": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Update Subscription Account + * @description Change the fee, currency, billing period start, or label of a subscription account. + * Only the fields named in the request are written, so two admins changing different + * fields at the same time both land. + */ + post: operations["update_subscription_account_subscription_accounts_update_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/subscription_accounts/usage": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Subscription Usage + * @description Subscription-backed accounts behind the proxy's deployments or carrying a stored fee, each with + * its configured monthly fee and the fixed cost attributed to the date range: the fee once per billing + * period whose start date falls inside the range, however many deployments share the account. + */ + get: operations["get_subscription_usage_subscription_accounts_usage_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/systemone": { parameters: { query?: never; @@ -27401,6 +27486,19 @@ export interface components { /** Entities */ entities: components["schemas"]["BedrockChecksSensitiveInformationEntityItem"][]; }; + /** BillingPeriod */ + BillingPeriod: { + /** + * End + * Format: date + */ + end: string; + /** + * Start + * Format: date + */ + start: string; + }; /** BlockKeyRequest */ BlockKeyRequest: { /** Key */ @@ -29063,7 +29161,7 @@ export interface components { /** Token */ token: string; /** Top Logprobs */ - top_logprobs: components["schemas"]["TopLogprob"][]; + top_logprobs?: components["schemas"]["TopLogprob"][]; } & { [key: string]: unknown; }; @@ -29194,6 +29292,8 @@ export interface components { ChoiceLogprobs: { /** Content */ content?: components["schemas"]["ChatCompletionTokenLogprob"][] | null; + /** Refusal */ + refusal?: components["schemas"]["ChatCompletionTokenLogprob"][] | null; } & { [key: string]: unknown; }; @@ -31881,6 +31981,11 @@ export interface components { */ type: string; }; + /** DeleteSubscriptionAccountRequest */ + DeleteSubscriptionAccountRequest: { + /** Subscription Account Id */ + subscription_account_id: string; + }; /** DeleteTeamRequest */ DeleteTeamRequest: { /** Team Ids */ @@ -36016,6 +36121,37 @@ export interface components { */ user: string | null; }; + /** LiteLLM_SubscriptionAccountTable */ + LiteLLM_SubscriptionAccountTable: { + /** Account Id */ + account_id: string; + /** Billing Period Start */ + billing_period_start: string; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** Created By */ + created_by: string; + /** Currency */ + currency: string; + /** Custom Llm Provider */ + custom_llm_provider: string; + /** Label */ + label?: string | null; + /** Monthly Fee */ + monthly_fee: number; + /** Subscription Account Id */ + subscription_account_id: string; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + /** Updated By */ + updated_by: string; + }; /** LiteLLM_TeamMembership */ LiteLLM_TeamMembership: { /** Budget Id */ @@ -46224,6 +46360,55 @@ export interface components { /** Window Start */ window_start: number; }; + /** SubscriptionAccountFeeRequest */ + SubscriptionAccountFeeRequest: { + /** Account Id */ + account_id: string; + /** + * Billing Period Start + * Format: date + */ + billing_period_start: string; + /** Currency */ + currency: string; + /** Custom Llm Provider */ + custom_llm_provider: string; + /** Label */ + label?: string | null; + /** Monthly Fee */ + monthly_fee: number; + }; + /** SubscriptionAccountUsage */ + SubscriptionAccountUsage: { + /** Account Id */ + account_id: string | null; + /** Billing Periods */ + billing_periods: components["schemas"]["BillingPeriod"][]; + /** Custom Llm Provider */ + custom_llm_provider: string; + /** Deployments */ + deployments: string[]; + fee: components["schemas"]["LiteLLM_SubscriptionAccountTable"] | null; + /** Fixed Cost */ + fixed_cost: number | null; + }; + /** SubscriptionUsageResponse */ + SubscriptionUsageResponse: { + /** Accounts */ + accounts: components["schemas"]["SubscriptionAccountUsage"][]; + /** + * End Date + * Format: date + */ + end_date: string; + /** + * Start Date + * Format: date + */ + start_date: string; + /** Subscription Providers */ + subscription_providers: string[]; + }; /** * SuccessfulKeyUpdate * @description Successfully updated key with its updated information @@ -48834,6 +49019,12 @@ export interface components { UpdateRouterConfig: { /** Allowed Fails */ allowed_fails?: number | null; + /** Content Policy Fallbacks */ + content_policy_fallbacks?: { + [key: string]: (string | { + [key: string]: components["schemas"]["JsonValue"]; + })[]; + }[] | null; /** Context Window Fallbacks */ context_window_fallbacks?: { [key: string]: unknown; @@ -48897,6 +49088,19 @@ export interface components { UpdateSearchToolRequest: { search_tool: components["schemas"]["SearchTool"]; }; + /** UpdateSubscriptionAccountFeeRequest */ + UpdateSubscriptionAccountFeeRequest: { + /** Billing Period Start */ + billing_period_start?: string | null; + /** Currency */ + currency?: string | null; + /** Label */ + label?: string | null; + /** Monthly Fee */ + monthly_fee?: number | null; + /** Subscription Account Id */ + subscription_account_id: string; + }; /** * UpdateTeamMemberPermissionsRequest * @description Request to update the team member permissions for a team @@ -72987,6 +73191,139 @@ export interface operations { }; }; }; + delete_subscription_account_subscription_accounts_delete_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["DeleteSubscriptionAccountRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["LiteLLM_SubscriptionAccountTable"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + new_subscription_account_subscription_accounts_new_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SubscriptionAccountFeeRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["LiteLLM_SubscriptionAccountTable"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_subscription_account_subscription_accounts_update_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UpdateSubscriptionAccountFeeRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["LiteLLM_SubscriptionAccountTable"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_subscription_usage_subscription_accounts_usage_get: { + parameters: { + query: { + /** @description First day of the Usage page filter, YYYY-MM-DD */ + start_date: string; + /** @description Last day of the Usage page filter, YYYY-MM-DD */ + end_date: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SubscriptionUsageResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; systemone_systemone_post: { parameters: { query?: never;