diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index f9dcec30612..a6c32d78c00 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -35,11 +35,6 @@ from litellm.types.utils import ( StandardLoggingGuardrailInformation, ) -try: - from fastapi.exceptions import HTTPException -except ImportError: - HTTPException = None - if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation @@ -107,9 +102,9 @@ def is_guardrail_intervention(e: Exception) -> bool: ), ): return True - if HTTPException is not None and isinstance(e, HTTPException) and e.status_code in _GUARDRAIL_BLOCK_STATUS_CODES: - return True - return False + from litellm.proxy.guardrails.exception_utils import is_fastapi_http_exception + + return is_fastapi_http_exception(e, _GUARDRAIL_BLOCK_STATUS_CODES) def _strict_guardrail_modes_enabled() -> bool: diff --git a/litellm/integrations/gcs_bucket/gcs_bucket.py b/litellm/integrations/gcs_bucket/gcs_bucket.py index e338f490496..092357ae92b 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket.py @@ -14,7 +14,6 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase from litellm.litellm_core_utils.cloud_storage_security import ( sanitize_cloud_object_component, ) -from litellm.proxy._types import CommonProxyErrors from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus from litellm.types.integrations.gcs_bucket import * from litellm.types.utils import StandardLoggingPayload @@ -27,6 +26,7 @@ else: class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): def __init__(self, bucket_name: str | None = None) -> None: + from litellm.proxy._types import CommonProxyErrors from litellm.proxy.proxy_server import premium_user self.batch_size = int(os.getenv("GCS_BATCH_SIZE", GCS_DEFAULT_BATCH_SIZE)) @@ -52,6 +52,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): #### ASYNC #### async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + from litellm.proxy._types import CommonProxyErrors from litellm.proxy.proxy_server import premium_user if premium_user is not True: diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index baa9aab1087..6a617d962ea 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -519,7 +519,6 @@ def _get_token_base_cost( current_time: datetime | None = None, *, threshold_is_inclusive: bool = False, - missing_cache_read_uses_input: bool = False, ) -> tuple[float, float, float, float, float]: """ Return prompt cost, completion cost, and cache costs for a given model and usage. @@ -530,13 +529,11 @@ def _get_token_base_cost( `threshold_is_inclusive` switches that comparison to >=, for providers such as xAI that bill the higher tier once the prompt reaches the threshold. - `missing_cache_read_uses_input` resolves an absent cache-read rate to the resolved - input rate instead of 0.0; an explicit 0.0 rate stays a real price either way. - - An absent cache-creation rate always resolves to the resolved input rate, the way the - tiered table and custom deployment pricing already do, since a provider that publishes - no write price bills cache writes as ordinary input. An absent 1h write rate resolves - to the cache-creation rate, off-peak included. An explicit 0.0 stays a real price for both. + An absent cache-creation or cache-read rate always resolves to the resolved input + rate, the way the tiered table and custom deployment pricing already do, since a + provider that publishes no cache price bills cached tokens as ordinary input. An + absent 1h write rate resolves to the cache-creation rate, off-peak included. An + explicit 0.0 stays a real price for all of them. Returns: Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost) @@ -663,8 +660,7 @@ def _get_token_base_cost( "input_cost_per_token", prompt_base_cost, ) - if cache_read_cost is None: - cache_read_cost = input_rate_for_missing_cache_rates if missing_cache_read_uses_input else 0.0 + resolved_cache_read_cost: Final = input_rate_for_missing_cache_rates if cache_read_cost is None else cache_read_cost resolved_cache_creation_cost: Final = ( input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost ) @@ -677,7 +673,7 @@ def _get_token_base_cost( completion_base_cost, resolved_cache_creation_cost, cache_creation_cost_above_1hr, - cache_read_cost, + resolved_cache_read_cost, ), ) @@ -1588,7 +1584,6 @@ def calculate_prompt_caching_savings( service_tier=service_tier, current_time=billed_at, threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), - missing_cache_read_uses_input=True, ) write_rate: Final = cache_creation_cost or prompt_base_cost write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 4c61fac82bb..6c1b7946394 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -15,6 +15,7 @@ from typing_extensions import ParamSpec, TypeVar import litellm from litellm import verbose_logger +from litellm._lazy_imports import _get_default_encoding from litellm.constants import ( DEFAULT_IMAGE_HEIGHT, DEFAULT_IMAGE_TOKEN_COUNT, @@ -29,7 +30,6 @@ from litellm.constants import ( TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.asyncify import asyncify -from litellm.litellm_core_utils.default_encoding import encoding as default_encoding from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import _get_httpx_client from litellm.types.llms.anthropic import ( @@ -638,7 +638,7 @@ def _get_exact_count_function( else: def encode_length(text: str) -> int: - return len(default_encoding.encode(text, disallowed_special=())) + return len(_get_default_encoding().encode(text, disallowed_special=())) return _get_tiktoken_count_function(encode_length) diff --git a/litellm/proxy/guardrails/exception_utils.py b/litellm/proxy/guardrails/exception_utils.py new file mode 100644 index 00000000000..47f2655fdaf --- /dev/null +++ b/litellm/proxy/guardrails/exception_utils.py @@ -0,0 +1,9 @@ +from collections.abc import Collection + + +def is_fastapi_http_exception(e: Exception, block_status_codes: Collection[int]) -> bool: + try: + from fastapi.exceptions import HTTPException + except ImportError: + return False + return isinstance(e, HTTPException) and e.status_code in block_status_codes diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 41a3ded7022..3292a3ac458 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1,6 +1,7 @@ import base64 import re from collections.abc import Iterable, Mapping, Sequence +from functools import reduce from typing import Any, Final, Optional, TypeVar, Union, cast, get_type_hints, overload from pydantic import BaseModel @@ -8,6 +9,7 @@ from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire pay import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.types.llms.openai import ( AllMessageValues, @@ -29,6 +31,11 @@ from litellm.types.utils import ( ) +def _apply_nested_drop_params(params: dict[str, object], additional_drop_params: list[str] | None) -> dict[str, object]: + nested_paths: Final = tuple(path for path in additional_drop_params or () if is_nested_path(path)) + return reduce(lambda acc, path: delete_nested_value(acc, path), nested_paths, params) + + def _output_token_detail(details: object, field: str) -> int | None: value: Final = getattr(details, field, None) return value if isinstance(value, int) else None @@ -265,20 +272,24 @@ class ResponsesAPIRequestUtils: special_params: Final[dict[str, object]] = params.pop("kwargs", {}) additional_drop_params: Final[list[str] | None] = params.pop("additional_drop_params", None) - non_default_params: Final = PreProcessNonDefaultParams.base_pre_process_non_default_params( - passed_params=params, - special_params=special_params, - custom_llm_provider=custom_llm_provider, - additional_drop_params=additional_drop_params, - default_param_values={k: None for k in valid_keys}, - additional_endpoint_specific_params=["input"], + non_default_params: Final = _apply_nested_drop_params( + PreProcessNonDefaultParams.base_pre_process_non_default_params( + passed_params=params, + special_params=special_params, + custom_llm_provider=custom_llm_provider, + additional_drop_params=additional_drop_params, + default_param_values={k: None for k in valid_keys}, + additional_endpoint_specific_params=["input"], + ), + additional_drop_params, ) # decode previous_response_id if it's a litellm encoded id - if "previous_response_id" in non_default_params: + previous_response_id: Final = non_default_params.get("previous_response_id") + if isinstance(previous_response_id, str): decoded_previous_response_id: Final = ( ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id( - non_default_params["previous_response_id"] + previous_response_id ) ) non_default_params["previous_response_id"] = decoded_previous_response_id @@ -286,7 +297,8 @@ class ResponsesAPIRequestUtils: if "metadata" in non_default_params: from litellm.utils import add_openai_metadata - converted_metadata: Final = add_openai_metadata(non_default_params["metadata"]) + raw_metadata: Final = non_default_params["metadata"] + converted_metadata: Final = add_openai_metadata(raw_metadata if _is_object_dict(raw_metadata) else None) if converted_metadata is not None: non_default_params["metadata"] = converted_metadata else: diff --git a/litellm/router.py b/litellm/router.py index 633f060f208..864d5c6053e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -155,7 +155,7 @@ from litellm.router_utils.batch_utils import ( replace_model_in_jsonl, should_replace_model_in_jsonl, ) -from litellm.router_utils.client_initalization_utils import InitalizeCachedClient +from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit from litellm.router_utils.clientside_credential_handler import ( get_dynamic_litellm_params, is_clientside_credential, @@ -3642,24 +3642,22 @@ class Router: input_kwargs.pop("silent_model", None) input_kwargs.pop("include_fallback_errors", None) - _response: Final = litellm.acompletion(**input_kwargs) - logging_obj: Final[LiteLLMLogging | None] = kwargs.get("litellm_logging_obj", None) - rpm_semaphore: Final = self._get_client( + max_parallel_requests_limit: Final = self._get_client( deployment=deployment, kwargs=kwargs, client_type="max_parallel_requests", ) async with contextlib.AsyncExitStack() as deployment_slot: - if isinstance(rpm_semaphore, asyncio.Semaphore): - await deployment_slot.enter_async_context(rpm_semaphore) + if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit): + deployment_slot.enter_context(max_parallel_requests_limit) await self.async_routing_strategy_pre_call_checks( deployment=deployment, logging_obj=logging_obj, parent_otel_span=parent_otel_span, ) - response = await _response + response = await litellm.acompletion(**input_kwargs) ## CHECK CONTENT FILTER ERROR ## if isinstance(response, ModelResponse): @@ -4586,38 +4584,16 @@ class Router: ) self.total_calls[model_name] += 1 - response = litellm.aimage_generation( - **{ - **data, - "prompt": prompt, - "caching": self.cache_responses, - "client": model_client, - **kwargs, - } - ) - - ### CONCURRENCY-SAFE RPM CHECKS ### - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - """ - - Check rpm limits before making the call - - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) - """ - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span + async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): + response = await litellm.aimage_generation( + **{ + **data, + "prompt": prompt, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } ) - response = await response self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.aimage_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -4691,38 +4667,16 @@ class Router: ) self.total_calls[model_name] += 1 - response = litellm.atranscription( - **{ - **data, - "file": file, - "caching": self.cache_responses, - "client": model_client, - **kwargs, - } - ) - - ### CONCURRENCY-SAFE RPM CHECKS ### - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - """ - - Check rpm limits before making the call - - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) - """ - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span + async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): + response = await litellm.atranscription( + **{ + **data, + "file": file, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } ) - response = await response self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.atranscription(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -4806,38 +4760,16 @@ class Router: ) self.total_calls[model_name] += 1 - response = litellm.aspeech( - **{ - **data, - "input": input, - "voice": data.get("voice") if voice is None else voice, - "client": model_client, - **kwargs, - } - ) - - ### CONCURRENCY-SAFE RPM CHECKS ### - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - """ - - Check rpm limits before making the call - - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) - """ - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span + async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): + response = await litellm.aspeech( + **{ + **data, + "input": input, + "voice": data.get("voice") if voice is None else voice, + "client": model_client, + **kwargs, + } ) - response = await response self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.aspeech(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -5002,37 +4934,16 @@ class Router: ) self.total_calls[model_name] += 1 - response = litellm.atext_completion( - **{ - **data, - "prompt": prompt, - "caching": self.cache_responses, - "client": model_client, - **kwargs, - } - ) - - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - """ - - Check rpm limits before making the call - - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) - """ - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span + async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): + response = await litellm.atext_completion( + **{ + **data, + "prompt": prompt, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } ) - response = await response self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.atext_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -5093,37 +5004,16 @@ class Router: ) self.total_calls[model_name] += 1 - response = litellm.aadapter_completion( - **{ - **data, - "adapter_id": adapter_id, - "caching": self.cache_responses, - "client": model_client, - **kwargs, - } - ) - - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - """ - - Check rpm limits before making the call - - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) - """ - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span + async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): + response = await litellm.aadapter_completion( + **{ + **data, + "adapter_id": adapter_id, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } ) - response = await response self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.aadapter_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -5353,29 +5243,8 @@ class Router: if custom_llm_provider is not None: response_kwargs["custom_llm_provider"] = custom_llm_provider - response = original_generic_function(**response_kwargs) - - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - """ - - Check rpm limits before making the call - - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) - """ - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response + async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): + response = await original_generic_function(**response_kwargs) if self._should_raise_anthropic_refusal_error( model=model, @@ -5983,38 +5852,16 @@ class Router: ) self.total_calls[model_name] += 1 - response = litellm.aembedding( - **{ - **data, - "input": input, - "caching": self.cache_responses, - "client": model_client, - **kwargs, - } - ) - - ### CONCURRENCY-SAFE RPM CHECKS ### - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - """ - - Check rpm limits before making the call - - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) - """ - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span + async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): + response = await litellm.aembedding( + **{ + **data, + "input": input, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } ) - response = await response self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.aembedding(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -6123,37 +5970,18 @@ class Router: "gcs_bucket_name" in data ): # TODO: Remove this once we have a better way to handle GCS bucket name: Problem is that we need to pass the gcs_bucket_name to the router for the create_file call but it doesn't show up there kwargs_copy.setdefault("litellm_metadata", {})["gcs_bucket_name"] = data["gcs_bucket_name"] - response = litellm.acreate_file( - **{ - **data, - "custom_llm_provider": custom_llm_provider, - "caching": self.cache_responses, - "client": model_client, - **kwargs_copy, - } - ) - - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs_copy, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - """ - - Check rpm limits before making the call - - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) - """ - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span + async with self._deployment_slot( + deployment=deployment, kwargs=kwargs_copy, parent_otel_span=parent_otel_span + ): + response = await litellm.acreate_file( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self.cache_responses, + "client": model_client, + **kwargs_copy, + } ) - response = await response self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.acreate_file(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -6243,33 +6071,16 @@ class Router: ) custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider - response = avector_store_create_sdk( - **{ - **data, - "custom_llm_provider": custom_llm_provider, - "caching": self.cache_responses, - "client": model_client, - **kwargs, - } - ) - - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span + async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): + response = await avector_store_create_sdk( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } ) - response = await response self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.avector_store_create(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -6355,37 +6166,16 @@ class Router: ) custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider - response = litellm.acreate_batch( - **{ - **data, - "custom_llm_provider": custom_llm_provider, - "caching": self.cache_responses, - "client": model_client, - **kwargs, - } - ) - - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - """ - - Check rpm limits before making the call - - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) - """ - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span + async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): + response = await litellm.acreate_batch( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } ) - response = await response self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.acreate_batch(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -6576,37 +6366,16 @@ class Router: ) custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider - response = litellm.acancel_batch( - **{ - **data, - "custom_llm_provider": custom_llm_provider, - "caching": self.cache_responses, - "client": model_client, - **kwargs, - } - ) - - rpm_semaphore: Final = self._get_client( - deployment=deployment, - kwargs=kwargs, - client_type="max_parallel_requests", - ) - - if rpm_semaphore is not None and isinstance(rpm_semaphore, asyncio.Semaphore): - async with rpm_semaphore: - """ - - Check rpm limits before making the call - - If allowed, increment the rpm limit (allows global value to be updated, concurrency-safe) - """ - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span - ) - response = await response - else: - await self.async_routing_strategy_pre_call_checks( - deployment=deployment, parent_otel_span=parent_otel_span + async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): + response = await litellm.acancel_batch( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } ) - response = await response self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.acancel_batch(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -8741,6 +8510,23 @@ class Router: ) raise e + @contextlib.asynccontextmanager + async def _deployment_slot( + self, deployment: dict, kwargs: Mapping[str, object], parent_otel_span: Span | None + ) -> AsyncGenerator[None, None]: + """Holds the deployment's max_parallel_requests slot, if it has one, around the provider call. Routing + strategy pre-call checks run inside the slot so their rpm accounting stays concurrency-safe.""" + max_parallel_requests_limit: Final = self._get_client( + deployment=deployment, + kwargs=kwargs, + client_type="max_parallel_requests", + ) + async with contextlib.AsyncExitStack() as slot: + if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit): + slot.enter_context(max_parallel_requests_limit) + await self.async_routing_strategy_pre_call_checks(deployment=deployment, parent_otel_span=parent_otel_span) + yield + async def async_callback_filter_deployments( self, model: str, diff --git a/litellm/router_utils/client_initalization_utils.py b/litellm/router_utils/client_initalization_utils.py index 24324334a86..55b4c071cb0 100644 --- a/litellm/router_utils/client_initalization_utils.py +++ b/litellm/router_utils/client_initalization_utils.py @@ -1,6 +1,8 @@ -import asyncio +from types import TracebackType from typing import TYPE_CHECKING, Any, Final +from litellm.exceptions import RateLimitError, RateLimitErrorCategory, RateLimitType +from litellm.types.router import RouterErrors from litellm.utils import calculate_max_parallel_requests if TYPE_CHECKING: @@ -11,6 +13,43 @@ else: LitellmRouter = Any +class MaxParallelRequestsLimit: + """A deployment's max_parallel_requests slots. A caller arriving while every slot is in use gets a 429 instead + of waiting for one to free up.""" + + def __init__(self, max_parallel_requests: int, model_id: str, model_group: str) -> None: + self.max_parallel_requests: Final = max_parallel_requests + self.model_id: Final = model_id + self.model_group: Final = model_group + self.in_flight = 0 + + def __enter__(self) -> None: + self.acquire() + + def __exit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None + ) -> None: + self.release() + + def acquire(self) -> None: + if self.in_flight >= self.max_parallel_requests: + raise RateLimitError( + message=( + f"{RouterErrors.max_parallel_requests_exceeded.value} Deployment model_group={self.model_group}, " + f"id={self.model_id} already has max_parallel_requests={self.max_parallel_requests} requests in " + "flight. Raise max_parallel_requests (or the rpm/tpm it is derived from) for this deployment" + ), + llm_provider="", + model=self.model_group, + category=RateLimitErrorCategory.LITELLM_RATE_LIMIT, + rate_limit_type=RateLimitType.CONCURRENT_REQUESTS, + ) + self.in_flight += 1 + + def release(self) -> None: + self.in_flight -= 1 + + class InitalizeCachedClient: @staticmethod def set_max_parallel_requests_client(litellm_router_instance: LitellmRouter, model: dict): @@ -26,10 +65,14 @@ class InitalizeCachedClient: default_max_parallel_requests=litellm_router_instance.default_max_parallel_requests, ) if calculated_max_parallel_requests: - semaphore: Final = asyncio.Semaphore(calculated_max_parallel_requests) + limit: Final = MaxParallelRequestsLimit( + max_parallel_requests=calculated_max_parallel_requests, + model_id=model_id, + model_group=model.get("model_name", ""), + ) cache_key: Final = f"{model_id}_max_parallel_requests_client" litellm_router_instance.cache.set_cache( key=cache_key, - value=semaphore, + value=limit, local_only=True, ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 592039bd2c0..dbd4180e5ea 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -653,6 +653,7 @@ class RouterErrors(enum.Enum): """ user_defined_ratelimit_error = "Deployment over user-defined ratelimit." + max_parallel_requests_exceeded = "Deployment has all max_parallel_requests slots in use." no_deployments_available = "No deployments available for selected model" all_deployments_in_cooldown = "All deployments for selected model are in cooldown" no_deployments_with_tag_routing = "Not allowed to access model due to tags configuration" diff --git a/tests/local_testing/test_router_max_parallel_requests.py b/tests/local_testing/test_router_max_parallel_requests.py index 65602c968bc..051c69c9322 100644 --- a/tests/local_testing/test_router_max_parallel_requests.py +++ b/tests/local_testing/test_router_max_parallel_requests.py @@ -11,6 +11,7 @@ import pytest from typing import Optional import litellm +from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit from litellm.utils import calculate_max_parallel_requests """ @@ -93,26 +94,26 @@ def test_setting_mpr_limits_per_model( default_max_parallel_requests=default_max_parallel_requests, ) - mpr_client: Optional[asyncio.Semaphore] = router._get_client( + mpr_client: Optional[MaxParallelRequestsLimit] = router._get_client( deployment=deployment, kwargs={}, client_type="max_parallel_requests", ) if max_parallel_requests is not None: - assert max_parallel_requests == mpr_client._value + assert max_parallel_requests == mpr_client.max_parallel_requests elif rpm is not None: - assert rpm == mpr_client._value + assert rpm == mpr_client.max_parallel_requests elif tpm is not None: calculated_rpm = int(tpm / 1000 * 6) if calculated_rpm == 0: calculated_rpm = 1 print( - f"test calculated_rpm: {calculated_rpm}, calculated_max_parallel_requests={mpr_client._value}" + f"test calculated_rpm: {calculated_rpm}, calculated_max_parallel_requests={mpr_client.max_parallel_requests}" ) - assert calculated_rpm == mpr_client._value + assert calculated_rpm == mpr_client.max_parallel_requests elif default_max_parallel_requests is not None: - assert mpr_client._value == default_max_parallel_requests + assert mpr_client.max_parallel_requests == default_max_parallel_requests else: assert mpr_client is None diff --git a/tests/test_litellm/integrations/gcs_bucket/test_gcs_bucket_base.py b/tests/test_litellm/integrations/gcs_bucket/test_gcs_bucket_base.py index a458752bed0..fb53994089b 100644 --- a/tests/test_litellm/integrations/gcs_bucket/test_gcs_bucket_base.py +++ b/tests/test_litellm/integrations/gcs_bucket/test_gcs_bucket_base.py @@ -131,6 +131,13 @@ class TestGCSBucketBase: class TestGCSBucketLoggerBucketName: + @pytest.mark.asyncio + async def test_constructor_rejects_non_premium_user(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + + with pytest.raises(ValueError, match="GCS Bucket logging is a premium feature"): + GCSBucketLogger(bucket_name="config-bucket") + @pytest.mark.asyncio async def test_the_bucket_name_it_is_constructed_with_survives(self, monkeypatch): """Reading config.yaml out of a GCS bucket asks for that bucket, not the logging one (LIT-6982).""" @@ -145,3 +152,11 @@ class TestGCSBucketLoggerBucketName: monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) assert GCSBucketLogger().BUCKET_NAME == "logging-bucket" + + @pytest.mark.asyncio + async def test_async_logging_rejects_non_premium_user(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + logger = object.__new__(GCSBucketLogger) + + with pytest.raises(ValueError, match="GCS Bucket logging is a premium feature"): + await logger.async_log_success_event({}, None, None, None) diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index b47aee79efc..6ffbd4e3f1f 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -1754,6 +1754,20 @@ class TestCustomGuardrailSpendLogMatchRedaction: class TestGuardrailInterventionClassification: """A routing decision is a deliberate guardrail intervention, not a failure.""" + def test_http_exception_classification_returns_false_without_fastapi(self, monkeypatch): + import builtins + + real_import = builtins.__import__ + + def import_without_fastapi(name, *args, **kwargs): + if name == "fastapi.exceptions": + raise ImportError("fastapi is unavailable") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_without_fastapi) + + assert CustomGuardrail._is_guardrail_intervention(Exception("not an intervention")) is False + def test_sensitive_data_route_exception_is_intervention(self): from litellm.exceptions import SensitiveDataRouteException diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 776d78a04e0..686a792fa0f 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -48,7 +48,7 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: @pytest.mark.parametrize("prompt_tokens", [100, 200000, 200001]) @pytest.mark.parametrize("read_rate", [None, 0.0, 0.25e-6]) @pytest.mark.parametrize("service_tier", [None, "priority"]) -def test_missing_cache_read_policy_preserves_billing(prompt_tokens, read_rate, service_tier): +def test_missing_cache_read_rate_resolves_to_input_rate(prompt_tokens, read_rate, service_tier): info = { "input_cost_per_token": 3e-6, "input_cost_per_token_priority": 4e-6, @@ -59,16 +59,43 @@ def test_missing_cache_read_policy_preserves_billing(prompt_tokens, read_rate, s } usage = Usage(prompt_tokens=prompt_tokens, prompt_tokens_details={"cached_tokens": 100}) billed = _get_token_base_cost(info, usage, service_tier=service_tier) - savings = _get_token_base_cost(info, usage, service_tier=service_tier, missing_cache_read_uses_input=True) prompt_cost, _ = generic_cost_per_token( "policy-fixture", usage, "openai", service_tier=service_tier, model_info=info ) - assert billed[4] == pytest.approx(read_rate or 0.0) - assert savings[:4] == billed[:4] - assert savings[4] == pytest.approx(billed[0] if read_rate is None else read_rate) + assert billed[4] == pytest.approx(read_rate if read_rate is not None else billed[0]) assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4]) +def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None: + model_info: ModelInfo = { + "key": "bare-model", + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": 2.4e-7, + "output_cost_per_token": 9.7e-7, + "litellm_provider": "bedrock", + "mode": "chat", + "supported_openai_params": None, + } + usage = Usage( + prompt_tokens=12928, + completion_tokens=380, + total_tokens=13308, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=12288), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model="bare-model", + usage=usage, + custom_llm_provider="bedrock", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(12928 * 2.4e-7) + assert completion_cost == pytest.approx(380 * 9.7e-7) + + def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: model_info: ModelInfo = { "key": "gemini-embedding-2", @@ -180,11 +207,7 @@ def test_missing_cache_read_uses_off_peak_input_rate(): } when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc) billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when) - savings = _get_token_base_cost( - info, Usage(prompt_tokens=100), current_time=when, missing_cache_read_uses_input=True - ) - assert billed[4] == 0.0 - assert savings[0] == savings[4] == 5e-6 + assert billed[0] == billed[4] == 5e-6 def test_reasoning_tokens_no_price_set(_local_model_cost_map): diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 60f25c48443..ba3a6be609f 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -98,6 +98,13 @@ def test_token_counter_short_text_matches_tiktoken(text): assert token_counter_new(model="us.anthropic.claude-sonnet-4-6", text=text) == expected +def test_token_counter_default_encoding_matches_cl100k(): + encoding: Final = tiktoken.get_encoding("cl100k_base") + expected: Final = len(encoding.encode("hello world", disallowed_special=())) + + assert token_counter_new(model=None, text="hello world") == expected + + def test_token_counter_text_over_chunk_boundary_stays_close_to_tiktoken(): text = ("The quick brown fox jumps over the lazy dog. " * 30)[:1025] encoding = tiktoken.get_encoding("cl100k_base") diff --git a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py index 8c3e88b864a..94d388fce60 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py @@ -975,9 +975,9 @@ class TestEstimateCostCacheAndReasoningTokens: @pytest.mark.asyncio async def test_a_model_without_cache_or_reasoning_prices_estimates_what_the_proxy_bills(self, monkeypatch): - """The cost calculator bills cache reads of a cost-map model without cache prices at zero, - its cache writes at the input rate, and its reasoning tokens at the output rate. The estimate - reports those effective rates.""" + """The cost calculator bills cache reads and writes of a cost-map model without cache prices + at the input rate, and its reasoning tokens at the output rate. The estimate reports those + effective rates.""" monkeypatch.setitem( litellm.model_cost, A_MAPPED_MODEL, @@ -986,14 +986,12 @@ class TestEstimateCostCacheAndReasoningTokens: response = await _estimate_with_cache_and_reasoning(None, model=A_MAPPED_MODEL) - assert response.cache_read_cost_per_request == 0.0 + assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 5e-6) assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 5e-6) assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 6e-6) - assert response.input_cost_per_request == pytest.approx((TEXT_INPUT_TOKENS + CACHE_CREATION_TOKENS) * 5e-6) - assert response.cost_per_request == pytest.approx( - (TEXT_INPUT_TOKENS + CACHE_CREATION_TOKENS) * 5e-6 + OUTPUT_TOKENS * 6e-6 - ) - assert response.cache_read_input_token_cost == 0.0 + assert response.input_cost_per_request == pytest.approx(INPUT_TOKENS * 5e-6) + assert response.cost_per_request == pytest.approx(INPUT_TOKENS * 5e-6 + OUTPUT_TOKENS * 6e-6) + assert response.cache_read_input_token_cost == pytest.approx(5e-6) assert response.cache_creation_input_token_cost == pytest.approx(5e-6) assert response.output_cost_per_reasoning_token == pytest.approx(6e-6) diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 4d06b5e7bdc..7a482488706 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -119,6 +119,63 @@ class TestResponsesAPIRequestUtils: assert result["max_output_tokens"] == 100 assert result["prompt"] == {"id": "pmpt_456"} + def test_get_requested_response_api_optional_param_drops_nested_path(self): + """Nested additional_drop_params paths like reasoning.summary must be honored""" + params = { + "temperature": 0.1, + "reasoning": {"effort": "high", "summary": "auto"}, + "additional_drop_params": ["reasoning.summary"], + } + + result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params) + + assert result["reasoning"] == {"effort": "high"} + assert result["temperature"] == 0.1 + + def test_get_requested_response_api_optional_param_drops_array_path(self): + """Array wildcard paths like tools[*].input_examples must be honored""" + params = { + "tools": [{"type": "function", "name": "t", "input_examples": ["x"]}], + "additional_drop_params": ["tools[*].input_examples"], + } + + result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params) + + assert result["tools"] == [{"type": "function", "name": "t"}] + + def test_get_requested_response_api_optional_param_drops_top_level(self): + """Top-level additional_drop_params keys must still be honored""" + params = { + "reasoning": {"effort": "high", "summary": "auto"}, + "additional_drop_params": ["reasoning"], + } + + result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params) + + assert "reasoning" not in result + + def test_get_requested_response_api_optional_param_non_matching_nested_path(self): + """A nested path that does not match anything leaves params untouched""" + params = { + "reasoning": {"effort": "high", "summary": "auto"}, + "additional_drop_params": ["reasoning.nope"], + } + + result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params) + + assert result["reasoning"] == {"effort": "high", "summary": "auto"} + + def test_get_requested_response_api_optional_param_none_drop_params(self): + """additional_drop_params=None is a no-op""" + params = { + "reasoning": {"effort": "high", "summary": "auto"}, + "additional_drop_params": None, + } + + result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params) + + assert result["reasoning"] == {"effort": "high", "summary": "auto"} + def test_decode_previous_response_id_to_original_previous_response_id(self): """Test decoding a LiteLLM encoded previous_response_id to the original previous_response_id""" # Setup diff --git a/tests/test_litellm/router_utils/test_client_initalization_utils.py b/tests/test_litellm/router_utils/test_client_initalization_utils.py new file mode 100644 index 00000000000..6f9a7b730ac --- /dev/null +++ b/tests/test_litellm/router_utils/test_client_initalization_utils.py @@ -0,0 +1,125 @@ +import asyncio +from typing import Final + +import pytest + +import litellm +from litellm import Router +from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit + + +def _limit(max_parallel_requests: int = 1) -> MaxParallelRequestsLimit: + return MaxParallelRequestsLimit( + max_parallel_requests=max_parallel_requests, model_id="deployment-1", model_group="gpt-5.6" + ) + + +async def _hold(limit: MaxParallelRequestsLimit, release: asyncio.Event) -> str: + with limit: + await release.wait() + return "ok" + + +def _expect_rejection(limit: MaxParallelRequestsLimit) -> litellm.RateLimitError: + with pytest.raises(litellm.RateLimitError) as excinfo: + limit.acquire() + return excinfo.value + + +@pytest.mark.asyncio +async def test_request_arriving_while_every_slot_is_in_use_gets_429_without_waiting(): + limit: Final = _limit(max_parallel_requests=2) + release: Final = asyncio.Event() + holders: Final = [asyncio.create_task(_hold(limit, release)) for _ in range(2)] + await asyncio.sleep(0) + assert limit.in_flight == 2 + + rejection: Final = _expect_rejection(limit) + + assert rejection.status_code == 429 + assert "deployment-1" in rejection.message + assert "gpt-5.6" in rejection.message + assert "max_parallel_requests=2" in rejection.message + assert limit.in_flight == 2 + + release.set() + assert await asyncio.wait_for(asyncio.gather(*holders), timeout=2) == ["ok", "ok"] + assert limit.in_flight == 0 + with limit: + assert limit.in_flight == 1 + assert limit.in_flight == 0 + + +@pytest.mark.asyncio +async def test_burst_over_the_cap_admits_exactly_max_parallel_requests_and_rejects_the_rest(): + limit: Final = _limit(max_parallel_requests=3) + release: Final = asyncio.Event() + + async def attempt() -> str: + try: + return await _hold(limit, release) + except litellm.RateLimitError as e: + return f"rejected:{e.status_code}" + + callers: Final = [asyncio.create_task(attempt()) for _ in range(10)] + await asyncio.sleep(0) + assert limit.in_flight == 3 + release.set() + outcomes: Final = await asyncio.wait_for(asyncio.gather(*callers), timeout=2) + assert outcomes.count("ok") == 3 + assert outcomes.count("rejected:429") == 7 + assert limit.in_flight == 0 + + +def test_slot_is_released_when_the_held_call_raises(): + limit: Final = _limit() + with pytest.raises(RuntimeError): + with limit: + raise RuntimeError("provider blew up") + assert limit.in_flight == 0 + with limit: + assert limit.in_flight == 1 + + +def _router_limit(router: Router, model_name: str) -> MaxParallelRequestsLimit: + deployment: Final = router.get_deployment_by_model_group_name(model_group_name=model_name) + assert deployment is not None + client: Final = router._get_client( + deployment=deployment.model_dump(), kwargs={}, client_type="max_parallel_requests" + ) + assert isinstance(client, MaxParallelRequestsLimit) + return client + + +@pytest.mark.parametrize( + ("litellm_params", "expected_cap"), + [ + ({"max_parallel_requests": 2, "rpm": 7, "tpm": 100_000}, 2), + ({"rpm": 7, "tpm": 100_000}, 7), + ({"tpm": 100_000}, 600), + ({"tpm": 100}, 1), + ], +) +@pytest.mark.asyncio +async def test_router_deployment_rejects_past_its_derived_cap(litellm_params: dict[str, int], expected_cap: int): + router: Final = Router( + model_list=[{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6", **litellm_params}}] + ) + limit: Final = _router_limit(router, "gpt-5.6") + assert limit.max_parallel_requests == expected_cap + release: Final = asyncio.Event() + holders: Final = [asyncio.create_task(_hold(limit, release)) for _ in range(expected_cap)] + await asyncio.sleep(0) + assert limit.in_flight == expected_cap + assert f"max_parallel_requests={expected_cap}" in _expect_rejection(limit).message + release.set() + assert await asyncio.wait_for(asyncio.gather(*holders), timeout=2) == ["ok"] * expected_cap + + +def test_router_without_any_concurrency_setting_has_no_limit(): + router: Final = Router(model_list=[{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6"}}]) + deployment: Final = router.get_deployment_by_model_group_name(model_group_name="gpt-5.6") + assert deployment is not None + assert ( + router._get_client(deployment=deployment.model_dump(), kwargs={}, client_type="max_parallel_requests") is None + ) diff --git a/tests/test_litellm/test_lazy_imports.py b/tests/test_litellm/test_lazy_imports.py index 2b16a812611..07ead78207b 100644 --- a/tests/test_litellm/test_lazy_imports.py +++ b/tests/test_litellm/test_lazy_imports.py @@ -1,6 +1,9 @@ """Simple tests for lazy import functionality.""" +import os +import subprocess import sys +from typing import Final import pytest @@ -38,6 +41,22 @@ from litellm._lazy_imports import ( ) +def test_import_litellm_does_not_load_fastapi_or_bpe_table(): + result: Final = subprocess.run( + [ + sys.executable, + "-c", + "import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.litellm_core_utils.default_encoding') if m in sys.modules))", + ], + check=True, + capture_output=True, + text=True, + env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, + ) + + assert result.stdout.strip() == "" + + def _clear_names_from_globals(names: tuple): """Clear all names from litellm globals.""" # Get the actual globals dict, not a copy diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 1e6636ec3d6..26f9803a022 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1,11 +1,13 @@ import asyncio import copy import functools +import gc import json import logging import os import sys import threading +import warnings from collections.abc import Awaitable, Callable, Mapping from datetime import datetime, timedelta from types import SimpleNamespace @@ -45,6 +47,8 @@ from litellm.router import ( _is_retriable_anthropic_status, ) from litellm.router_strategy import simple_shuffle +from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit +from litellm.router_utils.cooldown_handlers import _async_get_cooldown_deployments from litellm.types.llms.openai import ChatCompletionRequest from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo, PreRoutingHookResponse, RetryPolicy @@ -1517,7 +1521,9 @@ async def test_router_ageneric_api_call_with_fallbacks_helper(): }, } - mock_semaphore = asyncio.Semaphore(1) + mock_semaphore = MaxParallelRequestsLimit( + max_parallel_requests=1, model_id="deployment-1", model_group="gpt-3.5-turbo" + ) with patch.object( router, "_update_kwargs_with_deployment" @@ -15962,7 +15968,7 @@ def _max_parallel_router(max_parallel_requests: int) -> Router: @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) -async def test_router_max_parallel_requests_bounds_in_flight_upstream_calls( +async def test_router_max_parallel_requests_admits_the_cap_and_rejects_the_rest_with_429( monkeypatch: pytest.MonkeyPatch, stream: bool ): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -15988,24 +15994,33 @@ async def test_router_max_parallel_requests_bounds_in_flight_upstream_calls( }, ) - async def one_call() -> None: - response = await router.acompletion( - model="gpt-5.6", messages=[{"role": "user", "content": "hi"}], stream=stream - ) + async def one_call() -> str: + try: + response = await router.acompletion( + model="gpt-5.6", messages=[{"role": "user", "content": "hi"}], stream=stream + ) + except litellm.RateLimitError as e: + return f"rejected:{e.status_code}" if stream: async for _ in response: pass + return "ok" with respx.mock(assert_all_called=True) as respx_mock: - respx_mock.post("https://max-parallel.local/v1/chat/completions").mock(side_effect=upstream) - await asyncio.wait_for(asyncio.gather(*(one_call() for _ in range(10))), timeout=10) + route: Final = respx_mock.post("https://max-parallel.local/v1/chat/completions").mock(side_effect=upstream) + outcomes: Final = await asyncio.wait_for(asyncio.gather(*(one_call() for _ in range(10))), timeout=10) - assert tracker.peak <= 2 + assert outcomes.count("ok") == 2 + assert outcomes.count("rejected:429") == 8 + assert route.call_count == 2 + assert tracker.peak == 2 assert tracker.current == 0 @pytest.mark.asyncio -async def test_router_max_parallel_requests_slot_released_when_stream_closed_early(monkeypatch: pytest.MonkeyPatch): +async def test_router_max_parallel_requests_slot_held_until_stream_closed_then_released( + monkeypatch: pytest.MonkeyPatch, +): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) tracker: Final = _InFlightTracker() router: Final = _max_parallel_router(max_parallel_requests=1) @@ -16028,16 +16043,232 @@ async def test_router_max_parallel_requests_slot_released_when_stream_closed_ear async for _ in second: pass - second_task: Final = asyncio.create_task(second_call()) - await asyncio.sleep(0.05) assert tracker.current == 1 + with pytest.raises(litellm.RateLimitError) as while_streaming: + await second_call() + assert while_streaming.value.status_code == 429 await first.aclose() - await asyncio.wait_for(second_task, timeout=2) + await asyncio.wait_for(second_call(), timeout=2) assert tracker.peak == 1 assert tracker.current == 0 +@pytest.mark.asyncio +async def test_router_max_parallel_requests_overflow_is_429_without_cooldown_or_provider_call( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = Router( + model_list=[ + { + "model_name": "gpt-5.6", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-fake", + "api_base": "https://max-parallel.local/v1", + "max_parallel_requests": 1, + }, + "model_info": {"id": "capped-deployment"}, + }, + { + "model_name": "gpt-5.6", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-fake", + "api_base": "https://max-parallel-sibling.local/v1", + }, + "model_info": {"id": "sibling-deployment"}, + }, + ], + num_retries=0, + ) + + async def upstream(request: httpx.Request) -> httpx.Response: + await asyncio.sleep(0.2) + return httpx.Response( + 200, + json={ + "id": "c", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "x"}, "finish_reason": "stop"}], + }, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = respx_mock.post("https://max-parallel.local/v1/chat/completions").mock(side_effect=upstream) + sibling_route: Final = respx_mock.post("https://max-parallel-sibling.local/v1/chat/completions").mock( + side_effect=upstream + ) + results: Final = await asyncio.wait_for( + asyncio.gather( + *( + router.acompletion(model="capped-deployment", messages=[{"role": "user", "content": "hi"}]) + for _ in range(3) + ), + return_exceptions=True, + ), + timeout=10, + ) + + rejected: Final = [r for r in results if isinstance(r, BaseException)] + assert len(rejected) == 2 and len(results) == 3 + assert all(isinstance(r, litellm.RateLimitError) and r.status_code == 429 for r in rejected) + assert all("capped-deployment" in r.message and "max_parallel_requests=1" in r.message for r in rejected) + assert route.call_count == 1 + assert sibling_route.call_count == 0 + assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] + + +@pytest.mark.asyncio +async def test_router_embedding_path_rejects_past_max_parallel_requests_without_orphan_coroutines( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "sk-fake", + "api_base": "https://max-parallel-embed.local/v1", + "max_parallel_requests": 1, + }, + "model_info": {"id": "embed-capped-deployment"}, + } + ], + num_retries=0, + ) + + async def upstream(request: httpx.Request) -> httpx.Response: + await asyncio.sleep(0.2) + return httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + ) + + with respx.mock() as respx_mock, warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + route: Final = respx_mock.post("https://max-parallel-embed.local/v1/embeddings").mock(side_effect=upstream) + results: Final = await asyncio.wait_for( + asyncio.gather( + *(router.aembedding(model="embed", input=["hi"]) for _ in range(3)), + return_exceptions=True, + ), + timeout=10, + ) + gc.collect() + + rejected: Final = [r for r in results if isinstance(r, BaseException)] + assert len(rejected) == 2 and len(results) == 3 + assert all(isinstance(r, litellm.RateLimitError) and r.status_code == 429 for r in rejected) + assert all("embed-capped-deployment" in r.message for r in rejected) + assert route.call_count == 1 + assert [str(w.message) for w in caught if "never awaited" in str(w.message)] == [] + + +@pytest.mark.asyncio +async def test_router_max_parallel_requests_overflow_takes_the_ordinary_429_fallback_path( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = Router( + model_list=[ + { + "model_name": "gpt-5.6", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-fake", + "api_base": "https://max-parallel-primary.local/v1", + "max_parallel_requests": 1, + }, + "model_info": {"id": "capped-primary-deployment"}, + }, + { + "model_name": "gpt-5.6-fallback", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-fake", + "api_base": "https://max-parallel-fallback.local/v1", + }, + "model_info": {"id": "fallback-deployment"}, + }, + ], + fallbacks=[{"gpt-5.6": ["gpt-5.6-fallback"]}], + num_retries=0, + ) + + async def upstream(request: httpx.Request) -> httpx.Response: + await asyncio.sleep(0.2) + return httpx.Response( + 200, + json={ + "id": "c", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "x"}, "finish_reason": "stop"}], + }, + ) + + with respx.mock() as respx_mock: + primary: Final = respx_mock.post("https://max-parallel-primary.local/v1/chat/completions").mock( + side_effect=upstream + ) + fallback: Final = respx_mock.post("https://max-parallel-fallback.local/v1/chat/completions").mock( + side_effect=upstream + ) + results: Final = await asyncio.wait_for( + asyncio.gather( + *(router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) for _ in range(3)) + ), + timeout=10, + ) + + assert len(results) == 3 + assert primary.call_count == 1 + assert fallback.call_count == 2 + assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] + + +@pytest.mark.asyncio +async def test_router_deployment_slot_rejects_while_held_and_frees_slot_on_exit(): + router: Final = Router( + model_list=[ + { + "model_name": "gpt-5.6", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-fake", + "max_parallel_requests": 1, + }, + "model_info": {"id": "slot-deployment"}, + } + ] + ) + deployment: Final = router.get_deployment(model_id="slot-deployment") + assert deployment is not None + kwargs: Final = {"model": "gpt-5.6"} + + async with router._deployment_slot(deployment=deployment.model_dump(), kwargs=kwargs, parent_otel_span=None): + with pytest.raises(litellm.RateLimitError) as overflow: + async with router._deployment_slot(deployment=deployment.model_dump(), kwargs=kwargs, parent_otel_span=None): + pass + assert overflow.value.status_code == 429 + assert "slot-deployment" in overflow.value.message + + async with router._deployment_slot(deployment=deployment.model_dump(), kwargs=kwargs, parent_otel_span=None): + pass + + @pytest.mark.asyncio async def test_router_deployment_drop_params_string_true_is_honored(monkeypatch): from litellm import Router diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index be440dedca5..5c85c940b73 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -61,6 +61,7 @@ from litellm.utils import ( _snapshot_exception_for_hook, async_post_call_failure_deployment_hook, async_post_call_success_deployment_hook, + calculate_max_parallel_requests, client, get_non_default_completion_params, get_optional_params_image_gen, @@ -5774,3 +5775,32 @@ def test_register_model_without_a_provider_never_inherits_another_providers_entr assert info["output_cost_per_token"] == 4e-07 finally: _invalidate_model_cost_lowercase_map() + + +@pytest.mark.parametrize( + ("max_parallel_requests", "rpm", "tpm", "default_max_parallel_requests", "expected"), + [ + (3, 100, 100_000, 7, 3), + (None, 100, 100_000, 7, 100), + (None, None, 100_000, 7, 600), + (None, None, 50, 7, 1), + (None, None, None, 7, 7), + (None, None, None, None, None), + ], +) +def test_calculate_max_parallel_requests_precedence( + max_parallel_requests: int | None, + rpm: int | None, + tpm: int | None, + default_max_parallel_requests: int | None, + expected: int | None, +) -> None: + assert ( + calculate_max_parallel_requests( + max_parallel_requests=max_parallel_requests, + rpm=rpm, + tpm=tpm, + default_max_parallel_requests=default_max_parallel_requests, + ) + == expected + )